diff --git a/libs/enzyme/__init__.py b/libs/enzyme/__init__.py new file mode 100644 index 00000000..7bf572e9 --- /dev/null +++ b/libs/enzyme/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import mimetypes +import os +import sys +from exceptions import * + +PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']), + ('flv', ['video/flv'], ['flv']), + ('mkv', ['video/x-matroska', 'application/mkv'], ['mkv', 'mka', 'webm']), + ('mp4', ['video/quicktime', 'video/mp4'], ['mov', 'qt', 'mp4', 'mp4a', '3gp', '3gp2', '3g2', 'mk2']), + ('mpeg', ['video/mpeg'], ['mpeg', 'mpg', 'mp4', 'ts']), + ('ogm', ['application/ogg'], ['ogm', 'ogg', 'ogv']), + ('real', ['video/real'], ['rm', 'ra', 'ram']), + ('riff', ['video/avi'], ['wav', 'avi'])] + + +def parse(path): + if not os.path.isfile(path): + raise ValueError('Invalid path') + extension = os.path.splitext(path)[1][1:] + mimetype = mimetypes.guess_type(path)[0] + parser_ext = None + parser_mime = None + for (parser_name, parser_mimetypes, parser_extensions) in PARSERS: + if mimetype in parser_mimetypes: + parser_mime = parser_name + if extension in parser_extensions: + parser_ext = parser_name + parser = parser_mime or parser_ext + if not parser: + raise NoParserError() + mod = __import__(parser, globals=globals(), locals=locals(), fromlist=[], level=-1) + with open(path, 'rb') as f: + p = mod.Parser(f) + return p diff --git a/libs/enzyme/asf.py b/libs/enzyme/asf.py new file mode 100644 index 00000000..e7cca1b2 --- /dev/null +++ b/libs/enzyme/asf.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +import struct +import string +import logging +from exceptions import * +import core + +# get logging object +log = logging.getLogger(__name__) + +def _guid(input): + # Remove any '-' + s = string.join(string.split(input,'-'), '') + r = '' + if len(s) != 32: + return '' + x = '' + for i in range(0,16): + r+=chr(int(s[2*i:2*i+2],16)) + guid = struct.unpack('>IHHBB6s',r) + return guid + +GUIDS = { + 'ASF_Header_Object' : _guid('75B22630-668E-11CF-A6D9-00AA0062CE6C'), + 'ASF_Data_Object' : _guid('75B22636-668E-11CF-A6D9-00AA0062CE6C'), + 'ASF_Simple_Index_Object' : _guid('33000890-E5B1-11CF-89F4-00A0C90349CB'), + 'ASF_Index_Object' : _guid('D6E229D3-35DA-11D1-9034-00A0C90349BE'), + 'ASF_Media_Object_Index_Object' : _guid('FEB103F8-12AD-4C64-840F-2A1D2F7AD48C'), + 'ASF_Timecode_Index_Object' : _guid('3CB73FD0-0C4A-4803-953D-EDF7B6228F0C'), + + 'ASF_File_Properties_Object' : _guid('8CABDCA1-A947-11CF-8EE4-00C00C205365'), + 'ASF_Stream_Properties_Object' : _guid('B7DC0791-A9B7-11CF-8EE6-00C00C205365'), + 'ASF_Header_Extension_Object' : _guid('5FBF03B5-A92E-11CF-8EE3-00C00C205365'), + 'ASF_Codec_List_Object' : _guid('86D15240-311D-11D0-A3A4-00A0C90348F6'), + 'ASF_Script_Command_Object' : _guid('1EFB1A30-0B62-11D0-A39B-00A0C90348F6'), + 'ASF_Marker_Object' : _guid('F487CD01-A951-11CF-8EE6-00C00C205365'), + 'ASF_Bitrate_Mutual_Exclusion_Object' : _guid('D6E229DC-35DA-11D1-9034-00A0C90349BE'), + 'ASF_Error_Correction_Object' : _guid('75B22635-668E-11CF-A6D9-00AA0062CE6C'), + 'ASF_Content_Description_Object' : _guid('75B22633-668E-11CF-A6D9-00AA0062CE6C'), + 'ASF_Extended_Content_Description_Object' : _guid('D2D0A440-E307-11D2-97F0-00A0C95EA850'), + 'ASF_Content_Branding_Object' : _guid('2211B3FA-BD23-11D2-B4B7-00A0C955FC6E'), + 'ASF_Stream_Bitrate_Properties_Object' : _guid('7BF875CE-468D-11D1-8D82-006097C9A2B2'), + 'ASF_Content_Encryption_Object' : _guid('2211B3FB-BD23-11D2-B4B7-00A0C955FC6E'), + 'ASF_Extended_Content_Encryption_Object' : _guid('298AE614-2622-4C17-B935-DAE07EE9289C'), + 'ASF_Alt_Extended_Content_Encryption_Obj' : _guid('FF889EF1-ADEE-40DA-9E71-98704BB928CE'), + 'ASF_Digital_Signature_Object' : _guid('2211B3FC-BD23-11D2-B4B7-00A0C955FC6E'), + 'ASF_Padding_Object' : _guid('1806D474-CADF-4509-A4BA-9AABCB96AAE8'), + + 'ASF_Extended_Stream_Properties_Object' : _guid('14E6A5CB-C672-4332-8399-A96952065B5A'), + 'ASF_Advanced_Mutual_Exclusion_Object' : _guid('A08649CF-4775-4670-8A16-6E35357566CD'), + 'ASF_Group_Mutual_Exclusion_Object' : _guid('D1465A40-5A79-4338-B71B-E36B8FD6C249'), + 'ASF_Stream_Prioritization_Object' : _guid('D4FED15B-88D3-454F-81F0-ED5C45999E24'), + 'ASF_Bandwidth_Sharing_Object' : _guid('A69609E6-517B-11D2-B6AF-00C04FD908E9'), + 'ASF_Language_List_Object' : _guid('7C4346A9-EFE0-4BFC-B229-393EDE415C85'), + 'ASF_Metadata_Object' : _guid('C5F8CBEA-5BAF-4877-8467-AA8C44FA4CCA'), + 'ASF_Metadata_Library_Object' : _guid('44231C94-9498-49D1-A141-1D134E457054'), + 'ASF_Index_Parameters_Object' : _guid('D6E229DF-35DA-11D1-9034-00A0C90349BE'), + 'ASF_Media_Object_Index_Parameters_Obj' : _guid('6B203BAD-3F11-4E84-ACA8-D7613DE2CFA7'), + 'ASF_Timecode_Index_Parameters_Object' : _guid('F55E496D-9797-4B5D-8C8B-604DFE9BFB24'), + + 'ASF_Audio_Media' : _guid('F8699E40-5B4D-11CF-A8FD-00805F5C442B'), + 'ASF_Video_Media' : _guid('BC19EFC0-5B4D-11CF-A8FD-00805F5C442B'), + 'ASF_Command_Media' : _guid('59DACFC0-59E6-11D0-A3AC-00A0C90348F6'), + 'ASF_JFIF_Media' : _guid('B61BE100-5B4E-11CF-A8FD-00805F5C442B'), + 'ASF_Degradable_JPEG_Media' : _guid('35907DE0-E415-11CF-A917-00805F5C442B'), + 'ASF_File_Transfer_Media' : _guid('91BD222C-F21C-497A-8B6D-5AA86BFC0185'), + 'ASF_Binary_Media' : _guid('3AFB65E2-47EF-40F2-AC2C-70A90D71D343'), + + 'ASF_Web_Stream_Media_Subtype' : _guid('776257D4-C627-41CB-8F81-7AC7FF1C40CC'), + 'ASF_Web_Stream_Format' : _guid('DA1E6B13-8359-4050-B398-388E965BF00C'), + + 'ASF_No_Error_Correction' : _guid('20FB5700-5B55-11CF-A8FD-00805F5C442B'), + 'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220'), + } + + +class Asf(core.AVContainer): + """ + ASF video parser. The ASF format is also used for Microsft Windows + Media files like wmv. + """ + def __init__(self, file): + core.AVContainer.__init__(self) + self.mime = 'video/x-ms-asf' + self.type = 'asf format' + self._languages = [] + self._extinfo = {} + + h = file.read(30) + if len(h) < 30: + raise ParseError() + + (guidstr, objsize, objnum, reserved1, \ + reserved2) = struct.unpack('<16sQIBB',h) + guid = self._parseguid(guidstr) + + if (guid != GUIDS['ASF_Header_Object']): + raise ParseError() + if reserved1 != 0x01 or reserved2 != 0x02: + raise ParseError() + + log.debug("asf header size: %d / %d objects" % (objsize,objnum)) + header = file.read(objsize-30) + for i in range(0,objnum): + h = self._getnextheader(header) + header = header[h[1]:] + + del self._languages + del self._extinfo + + + def _findstream(self, id): + for stream in self.video + self.audio: + if stream.id == id: + return stream + + def _apply_extinfo(self, streamid): + stream = self._findstream(streamid) + if not stream or streamid not in self._extinfo: + return + stream.bitrate, stream.fps, langid, metadata = self._extinfo[streamid] + if langid is not None and langid >= 0 and langid < len(self._languages): + stream.language = self._languages[langid] + if metadata: + stream._appendtable('ASFMETADATA', metadata) + + + def _parseguid(self,string): + return struct.unpack('> 15 + if encrypted: + self._set('encrypted', True) + if streamtype == GUIDS['ASF_Video_Media']: + vi = core.VideoStream() + vi.width, vi.height, depth, codec, = struct.unpack('<4xII2xH4s', s[89:89+20]) + vi.codec = codec + vi.id = strno + self.video.append(vi) + elif streamtype == GUIDS['ASF_Audio_Media']: + ai = core.AudioStream() + twocc, ai.channels, ai.samplerate, bitrate, block, \ + ai.samplebits, = struct.unpack(' +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +import re +import logging +import fourcc +import language +from exceptions import * +from strutils import str_to_unicode, unicode_to_str + +UNPRINTABLE_KEYS = ['thumbnail', 'url', 'codec_private'] +EXTENSION_DEVICE = 'device' +EXTENSION_DIRECTORY = 'directory' +EXTENSION_STREAM = 'stream' +MEDIACORE = ['title', 'caption', 'comment', 'size', 'type', 'subtype', 'timestamp', + 'keywords', 'country', 'language', 'langcode', 'url', 'artist', + 'mime', 'datetime', 'tags', 'hash'] +AUDIOCORE = ['channels', 'samplerate', 'length', 'encoder', 'codec', 'format', + 'samplebits', 'bitrate', 'fourcc', 'trackno', 'id', 'userdate', + 'enabled', 'default', 'codec_private'] +MUSICCORE = ['trackof', 'album', 'genre', 'discs', 'thumbnail'] +VIDEOCORE = ['length', 'encoder', 'bitrate', 'samplerate', 'codec', 'format', + 'samplebits', 'width', 'height', 'fps', 'aspect', 'trackno', + 'fourcc', 'id', 'enabled', 'default', 'codec_private'] +AVCORE = ['length', 'encoder', 'trackno', 'trackof', 'copyright', 'product', + 'genre', 'writer', 'producer', 'studio', 'rating', 'actors', 'thumbnail', + 'delay', 'image', 'video', 'audio', 'subtitles', 'chapters', 'software', + 'summary', 'synopsis', 'season', 'episode', 'series'] + +# get logging object +log = logging.getLogger(__name__) + + +class Media(object): + media = None + + """ + Media is the base class to all Media Metadata Containers. It defines + the basic structures that handle metadata. Media and its derivates + contain a common set of metadata attributes that is listed in keys. + Specific derivates contain additional keys to the dublin core set that is + defined in Media. + """ + _keys = MEDIACORE + table_mapping = {} + + def __init__(self, hash=None): + if hash is not None: + # create Media based on dict + for key, value in hash.items(): + if isinstance(value, list) and value and isinstance(value[0], dict): + value = [Media(x) for x in value] + self._set(key, value) + return + + self._keys = self._keys[:] + self.tables = {} + # Tags, unlike tables, are more well-defined dicts whose values are + # either Tag objects, other dicts (for nested tags), or lists of either + # (for multiple instances of the tag, e.g. actor). Where possible, + # parsers should transform tag names to conform to the Official + # Matroska tags defined at http://www.matroska.org/technical/specs/tagging/index.html + # All tag names will be lower-cased. + self.tags = Tags() + for key in set(self._keys) - set(['media', 'tags']): + setattr(self, key, None) + + # + # unicode and string convertion for debugging + # + #TODO: Fix that mess + def __unicode__(self): + result = u'' + + # print normal attributes + lists = [] + for key in self._keys: + value = getattr(self, key, None) + if value == None or key == 'url': + continue + if isinstance(value, list): + if not value: + continue + elif isinstance(value[0], basestring): + # Just a list of strings (keywords?), so don't treat it specially. + value = u', '.join(value) + else: + lists.append((key, value)) + continue + elif isinstance(value, dict): + # Tables or tags treated separately. + continue + if key in UNPRINTABLE_KEYS: + value = '' % len(value) + result += u'| %10s: %s\n' % (unicode(key), unicode(value)) + + # print tags (recursively, to support nested tags). + def print_tags(tags, suffix, show_label): + result = '' + for n, (name, tag) in enumerate(tags.items()): + result += u'| %12s%s%s = ' % (u'tags: ' if n == 0 and show_label else '', suffix, name) + if isinstance(tag, list): + # TODO: doesn't support lists/dicts within lists. + result += u'%s\n' % ', '.join(subtag.value for subtag in tag) + else: + result += u'%s\n' % (tag.value or '') + if isinstance(tag, dict): + result += print_tags(tag, ' ', False) + return result + result += print_tags(self.tags, '', True) + + # print lists + for key, l in lists: + for n, item in enumerate(l): + label = '+-- ' + key.rstrip('s').capitalize() + if key not in ['tracks', 'subtitles', 'chapters']: + label += ' Track' + result += u'%s #%d\n' % (label, n + 1) + result += '| ' + re.sub(r'\n(.)', r'\n| \1', unicode(item)) + + # print tables + if log.level >= 10: + for name, table in self.tables.items(): + result += '+-- Table %s\n' % str(name) + for key, value in table.items(): + try: + value = unicode(value) + if len(value) > 50: + value = u'' % len(value) + except (UnicodeDecodeError, TypeError), e: + try: + value = u'' % len(value) + except AttributeError: + value = u'' + result += u'| | %s: %s\n' % (unicode(key), value) + return result + + def __str__(self): + return unicode(self).encode() + + def __repr__(self): + if hasattr(self, 'url'): + return '<%s %s>' % (str(self.__class__)[8:-2], self.url) + else: + return '<%s>' % (str(self.__class__)[8:-2]) + + # + # internal functions + # + def _appendtable(self, name, hashmap): + """ + Appends a tables of additional metadata to the Object. + If such a table already exists, the given tables items are + added to the existing one. + """ + if name not in self.tables: + self.tables[name] = hashmap + else: + # Append to the already existing table + for k in hashmap.keys(): + self.tables[name][k] = hashmap[k] + + def _set(self, key, value): + """ + Set key to value and add the key to the internal keys list if + missing. + """ + if value is None and getattr(self, key, None) is None: + return + if isinstance(value, str): + value = str_to_unicode(value) + setattr(self, key, value) + if not key in self._keys: + self._keys.append(key) + + def _set_url(self, url): + """ + Set the URL of the source + """ + self.url = url + + def _finalize(self): + """ + Correct same data based on specific rules + """ + # make sure all strings are unicode + for key in self._keys: + if key in UNPRINTABLE_KEYS: + continue + value = getattr(self, key) + if value is None: + continue + if key == 'image': + if isinstance(value, unicode): + setattr(self, key, unicode_to_str(value)) + continue + if isinstance(value, str): + setattr(self, key, str_to_unicode(value)) + if isinstance(value, unicode): + setattr(self, key, value.strip().rstrip().replace(u'\0', u'')) + if isinstance(value, list) and value and isinstance(value[0], Media): + for submenu in value: + submenu._finalize() + + # copy needed tags from tables + for name, table in self.tables.items(): + mapping = self.table_mapping.get(name, {}) + for tag, attr in mapping.items(): + if self.get(attr): + continue + value = table.get(tag, None) + if value is not None: + if not isinstance(value, (str, unicode)): + value = str_to_unicode(str(value)) + elif isinstance(value, str): + value = str_to_unicode(value) + value = value.strip().rstrip().replace(u'\0', u'') + setattr(self, attr, value) + + if 'fourcc' in self._keys and 'codec' in self._keys and self.codec is not None: + # Codec may be a fourcc, in which case we resolve it to its actual + # name and set the fourcc attribute. + self.fourcc, self.codec = fourcc.resolve(self.codec) + if 'language' in self._keys: + self.langcode, self.language = language.resolve(self.language) + + # + # data access + # + def __contains__(self, key): + """ + Test if key exists in the dict + """ + return hasattr(self, key) + + def get(self, attr, default=None): + """ + Returns the given attribute. If the attribute is not set by + the parser return 'default'. + """ + return getattr(self, attr, default) + + def __getitem__(self, attr): + """ + Get the value of the given attribute + """ + return getattr(self, attr, None) + + def __setitem__(self, key, value): + """ + Set the value of 'key' to 'value' + """ + setattr(self, key, value) + + def has_key(self, key): + """ + Check if the object has an attribute 'key' + """ + return hasattr(self, key) + + def convert(self): + """ + Convert Media to dict. + """ + result = {} + for k in self._keys: + value = getattr(self, k, None) + if isinstance(value, list) and value and isinstance(value[0], Media): + value = [x.convert() for x in value] + result[k] = value + return result + + def keys(self): + """ + Return all keys for the attributes set by the parser. + """ + return self._keys + + +class Collection(Media): + """ + Collection of Digial Media like CD, DVD, Directory, Playlist + """ + _keys = Media._keys + ['id', 'tracks'] + + def __init__(self): + Media.__init__(self) + self.tracks = [] + + +class Tag(object): + """ + An individual tag, which will be a value stored in a Tags object. + + Tag values are strings (for binary data), unicode objects, or datetime + objects for tags that represent dates or times. + """ + def __init__(self, value=None, langcode='und', binary=False): + super(Tag, self).__init__() + self.value = value + self.langcode = langcode + self.binary = binary + + def __unicode__(self): + return unicode(self.value) + + def __str__(self): + return str(self.value) + + def __repr__(self): + if not self.binary: + return '' % repr(self.value) + else: + return '' % len(self.value) + + @property + def langcode(self): + return self._langcode + + @langcode.setter + def langcode(self, code): + self._langcode, self.language = language.resolve(code) + + +class Tags(dict, Tag): + """ + A dictionary containing Tag objects. Values can be other Tags objects + (for nested tags), lists, or Tag objects. + + A Tags object is more or less a dictionary but it also contains a value. + This is necessary in order to represent this kind of tag specification + (e.g. for Matroska):: + + + LAW_RATING + PG + + COUNTRY + US + + + + The attribute RATING has a value (PG), but it also has a child tag + COUNTRY that specifies the country code the rating belongs to. + """ + def __init__(self, value=None, langcode='und', binary=False): + super(Tags, self).__init__() + self.value = value + self.langcode = langcode + self.binary = False + + +class AudioStream(Media): + """ + Audio Tracks in a Multiplexed Container. + """ + _keys = Media._keys + AUDIOCORE + + +class Music(AudioStream): + """ + Digital Music. + """ + _keys = AudioStream._keys + MUSICCORE + + def _finalize(self): + """ + Correct same data based on specific rules + """ + AudioStream._finalize(self) + if self.trackof: + try: + # XXX Why is this needed anyway? + if int(self.trackno) < 10: + self.trackno = u'0%s' % int(self.trackno) + except (AttributeError, ValueError): + pass + + +class VideoStream(Media): + """ + Video Tracks in a Multiplexed Container. + """ + _keys = Media._keys + VIDEOCORE + + +class Chapter(Media): + """ + Chapter in a Multiplexed Container. + """ + _keys = ['enabled', 'name', 'pos', 'id'] + + def __init__(self, name=None, pos=0): + Media.__init__(self) + self.name = name + self.pos = pos + self.enabled = True + + +class Subtitle(Media): + """ + Subtitle Tracks in a Multiplexed Container. + """ + _keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title', + 'id', 'codec'] + + def __init__(self, language=None): + Media.__init__(self) + self.language = language + + +class AVContainer(Media): + """ + Container for Audio and Video streams. This is the Container Type for + all media, that contain more than one stream. + """ + _keys = Media._keys + AVCORE + + def __init__(self): + Media.__init__(self) + self.audio = [] + self.video = [] + self.subtitles = [] + self.chapters = [] + + def _finalize(self): + """ + Correct same data based on specific rules + """ + Media._finalize(self) + if not self.length and len(self.video) and self.video[0].length: + self.length = 0 + # Length not specified for container, so use the largest length + # of its tracks as container length. + for track in self.video + self.audio: + if track.length: + self.length = max(self.length, track.length) diff --git a/libs/enzyme/exceptions.py b/libs/enzyme/exceptions.py new file mode 100644 index 00000000..403aa947 --- /dev/null +++ b/libs/enzyme/exceptions.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +class Error(Exception): + pass + + +class NoParserError(Error): + pass + + +class ParseError(Error): + pass diff --git a/libs/enzyme/flv.py b/libs/enzyme/flv.py new file mode 100644 index 00000000..c2dd4067 --- /dev/null +++ b/libs/enzyme/flv.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +from exceptions import * +import core +import logging +import struct + +# get logging object +log = logging.getLogger(__name__) + +FLV_TAG_TYPE_AUDIO = 0x08 +FLV_TAG_TYPE_VIDEO = 0x09 +FLV_TAG_TYPE_META = 0x12 + +# audio flags +FLV_AUDIO_CHANNEL_MASK = 0x01 +FLV_AUDIO_SAMPLERATE_MASK = 0x0c +FLV_AUDIO_CODECID_MASK = 0xf0 + +FLV_AUDIO_SAMPLERATE_OFFSET = 2 +FLV_AUDIO_CODECID_OFFSET = 4 +FLV_AUDIO_CODECID = (0x0001, 0x0002, 0x0055, 0x0001) + +# video flags +FLV_VIDEO_CODECID_MASK = 0x0f +FLV_VIDEO_CODECID = ( 'FLV1', 'MSS1', 'VP60') # wild guess + +FLV_DATA_TYPE_NUMBER = 0x00 +FLV_DATA_TYPE_BOOL = 0x01 +FLV_DATA_TYPE_STRING = 0x02 +FLV_DATA_TYPE_OBJECT = 0x03 +FLC_DATA_TYPE_CLIP = 0x04 +FLV_DATA_TYPE_REFERENCE = 0x07 +FLV_DATA_TYPE_ECMARRAY = 0x08 +FLV_DATA_TYPE_ENDOBJECT = 0x09 +FLV_DATA_TYPE_ARRAY = 0x0a +FLV_DATA_TYPE_DATE = 0x0b +FLV_DATA_TYPE_LONGSTRING = 0x0c + +FLVINFO = { + 'creator': 'copyright', +} + +class FlashVideo(core.AVContainer): + """ + Experimental parser for Flash videos. It requires certain flags to + be set to report video resolutions and in most cases it does not + provide that information. + """ + table_mapping = { 'FLVINFO' : FLVINFO } + + def __init__(self,file): + core.AVContainer.__init__(self) + self.mime = 'video/flv' + self.type = 'Flash Video' + data = file.read(13) + if len(data) < 13 or struct.unpack('>3sBBII', data)[0] != 'FLV': + raise ParseError() + + for i in range(10): + if self.audio and self.video: + break + data = file.read(11) + if len(data) < 11: + break + chunk = struct.unpack('>BH4BI', data) + size = (chunk[1] << 8) + chunk[2] + + if chunk[0] == FLV_TAG_TYPE_AUDIO: + flags = ord(file.read(1)) + if not self.audio: + a = core.AudioStream() + a.channels = (flags & FLV_AUDIO_CHANNEL_MASK) + 1 + srate = (flags & FLV_AUDIO_SAMPLERATE_MASK) + a.samplerate = (44100 << (srate >> FLV_AUDIO_SAMPLERATE_OFFSET) >> 3) + codec = (flags & FLV_AUDIO_CODECID_MASK) >> FLV_AUDIO_CODECID_OFFSET + if codec < len(FLV_AUDIO_CODECID): + a.codec = FLV_AUDIO_CODECID[codec] + self.audio.append(a) + + file.seek(size - 1, 1) + + elif chunk[0] == FLV_TAG_TYPE_VIDEO: + flags = ord(file.read(1)) + if not self.video: + v = core.VideoStream() + codec = (flags & FLV_VIDEO_CODECID_MASK) - 2 + if codec < len(FLV_VIDEO_CODECID): + v.codec = FLV_VIDEO_CODECID[codec] + # width and height are in the meta packet, but I have + # no file with such a packet inside. So maybe we have + # to decode some parts of the video. + self.video.append(v) + + file.seek(size - 1, 1) + + elif chunk[0] == FLV_TAG_TYPE_META: + log.info('metadata %s', str(chunk)) + metadata = file.read(size) + try: + while metadata: + length, value = self._parse_value(metadata) + if isinstance(value, dict): + log.info('metadata: %s', value) + if value.get('creator'): + self.copyright = value.get('creator') + if value.get('width'): + self.width = value.get('width') + if value.get('height'): + self.height = value.get('height') + if value.get('duration'): + self.length = value.get('duration') + self._appendtable('FLVINFO', value) + if not length: + # parse error + break + metadata = metadata[length:] + except (IndexError, struct.error, TypeError): + pass + else: + log.info('unkown %s', str(chunk)) + file.seek(size, 1) + + file.seek(4, 1) + + def _parse_value(self, data): + """ + Parse the next metadata value. + """ + if ord(data[0]) == FLV_DATA_TYPE_NUMBER: + value = struct.unpack('>d', data[1:9])[0] + return 9, value + + if ord(data[0]) == FLV_DATA_TYPE_BOOL: + return 2, bool(data[1]) + + if ord(data[0]) == FLV_DATA_TYPE_STRING: + length = (ord(data[1]) << 8) + ord(data[2]) + return length + 3, data[3:length+3] + + if ord(data[0]) == FLV_DATA_TYPE_ECMARRAY: + init_length = len(data) + num = struct.unpack('>I', data[1:5])[0] + data = data[5:] + result = {} + for i in range(num): + length = (ord(data[0]) << 8) + ord(data[1]) + key = data[2:length+2] + data = data[length + 2:] + length, value = self._parse_value(data) + if not length: + return 0, result + result[key] = value + data = data[length:] + return init_length - len(data), result + + log.info('unknown code: %x. Stop metadata parser', ord(data[0])) + return 0, None + + +Parser = FlashVideo diff --git a/libs/enzyme/fourcc.py b/libs/enzyme/fourcc.py new file mode 100644 index 00000000..60cbb3c8 --- /dev/null +++ b/libs/enzyme/fourcc.py @@ -0,0 +1,853 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +import string +import re +import struct + +__all__ = ['resolve'] + + +def resolve(code): + """ + Transform a twocc or fourcc code into a name. Returns a 2-tuple of (cc, + codec) where both are strings and cc is a string in the form '0xXX' if it's + a twocc, or 'ABCD' if it's a fourcc. If the given code is not a known + twocc or fourcc, the return value will be (None, 'Unknown'), unless the + code is otherwise a printable string in which case it will be returned as + the codec. + """ + if isinstance(code, basestring): + codec = u'Unknown' + # Check for twocc + if re.match(r'^0x[\da-f]{1,4}$', code, re.I): + # Twocc in hex form + return code, TWOCC.get(int(code, 16), codec) + elif code.isdigit() and 0 <= int(code) <= 0xff: + # Twocc in decimal form + return hex(int(code)), TWOCC.get(int(code), codec) + elif len(code) == 2: + code = struct.unpack('H', code)[0] + return hex(code), TWOCC.get(code, codec) + elif len(code) != 4 and len([x for x in code if x not in string.printable]) == 0: + # Code is a printable string. + codec = unicode(code) + + if code[:2] == 'MS' and code[2:].upper() in FOURCC: + code = code[2:] + + if code.upper() in FOURCC: + return code.upper(), unicode(FOURCC[code.upper()]) + return None, codec + elif isinstance(code, (int, long)): + return hex(code), TWOCC.get(code, u'Unknown') + + return None, u'Unknown' + + +TWOCC = { + 0x0000: 'Unknown Wave Format', + 0x0001: 'PCM', + 0x0002: 'Microsoft ADPCM', + 0x0003: 'IEEE Float', + 0x0004: 'Compaq Computer VSELP', + 0x0005: 'IBM CVSD', + 0x0006: 'A-Law', + 0x0007: 'mu-Law', + 0x0008: 'Microsoft DTS', + 0x0009: 'Microsoft DRM', + 0x0010: 'OKI ADPCM', + 0x0011: 'Intel DVI/IMA ADPCM', + 0x0012: 'Videologic MediaSpace ADPCM', + 0x0013: 'Sierra Semiconductor ADPCM', + 0x0014: 'Antex Electronics G.723 ADPCM', + 0x0015: 'DSP Solutions DigiSTD', + 0x0016: 'DSP Solutions DigiFIX', + 0x0017: 'Dialogic OKI ADPCM', + 0x0018: 'MediaVision ADPCM', + 0x0019: 'Hewlett-Packard CU', + 0x0020: 'Yamaha ADPCM', + 0x0021: 'Speech Compression Sonarc', + 0x0022: 'DSP Group TrueSpeech', + 0x0023: 'Echo Speech EchoSC1', + 0x0024: 'Audiofile AF36', + 0x0025: 'Audio Processing Technology APTX', + 0x0026: 'AudioFile AF10', + 0x0027: 'Prosody 1612', + 0x0028: 'LRC', + 0x0030: 'Dolby AC2', + 0x0031: 'Microsoft GSM 6.10', + 0x0032: 'MSNAudio', + 0x0033: 'Antex Electronics ADPCME', + 0x0034: 'Control Resources VQLPC', + 0x0035: 'DSP Solutions DigiREAL', + 0x0036: 'DSP Solutions DigiADPCM', + 0x0037: 'Control Resources CR10', + 0x0038: 'Natural MicroSystems VBXADPCM', + 0x0039: 'Crystal Semiconductor IMA ADPCM', + 0x003A: 'EchoSC3', + 0x003B: 'Rockwell ADPCM', + 0x003C: 'Rockwell Digit LK', + 0x003D: 'Xebec', + 0x0040: 'Antex Electronics G.721 ADPCM', + 0x0041: 'G.728 CELP', + 0x0042: 'MSG723', + 0x0043: 'IBM AVC ADPCM', + 0x0045: 'ITU-T G.726 ADPCM', + 0x0050: 'MPEG 1, Layer 1,2', + 0x0052: 'RT24', + 0x0053: 'PAC', + 0x0055: 'MPEG Layer 3', + 0x0059: 'Lucent G.723', + 0x0060: 'Cirrus', + 0x0061: 'ESPCM', + 0x0062: 'Voxware', + 0x0063: 'Canopus Atrac', + 0x0064: 'G.726 ADPCM', + 0x0065: 'G.722 ADPCM', + 0x0066: 'DSAT', + 0x0067: 'DSAT Display', + 0x0069: 'Voxware Byte Aligned', + 0x0070: 'Voxware AC8', + 0x0071: 'Voxware AC10', + 0x0072: 'Voxware AC16', + 0x0073: 'Voxware AC20', + 0x0074: 'Voxware MetaVoice', + 0x0075: 'Voxware MetaSound', + 0x0076: 'Voxware RT29HW', + 0x0077: 'Voxware VR12', + 0x0078: 'Voxware VR18', + 0x0079: 'Voxware TQ40', + 0x0080: 'Softsound', + 0x0081: 'Voxware TQ60', + 0x0082: 'MSRT24', + 0x0083: 'G.729A', + 0x0084: 'MVI MV12', + 0x0085: 'DF G.726', + 0x0086: 'DF GSM610', + 0x0088: 'ISIAudio', + 0x0089: 'Onlive', + 0x0091: 'SBC24', + 0x0092: 'Dolby AC3 SPDIF', + 0x0093: 'MediaSonic G.723', + 0x0094: 'Aculab PLC Prosody 8KBPS', + 0x0097: 'ZyXEL ADPCM', + 0x0098: 'Philips LPCBB', + 0x0099: 'Packed', + 0x00A0: 'Malden Electronics PHONYTALK', + 0x00FF: 'AAC', + 0x0100: 'Rhetorex ADPCM', + 0x0101: 'IBM mu-law', + 0x0102: 'IBM A-law', + 0x0103: 'IBM AVC Adaptive Differential Pulse Code Modulation', + 0x0111: 'Vivo G.723', + 0x0112: 'Vivo Siren', + 0x0123: 'Digital G.723', + 0x0125: 'Sanyo LD ADPCM', + 0x0130: 'Sipro Lab Telecom ACELP.net', + 0x0131: 'Sipro Lab Telecom ACELP.4800', + 0x0132: 'Sipro Lab Telecom ACELP.8V3', + 0x0133: 'Sipro Lab Telecom ACELP.G.729', + 0x0134: 'Sipro Lab Telecom ACELP.G.729A', + 0x0135: 'Sipro Lab Telecom ACELP.KELVIN', + 0x0140: 'Windows Media Video V8', + 0x0150: 'Qualcomm PureVoice', + 0x0151: 'Qualcomm HalfRate', + 0x0155: 'Ring Zero Systems TUB GSM', + 0x0160: 'Windows Media Audio V1 / DivX audio (WMA)', + 0x0161: 'Windows Media Audio V7 / V8 / V9', + 0x0162: 'Windows Media Audio Professional V9', + 0x0163: 'Windows Media Audio Lossless V9', + 0x0170: 'UNISYS NAP ADPCM', + 0x0171: 'UNISYS NAP ULAW', + 0x0172: 'UNISYS NAP ALAW', + 0x0173: 'UNISYS NAP 16K', + 0x0200: 'Creative Labs ADPCM', + 0x0202: 'Creative Labs Fastspeech8', + 0x0203: 'Creative Labs Fastspeech10', + 0x0210: 'UHER Informatic ADPCM', + 0x0215: 'Ulead DV ACM', + 0x0216: 'Ulead DV ACM', + 0x0220: 'Quarterdeck', + 0x0230: 'I-link Worldwide ILINK VC', + 0x0240: 'Aureal Semiconductor RAW SPORT', + 0x0241: 'ESST AC3', + 0x0250: 'Interactive Products HSX', + 0x0251: 'Interactive Products RPELP', + 0x0260: 'Consistent Software CS2', + 0x0270: 'Sony ATRAC3 (SCX, same as MiniDisk LP2)', + 0x0300: 'Fujitsu FM Towns Snd', + 0x0400: 'BTV Digital', + 0x0401: 'Intel Music Coder (IMC)', + 0x0402: 'Ligos Indeo Audio', + 0x0450: 'QDesign Music', + 0x0680: 'VME VMPCM', + 0x0681: 'AT&T Labs TPC', + 0x0700: 'YMPEG Alpha', + 0x08AE: 'ClearJump LiteWave', + 0x1000: 'Olivetti GSM', + 0x1001: 'Olivetti ADPCM', + 0x1002: 'Olivetti CELP', + 0x1003: 'Olivetti SBC', + 0x1004: 'Olivetti OPR', + 0x1100: 'Lernout & Hauspie LH Codec', + 0x1101: 'Lernout & Hauspie CELP codec', + 0x1102: 'Lernout & Hauspie SBC codec', + 0x1103: 'Lernout & Hauspie SBC codec', + 0x1104: 'Lernout & Hauspie SBC codec', + 0x1400: 'Norris', + 0x1401: 'AT&T ISIAudio', + 0x1500: 'Soundspace Music Compression', + 0x181C: 'VoxWare RT24 speech codec', + 0x181E: 'Lucent elemedia AX24000P Music codec', + 0x1C07: 'Lucent SX8300P speech codec', + 0x1C0C: 'Lucent SX5363S G.723 compliant codec', + 0x1F03: 'CUseeMe DigiTalk (ex-Rocwell)', + 0x1FC4: 'NCT Soft ALF2CD ACM', + 0x2000: 'AC3', + 0x2001: 'Dolby DTS (Digital Theater System)', + 0x2002: 'RealAudio 1 / 2 14.4', + 0x2003: 'RealAudio 1 / 2 28.8', + 0x2004: 'RealAudio G2 / 8 Cook (low bitrate)', + 0x2005: 'RealAudio 3 / 4 / 5 Music (DNET)', + 0x2006: 'RealAudio 10 AAC (RAAC)', + 0x2007: 'RealAudio 10 AAC+ (RACP)', + 0x3313: 'makeAVIS', + 0x4143: 'Divio MPEG-4 AAC audio', + 0x434C: 'LEAD Speech', + 0x564C: 'LEAD Vorbis', + 0x674F: 'Ogg Vorbis (mode 1)', + 0x6750: 'Ogg Vorbis (mode 2)', + 0x6751: 'Ogg Vorbis (mode 3)', + 0x676F: 'Ogg Vorbis (mode 1+)', + 0x6770: 'Ogg Vorbis (mode 2+)', + 0x6771: 'Ogg Vorbis (mode 3+)', + 0x7A21: 'GSM-AMR (CBR, no SID)', + 0x7A22: 'GSM-AMR (VBR, including SID)', + 0xDFAC: 'DebugMode SonicFoundry Vegas FrameServer ACM Codec', + 0xF1AC: 'Free Lossless Audio Codec FLAC', + 0xFFFE: 'Extensible wave format', + 0xFFFF: 'development' +} + + +FOURCC = { + '1978': 'A.M.Paredes predictor (LossLess)', + '2VUY': 'Optibase VideoPump 8-bit 4:2:2 Component YCbCr', + '3IV0': 'MPEG4-based codec 3ivx', + '3IV1': '3ivx v1', + '3IV2': '3ivx v2', + '3IVD': 'FFmpeg DivX ;-) (MS MPEG-4 v3)', + '3IVX': 'MPEG4-based codec 3ivx', + '8BPS': 'Apple QuickTime Planar RGB with Alpha-channel', + 'AAS4': 'Autodesk Animator codec (RLE)', + 'AASC': 'Autodesk Animator', + 'ABYR': 'Kensington ABYR', + 'ACTL': 'Streambox ACT-L2', + 'ADV1': 'Loronix WaveCodec', + 'ADVJ': 'Avid M-JPEG Avid Technology Also known as AVRn', + 'AEIK': 'Intel Indeo Video 3.2', + 'AEMI': 'Array VideoONE MPEG1-I Capture', + 'AFLC': 'Autodesk Animator FLC', + 'AFLI': 'Autodesk Animator FLI', + 'AHDV': 'CineForm 10-bit Visually Perfect HD', + 'AJPG': '22fps JPEG-based codec for digital cameras', + 'AMPG': 'Array VideoONE MPEG', + 'ANIM': 'Intel RDX (ANIM)', + 'AP41': 'AngelPotion Definitive', + 'AP42': 'AngelPotion Definitive', + 'ASLC': 'AlparySoft Lossless Codec', + 'ASV1': 'Asus Video v1', + 'ASV2': 'Asus Video v2', + 'ASVX': 'Asus Video 2.0 (audio)', + 'ATM4': 'Ahead Nero Digital MPEG-4 Codec', + 'AUR2': 'Aura 2 Codec - YUV 4:2:2', + 'AURA': 'Aura 1 Codec - YUV 4:1:1', + 'AV1X': 'Avid 1:1x (Quick Time)', + 'AVC1': 'H.264 AVC', + 'AVD1': 'Avid DV (Quick Time)', + 'AVDJ': 'Avid Meridien JFIF with Alpha-channel', + 'AVDN': 'Avid DNxHD (Quick Time)', + 'AVDV': 'Avid DV', + 'AVI1': 'MainConcept Motion JPEG Codec', + 'AVI2': 'MainConcept Motion JPEG Codec', + 'AVID': 'Avid Motion JPEG', + 'AVIS': 'Wrapper for AviSynth', + 'AVMP': 'Avid IMX (Quick Time)', + 'AVR ': 'Avid ABVB/NuVista MJPEG with Alpha-channel', + 'AVRN': 'Avid Motion JPEG', + 'AVUI': 'Avid Meridien Uncompressed with Alpha-channel', + 'AVUP': 'Avid 10bit Packed (Quick Time)', + 'AYUV': '4:4:4 YUV (AYUV)', + 'AZPR': 'Quicktime Apple Video', + 'AZRP': 'Quicktime Apple Video', + 'BGR ': 'Uncompressed BGR32 8:8:8:8', + 'BGR(15)': 'Uncompressed BGR15 5:5:5', + 'BGR(16)': 'Uncompressed BGR16 5:6:5', + 'BGR(24)': 'Uncompressed BGR24 8:8:8', + 'BHIV': 'BeHere iVideo', + 'BINK': 'RAD Game Tools Bink Video', + 'BIT ': 'BI_BITFIELDS (Raw RGB)', + 'BITM': 'Microsoft H.261', + 'BLOX': 'Jan Jezabek BLOX MPEG Codec', + 'BLZ0': 'DivX for Blizzard Decoder Filter', + 'BT20': 'Conexant Prosumer Video', + 'BTCV': 'Conexant Composite Video Codec', + 'BTVC': 'Conexant Composite Video', + 'BW00': 'BergWave (Wavelet)', + 'BW10': 'Data Translation Broadway MPEG Capture', + 'BXBG': 'BOXX BGR', + 'BXRG': 'BOXX RGB', + 'BXY2': 'BOXX 10-bit YUV', + 'BXYV': 'BOXX YUV', + 'CC12': 'Intel YUV12', + 'CDV5': 'Canopus SD50/DVHD', + 'CDVC': 'Canopus DV', + 'CDVH': 'Canopus SD50/DVHD', + 'CFCC': 'Digital Processing Systems DPS Perception', + 'CFHD': 'CineForm 10-bit Visually Perfect HD', + 'CGDI': 'Microsoft Office 97 Camcorder Video', + 'CHAM': 'Winnov Caviara Champagne', + 'CJPG': 'Creative WebCam JPEG', + 'CLJR': 'Cirrus Logic YUV 4 pixels', + 'CLLC': 'Canopus LossLess', + 'CLPL': 'YV12', + 'CMYK': 'Common Data Format in Printing', + 'COL0': 'FFmpeg DivX ;-) (MS MPEG-4 v3)', + 'COL1': 'FFmpeg DivX ;-) (MS MPEG-4 v3)', + 'CPLA': 'Weitek 4:2:0 YUV Planar', + 'CRAM': 'Microsoft Video 1 (CRAM)', + 'CSCD': 'RenderSoft CamStudio lossless Codec', + 'CTRX': 'Citrix Scalable Video Codec', + 'CUVC': 'Canopus HQ', + 'CVID': 'Radius Cinepak', + 'CWLT': 'Microsoft Color WLT DIB', + 'CYUV': 'Creative Labs YUV', + 'CYUY': 'ATI YUV', + 'D261': 'H.261', + 'D263': 'H.263', + 'DAVC': 'Dicas MPEGable H.264/MPEG-4 AVC base profile codec', + 'DC25': 'MainConcept ProDV Codec', + 'DCAP': 'Pinnacle DV25 Codec', + 'DCL1': 'Data Connection Conferencing Codec', + 'DCT0': 'WniWni Codec', + 'DFSC': 'DebugMode FrameServer VFW Codec', + 'DIB ': 'Full Frames (Uncompressed)', + 'DIV1': 'FFmpeg-4 V1 (hacked MS MPEG-4 V1)', + 'DIV2': 'MS MPEG-4 V2', + 'DIV3': 'DivX v3 MPEG-4 Low-Motion', + 'DIV4': 'DivX v3 MPEG-4 Fast-Motion', + 'DIV5': 'DIV5', + 'DIV6': 'DivX MPEG-4', + 'DIVX': 'DivX', + 'DM4V': 'Dicas MPEGable MPEG-4', + 'DMB1': 'Matrox Rainbow Runner hardware MJPEG', + 'DMB2': 'Paradigm MJPEG', + 'DMK2': 'ViewSonic V36 PDA Video', + 'DP02': 'DynaPel MPEG-4', + 'DPS0': 'DPS Reality Motion JPEG', + 'DPSC': 'DPS PAR Motion JPEG', + 'DRWX': 'Pinnacle DV25 Codec', + 'DSVD': 'DSVD', + 'DTMT': 'Media-100 Codec', + 'DTNT': 'Media-100 Codec', + 'DUCK': 'Duck True Motion 1.0', + 'DV10': 'BlueFish444 (lossless RGBA, YUV 10-bit)', + 'DV25': 'Matrox DVCPRO codec', + 'DV50': 'Matrox DVCPRO50 codec', + 'DVAN': 'DVAN', + 'DVC ': 'Apple QuickTime DV (DVCPRO NTSC)', + 'DVCP': 'Apple QuickTime DV (DVCPRO PAL)', + 'DVCS': 'MainConcept DV Codec', + 'DVE2': 'InSoft DVE-2 Videoconferencing', + 'DVH1': 'Pinnacle DVHD100', + 'DVHD': 'DV 1125 lines at 30.00 Hz or 1250 lines at 25.00 Hz', + 'DVIS': 'VSYNC DualMoon Iris DV codec', + 'DVL ': 'Radius SoftDV 16:9 NTSC', + 'DVLP': 'Radius SoftDV 16:9 PAL', + 'DVMA': 'Darim Vision DVMPEG', + 'DVOR': 'BlueFish444 (lossless RGBA, YUV 10-bit)', + 'DVPN': 'Apple QuickTime DV (DV NTSC)', + 'DVPP': 'Apple QuickTime DV (DV PAL)', + 'DVR1': 'TARGA2000 Codec', + 'DVRS': 'VSYNC DualMoon Iris DV codec', + 'DVSD': 'DV', + 'DVSL': 'DV compressed in SD (SDL)', + 'DVX1': 'DVX1000SP Video Decoder', + 'DVX2': 'DVX2000S Video Decoder', + 'DVX3': 'DVX3000S Video Decoder', + 'DX50': 'DivX v5', + 'DXGM': 'Electronic Arts Game Video codec', + 'DXSB': 'DivX Subtitles Codec', + 'DXT1': 'Microsoft DirectX Compressed Texture (DXT1)', + 'DXT2': 'Microsoft DirectX Compressed Texture (DXT2)', + 'DXT3': 'Microsoft DirectX Compressed Texture (DXT3)', + 'DXT4': 'Microsoft DirectX Compressed Texture (DXT4)', + 'DXT5': 'Microsoft DirectX Compressed Texture (DXT5)', + 'DXTC': 'Microsoft DirectX Compressed Texture (DXTC)', + 'DXTN': 'Microsoft DirectX Compressed Texture (DXTn)', + 'EKQ0': 'Elsa EKQ0', + 'ELK0': 'Elsa ELK0', + 'EM2V': 'Etymonix MPEG-2 I-frame', + 'EQK0': 'Elsa graphics card quick codec', + 'ESCP': 'Eidos Escape', + 'ETV1': 'eTreppid Video ETV1', + 'ETV2': 'eTreppid Video ETV2', + 'ETVC': 'eTreppid Video ETVC', + 'FFDS': 'FFDShow supported', + 'FFV1': 'FFDShow supported', + 'FFVH': 'FFVH codec', + 'FLIC': 'Autodesk FLI/FLC Animation', + 'FLJP': 'D-Vision Field Encoded Motion JPEG', + 'FLV1': 'FLV1 codec', + 'FMJP': 'D-Vision fieldbased ISO MJPEG', + 'FRLE': 'SoftLab-NSK Y16 + Alpha RLE', + 'FRWA': 'SoftLab-Nsk Forward Motion JPEG w/ alpha channel', + 'FRWD': 'SoftLab-Nsk Forward Motion JPEG', + 'FRWT': 'SoftLab-NSK Vision Forward Motion JPEG with Alpha-channel', + 'FRWU': 'SoftLab-NSK Vision Forward Uncompressed', + 'FVF1': 'Iterated Systems Fractal Video Frame', + 'FVFW': 'ff MPEG-4 based on XviD codec', + 'GEPJ': 'White Pine (ex Paradigm Matrix) Motion JPEG Codec', + 'GJPG': 'Grand Tech GT891x Codec', + 'GLCC': 'GigaLink AV Capture codec', + 'GLZW': 'Motion LZW', + 'GPEG': 'Motion JPEG', + 'GPJM': 'Pinnacle ReelTime MJPEG Codec', + 'GREY': 'Apparently a duplicate of Y800', + 'GWLT': 'Microsoft Greyscale WLT DIB', + 'H260': 'H.260', + 'H261': 'H.261', + 'H262': 'H.262', + 'H263': 'H.263', + 'H264': 'H.264 AVC', + 'H265': 'H.265', + 'H266': 'H.266', + 'H267': 'H.267', + 'H268': 'H.268', + 'H269': 'H.269', + 'HD10': 'BlueFish444 (lossless RGBA, YUV 10-bit)', + 'HDX4': 'Jomigo HDX4', + 'HFYU': 'Huffman Lossless Codec', + 'HMCR': 'Rendition Motion Compensation Format (HMCR)', + 'HMRR': 'Rendition Motion Compensation Format (HMRR)', + 'I263': 'Intel ITU H.263 Videoconferencing (i263)', + 'I420': 'Intel Indeo 4', + 'IAN ': 'Intel RDX', + 'ICLB': 'InSoft CellB Videoconferencing', + 'IDM0': 'IDM Motion Wavelets 2.0', + 'IF09': 'Microsoft H.261', + 'IGOR': 'Power DVD', + 'IJPG': 'Intergraph JPEG', + 'ILVC': 'Intel Layered Video', + 'ILVR': 'ITU-T H.263+', + 'IMC1': 'IMC1', + 'IMC2': 'IMC2', + 'IMC3': 'IMC3', + 'IMC4': 'IMC4', + 'IMJG': 'Accom SphereOUS MJPEG with Alpha-channel', + 'IPDV': 'I-O Data Device Giga AVI DV Codec', + 'IPJ2': 'Image Power JPEG2000', + 'IR21': 'Intel Indeo 2.1', + 'IRAW': 'Intel YUV Uncompressed', + 'IUYV': 'Interlaced version of UYVY (line order 0,2,4 then 1,3,5 etc)', + 'IV30': 'Ligos Indeo 3.0', + 'IV31': 'Ligos Indeo 3.1', + 'IV32': 'Ligos Indeo 3.2', + 'IV33': 'Ligos Indeo 3.3', + 'IV34': 'Ligos Indeo 3.4', + 'IV35': 'Ligos Indeo 3.5', + 'IV36': 'Ligos Indeo 3.6', + 'IV37': 'Ligos Indeo 3.7', + 'IV38': 'Ligos Indeo 3.8', + 'IV39': 'Ligos Indeo 3.9', + 'IV40': 'Ligos Indeo Interactive 4.0', + 'IV41': 'Ligos Indeo Interactive 4.1', + 'IV42': 'Ligos Indeo Interactive 4.2', + 'IV43': 'Ligos Indeo Interactive 4.3', + 'IV44': 'Ligos Indeo Interactive 4.4', + 'IV45': 'Ligos Indeo Interactive 4.5', + 'IV46': 'Ligos Indeo Interactive 4.6', + 'IV47': 'Ligos Indeo Interactive 4.7', + 'IV48': 'Ligos Indeo Interactive 4.8', + 'IV49': 'Ligos Indeo Interactive 4.9', + 'IV50': 'Ligos Indeo Interactive 5.0', + 'IY41': 'Interlaced version of Y41P (line order 0,2,4,...,1,3,5...)', + 'IYU1': '12 bit format used in mode 2 of the IEEE 1394 Digital Camera 1.04 spec', + 'IYU2': '24 bit format used in mode 2 of the IEEE 1394 Digital Camera 1.04 spec', + 'IYUV': 'Intel Indeo iYUV 4:2:0', + 'JBYR': 'Kensington JBYR', + 'JFIF': 'Motion JPEG (FFmpeg)', + 'JPEG': 'Still Image JPEG DIB', + 'JPG ': 'JPEG compressed', + 'JPGL': 'Webcam JPEG Light', + 'KMVC': 'Karl Morton\'s Video Codec', + 'KPCD': 'Kodak Photo CD', + 'L261': 'Lead Technologies H.261', + 'L263': 'Lead Technologies H.263', + 'LAGS': 'Lagarith LossLess', + 'LBYR': 'Creative WebCam codec', + 'LCMW': 'Lead Technologies Motion CMW Codec', + 'LCW2': 'LEADTools MCMW 9Motion Wavelet)', + 'LEAD': 'LEAD Video Codec', + 'LGRY': 'Lead Technologies Grayscale Image', + 'LJ2K': 'LEADTools JPEG2000', + 'LJPG': 'LEAD MJPEG Codec', + 'LMP2': 'LEADTools MPEG2', + 'LOCO': 'LOCO Lossless Codec', + 'LSCR': 'LEAD Screen Capture', + 'LSVM': 'Vianet Lighting Strike Vmail (Streaming)', + 'LZO1': 'LZO compressed (lossless codec)', + 'M261': 'Microsoft H.261', + 'M263': 'Microsoft H.263', + 'M4CC': 'ESS MPEG4 Divio codec', + 'M4S2': 'Microsoft MPEG-4 (M4S2)', + 'MC12': 'ATI Motion Compensation Format (MC12)', + 'MC24': 'MainConcept Motion JPEG Codec', + 'MCAM': 'ATI Motion Compensation Format (MCAM)', + 'MCZM': 'Theory MicroCosm Lossless 64bit RGB with Alpha-channel', + 'MDVD': 'Alex MicroDVD Video (hacked MS MPEG-4)', + 'MDVF': 'Pinnacle DV/DV50/DVHD100', + 'MHFY': 'A.M.Paredes mhuffyYUV (LossLess)', + 'MJ2C': 'Morgan Multimedia Motion JPEG2000', + 'MJPA': 'Pinnacle ReelTime MJPG hardware codec', + 'MJPB': 'Motion JPEG codec', + 'MJPG': 'Motion JPEG DIB', + 'MJPX': 'Pegasus PICVideo Motion JPEG', + 'MMES': 'Matrox MPEG-2 I-frame', + 'MNVD': 'MindBend MindVid LossLess', + 'MP2A': 'MPEG-2 Audio', + 'MP2T': 'MPEG-2 Transport Stream', + 'MP2V': 'MPEG-2 Video', + 'MP41': 'Microsoft MPEG-4 V1 (enhansed H263)', + 'MP42': 'Microsoft MPEG-4 (low-motion)', + 'MP43': 'Microsoft MPEG-4 (fast-motion)', + 'MP4A': 'MPEG-4 Audio', + 'MP4S': 'Microsoft MPEG-4 (MP4S)', + 'MP4T': 'MPEG-4 Transport Stream', + 'MP4V': 'Apple QuickTime MPEG-4 native', + 'MPEG': 'MPEG-1', + 'MPG1': 'FFmpeg-1', + 'MPG2': 'FFmpeg-1', + 'MPG3': 'Same as Low motion DivX MPEG-4', + 'MPG4': 'Microsoft MPEG-4 Video High Speed Compressor', + 'MPGI': 'Sigma Designs MPEG', + 'MPNG': 'Motion PNG codec', + 'MRCA': 'Martin Regen Codec', + 'MRLE': 'Run Length Encoding', + 'MSS1': 'Windows Screen Video', + 'MSS2': 'Windows Media 9', + 'MSUC': 'MSU LossLess', + 'MSVC': 'Microsoft Video 1', + 'MSZH': 'Lossless codec (ZIP compression)', + 'MTGA': 'Motion TGA images (24, 32 bpp)', + 'MTX1': 'Matrox MTX1', + 'MTX2': 'Matrox MTX2', + 'MTX3': 'Matrox MTX3', + 'MTX4': 'Matrox MTX4', + 'MTX5': 'Matrox MTX5', + 'MTX6': 'Matrox MTX6', + 'MTX7': 'Matrox MTX7', + 'MTX8': 'Matrox MTX8', + 'MTX9': 'Matrox MTX9', + 'MV12': 'MV12', + 'MVI1': 'Motion Pixels MVI', + 'MVI2': 'Motion Pixels MVI', + 'MWV1': 'Aware Motion Wavelets', + 'MYUV': 'Media-100 844/X Uncompressed', + 'NAVI': 'nAVI', + 'NDIG': 'Ahead Nero Digital MPEG-4 Codec', + 'NHVU': 'NVidia Texture Format (GEForce 3)', + 'NO16': 'Theory None16 64bit uncompressed RAW', + 'NT00': 'NewTek LigtWave HDTV YUV with Alpha-channel', + 'NTN1': 'Nogatech Video Compression 1', + 'NTN2': 'Nogatech Video Compression 2 (GrabBee hardware coder)', + 'NUV1': 'NuppelVideo', + 'NV12': '8-bit Y plane followed by an interleaved U/V plane with 2x2 subsampling', + 'NV21': 'As NV12 with U and V reversed in the interleaved plane', + 'NVDS': 'nVidia Texture Format', + 'NVHS': 'NVidia Texture Format (GEForce 3)', + 'NVS0': 'nVidia GeForce Texture', + 'NVS1': 'nVidia GeForce Texture', + 'NVS2': 'nVidia GeForce Texture', + 'NVS3': 'nVidia GeForce Texture', + 'NVS4': 'nVidia GeForce Texture', + 'NVS5': 'nVidia GeForce Texture', + 'NVT0': 'nVidia GeForce Texture', + 'NVT1': 'nVidia GeForce Texture', + 'NVT2': 'nVidia GeForce Texture', + 'NVT3': 'nVidia GeForce Texture', + 'NVT4': 'nVidia GeForce Texture', + 'NVT5': 'nVidia GeForce Texture', + 'PDVC': 'I-O Data Device Digital Video Capture DV codec', + 'PGVV': 'Radius Video Vision', + 'PHMO': 'IBM Photomotion', + 'PIM1': 'Pegasus Imaging', + 'PIM2': 'Pegasus Imaging', + 'PIMJ': 'Pegasus Imaging Lossless JPEG', + 'PIXL': 'MiroVideo XL (Motion JPEG)', + 'PNG ': 'Apple PNG', + 'PNG1': 'Corecodec.org CorePNG Codec', + 'PVEZ': 'Horizons Technology PowerEZ', + 'PVMM': 'PacketVideo Corporation MPEG-4', + 'PVW2': 'Pegasus Imaging Wavelet Compression', + 'PVWV': 'Pegasus Imaging Wavelet 2000', + 'PXLT': 'Apple Pixlet (Wavelet)', + 'Q1.0': 'Q-Team QPEG 1.0 (www.q-team.de)', + 'Q1.1': 'Q-Team QPEG 1.1 (www.q-team.de)', + 'QDGX': 'Apple QuickDraw GX', + 'QPEG': 'Q-Team QPEG 1.0', + 'QPEQ': 'Q-Team QPEG 1.1', + 'R210': 'BlackMagic YUV (Quick Time)', + 'R411': 'Radius DV NTSC YUV', + 'R420': 'Radius DV PAL YUV', + 'RAVI': 'GroupTRON ReferenceAVI codec (dummy for MPEG compressor)', + 'RAV_': 'GroupTRON ReferenceAVI codec (dummy for MPEG compressor)', + 'RAW ': 'Full Frames (Uncompressed)', + 'RGB ': 'Full Frames (Uncompressed)', + 'RGB(15)': 'Uncompressed RGB15 5:5:5', + 'RGB(16)': 'Uncompressed RGB16 5:6:5', + 'RGB(24)': 'Uncompressed RGB24 8:8:8', + 'RGB1': 'Uncompressed RGB332 3:3:2', + 'RGBA': 'Raw RGB with alpha', + 'RGBO': 'Uncompressed RGB555 5:5:5', + 'RGBP': 'Uncompressed RGB565 5:6:5', + 'RGBQ': 'Uncompressed RGB555X 5:5:5 BE', + 'RGBR': 'Uncompressed RGB565X 5:6:5 BE', + 'RGBT': 'Computer Concepts 32-bit support', + 'RL4 ': 'RLE 4bpp RGB', + 'RL8 ': 'RLE 8bpp RGB', + 'RLE ': 'Microsoft Run Length Encoder', + 'RLE4': 'Run Length Encoded 4', + 'RLE8': 'Run Length Encoded 8', + 'RMP4': 'REALmagic MPEG-4 Video Codec', + 'ROQV': 'Id RoQ File Video Decoder', + 'RPZA': 'Apple Video 16 bit "road pizza"', + 'RT21': 'Intel Real Time Video 2.1', + 'RTV0': 'NewTek VideoToaster', + 'RUD0': 'Rududu video codec', + 'RV10': 'RealVideo codec', + 'RV13': 'RealVideo codec', + 'RV20': 'RealVideo G2', + 'RV30': 'RealVideo 8', + 'RV40': 'RealVideo 9', + 'RVX ': 'Intel RDX (RVX )', + 'S263': 'Sorenson Vision H.263', + 'S422': 'Tekram VideoCap C210 YUV 4:2:2', + 'SAMR': 'Adaptive Multi-Rate (AMR) audio codec', + 'SAN3': 'MPEG-4 codec (direct copy of DivX 3.11a)', + 'SDCC': 'Sun Communication Digital Camera Codec', + 'SEDG': 'Samsung MPEG-4 codec', + 'SFMC': 'CrystalNet Surface Fitting Method', + 'SHR0': 'BitJazz SheerVideo', + 'SHR1': 'BitJazz SheerVideo', + 'SHR2': 'BitJazz SheerVideo', + 'SHR3': 'BitJazz SheerVideo', + 'SHR4': 'BitJazz SheerVideo', + 'SHR5': 'BitJazz SheerVideo', + 'SHR6': 'BitJazz SheerVideo', + 'SHR7': 'BitJazz SheerVideo', + 'SJPG': 'CUseeMe Networks Codec', + 'SL25': 'SoftLab-NSK DVCPRO', + 'SL50': 'SoftLab-NSK DVCPRO50', + 'SLDV': 'SoftLab-NSK Forward DV Draw codec', + 'SLIF': 'SoftLab-NSK MPEG2 I-frames', + 'SLMJ': 'SoftLab-NSK Forward MJPEG', + 'SMC ': 'Apple Graphics (SMC) codec (256 color)', + 'SMSC': 'Radius SMSC', + 'SMSD': 'Radius SMSD', + 'SMSV': 'WorldConnect Wavelet Video', + 'SNOW': 'SNOW codec', + 'SP40': 'SunPlus YUV', + 'SP44': 'SunPlus Aiptek MegaCam Codec', + 'SP53': 'SunPlus Aiptek MegaCam Codec', + 'SP54': 'SunPlus Aiptek MegaCam Codec', + 'SP55': 'SunPlus Aiptek MegaCam Codec', + 'SP56': 'SunPlus Aiptek MegaCam Codec', + 'SP57': 'SunPlus Aiptek MegaCam Codec', + 'SP58': 'SunPlus Aiptek MegaCam Codec', + 'SPIG': 'Radius Spigot', + 'SPLC': 'Splash Studios ACM Audio Codec', + 'SPRK': 'Sorenson Spark', + 'SQZ2': 'Microsoft VXTreme Video Codec V2', + 'STVA': 'ST CMOS Imager Data (Bayer)', + 'STVB': 'ST CMOS Imager Data (Nudged Bayer)', + 'STVC': 'ST CMOS Imager Data (Bunched)', + 'STVX': 'ST CMOS Imager Data (Extended CODEC Data Format)', + 'STVY': 'ST CMOS Imager Data (Extended CODEC Data Format with Correction Data)', + 'SV10': 'Sorenson Video R1', + 'SVQ1': 'Sorenson Video R3', + 'SVQ3': 'Sorenson Video 3 (Apple Quicktime 5)', + 'SWC1': 'MainConcept Motion JPEG Codec', + 'T420': 'Toshiba YUV 4:2:0', + 'TGA ': 'Apple TGA (with Alpha-channel)', + 'THEO': 'FFVFW Supported Codec', + 'TIFF': 'Apple TIFF (with Alpha-channel)', + 'TIM2': 'Pinnacle RAL DVI', + 'TLMS': 'TeraLogic Motion Intraframe Codec (TLMS)', + 'TLST': 'TeraLogic Motion Intraframe Codec (TLST)', + 'TM20': 'Duck TrueMotion 2.0', + 'TM2A': 'Duck TrueMotion Archiver 2.0', + 'TM2X': 'Duck TrueMotion 2X', + 'TMIC': 'TeraLogic Motion Intraframe Codec (TMIC)', + 'TMOT': 'Horizons Technology TrueMotion S', + 'TR20': 'Duck TrueMotion RealTime 2.0', + 'TRLE': 'Akula Alpha Pro Custom AVI (LossLess)', + 'TSCC': 'TechSmith Screen Capture Codec', + 'TV10': 'Tecomac Low-Bit Rate Codec', + 'TVJP': 'TrueVision Field Encoded Motion JPEG', + 'TVMJ': 'Truevision TARGA MJPEG Hardware Codec', + 'TY0N': 'Trident TY0N', + 'TY2C': 'Trident TY2C', + 'TY2N': 'Trident TY2N', + 'U263': 'UB Video StreamForce H.263', + 'U +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__title__ = 'enzyme' +__description__ = 'Video metadata parser' +__version__ = '0.1' +__author__ = 'Antoine Bertin' +__email__ = 'diaoulael@gmail.com' +__license__ = 'GPLv3' diff --git a/libs/enzyme/language.py b/libs/enzyme/language.py new file mode 100644 index 00000000..9a52692a --- /dev/null +++ b/libs/enzyme/language.py @@ -0,0 +1,538 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + + +import re + +__all__ = ['resolve'] + + +def resolve(code): + """ + Transform the given (2- or 3-letter) language code to a human readable + language name. The return value is a 2-tuple containing the given + language code and the language name. If the language code cannot be + resolved, name will be 'Unknown ()'. + """ + if not code: + return None, None + if not isinstance(code, basestring): + raise ValueError('Invalid language code specified by parser') + + # Take up to 3 letters from the code. + code = re.split(r'[^a-z]', code.lower())[0][:3] + + for spec in codes: + if code in spec[:-1]: + return code, spec[-1] + + return code, u'Unknown (%s)' % code + + +# Parsed from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt +codes = ( + ('aar', 'aa', u'Afar'), + ('abk', 'ab', u'Abkhazian'), + ('ace', u'Achinese'), + ('ach', u'Acoli'), + ('ada', u'Adangme'), + ('ady', u'Adyghe'), + ('afa', u'Afro-Asiatic '), + ('afh', u'Afrihili'), + ('afr', 'af', u'Afrikaans'), + ('ain', u'Ainu'), + ('aka', 'ak', u'Akan'), + ('akk', u'Akkadian'), + ('alb', 'sq', u'Albanian'), + ('ale', u'Aleut'), + ('alg', u'Algonquian languages'), + ('alt', u'Southern Altai'), + ('amh', 'am', u'Amharic'), + ('ang', u'English, Old '), + ('anp', u'Angika'), + ('apa', u'Apache languages'), + ('ara', 'ar', u'Arabic'), + ('arc', u'Official Aramaic '), + ('arg', 'an', u'Aragonese'), + ('arm', 'hy', u'Armenian'), + ('arn', u'Mapudungun'), + ('arp', u'Arapaho'), + ('art', u'Artificial '), + ('arw', u'Arawak'), + ('asm', 'as', u'Assamese'), + ('ast', u'Asturian'), + ('ath', u'Athapascan languages'), + ('aus', u'Australian languages'), + ('ava', 'av', u'Avaric'), + ('ave', 'ae', u'Avestan'), + ('awa', u'Awadhi'), + ('aym', 'ay', u'Aymara'), + ('aze', 'az', u'Azerbaijani'), + ('bad', u'Banda languages'), + ('bai', u'Bamileke languages'), + ('bak', 'ba', u'Bashkir'), + ('bal', u'Baluchi'), + ('bam', 'bm', u'Bambara'), + ('ban', u'Balinese'), + ('baq', 'eu', u'Basque'), + ('bas', u'Basa'), + ('bat', u'Baltic '), + ('bej', u'Beja'), + ('bel', 'be', u'Belarusian'), + ('bem', u'Bemba'), + ('ben', 'bn', u'Bengali'), + ('ber', u'Berber '), + ('bho', u'Bhojpuri'), + ('bih', 'bh', u'Bihari'), + ('bik', u'Bikol'), + ('bin', u'Bini'), + ('bis', 'bi', u'Bislama'), + ('bla', u'Siksika'), + ('bnt', u'Bantu '), + ('bos', 'bs', u'Bosnian'), + ('bra', u'Braj'), + ('bre', 'br', u'Breton'), + ('btk', u'Batak languages'), + ('bua', u'Buriat'), + ('bug', u'Buginese'), + ('bul', 'bg', u'Bulgarian'), + ('bur', 'my', u'Burmese'), + ('byn', u'Blin'), + ('cad', u'Caddo'), + ('cai', u'Central American Indian '), + ('car', u'Galibi Carib'), + ('cat', 'ca', u'Catalan'), + ('cau', u'Caucasian '), + ('ceb', u'Cebuano'), + ('cel', u'Celtic '), + ('cha', 'ch', u'Chamorro'), + ('chb', u'Chibcha'), + ('che', 'ce', u'Chechen'), + ('chg', u'Chagatai'), + ('chi', 'zh', u'Chinese'), + ('chk', u'Chuukese'), + ('chm', u'Mari'), + ('chn', u'Chinook jargon'), + ('cho', u'Choctaw'), + ('chp', u'Chipewyan'), + ('chr', u'Cherokee'), + ('chu', 'cu', u'Church Slavic'), + ('chv', 'cv', u'Chuvash'), + ('chy', u'Cheyenne'), + ('cmc', u'Chamic languages'), + ('cop', u'Coptic'), + ('cor', 'kw', u'Cornish'), + ('cos', 'co', u'Corsican'), + ('cpe', u'Creoles and pidgins, English based '), + ('cpf', u'Creoles and pidgins, French-based '), + ('cpp', u'Creoles and pidgins, Portuguese-based '), + ('cre', 'cr', u'Cree'), + ('crh', u'Crimean Tatar'), + ('crp', u'Creoles and pidgins '), + ('csb', u'Kashubian'), + ('cus', u'Cushitic '), + ('cze', 'cs', u'Czech'), + ('dak', u'Dakota'), + ('dan', 'da', u'Danish'), + ('dar', u'Dargwa'), + ('day', u'Land Dayak languages'), + ('del', u'Delaware'), + ('den', u'Slave '), + ('dgr', u'Dogrib'), + ('din', u'Dinka'), + ('div', 'dv', u'Divehi'), + ('doi', u'Dogri'), + ('dra', u'Dravidian '), + ('dsb', u'Lower Sorbian'), + ('dua', u'Duala'), + ('dum', u'Dutch, Middle '), + ('dut', 'nl', u'Dutch'), + ('dyu', u'Dyula'), + ('dzo', 'dz', u'Dzongkha'), + ('efi', u'Efik'), + ('egy', u'Egyptian '), + ('eka', u'Ekajuk'), + ('elx', u'Elamite'), + ('eng', 'en', u'English'), + ('enm', u'English, Middle '), + ('epo', 'eo', u'Esperanto'), + ('est', 'et', u'Estonian'), + ('ewe', 'ee', u'Ewe'), + ('ewo', u'Ewondo'), + ('fan', u'Fang'), + ('fao', 'fo', u'Faroese'), + ('fat', u'Fanti'), + ('fij', 'fj', u'Fijian'), + ('fil', u'Filipino'), + ('fin', 'fi', u'Finnish'), + ('fiu', u'Finno-Ugrian '), + ('fon', u'Fon'), + ('fre', 'fr', u'French'), + ('frm', u'French, Middle '), + ('fro', u'French, Old '), + ('frr', u'Northern Frisian'), + ('frs', u'Eastern Frisian'), + ('fry', 'fy', u'Western Frisian'), + ('ful', 'ff', u'Fulah'), + ('fur', u'Friulian'), + ('gaa', u'Ga'), + ('gay', u'Gayo'), + ('gba', u'Gbaya'), + ('gem', u'Germanic '), + ('geo', 'ka', u'Georgian'), + ('ger', 'de', u'German'), + ('gez', u'Geez'), + ('gil', u'Gilbertese'), + ('gla', 'gd', u'Gaelic'), + ('gle', 'ga', u'Irish'), + ('glg', 'gl', u'Galician'), + ('glv', 'gv', u'Manx'), + ('gmh', u'German, Middle High '), + ('goh', u'German, Old High '), + ('gon', u'Gondi'), + ('gor', u'Gorontalo'), + ('got', u'Gothic'), + ('grb', u'Grebo'), + ('grc', u'Greek, Ancient '), + ('gre', 'el', u'Greek, Modern '), + ('grn', 'gn', u'Guarani'), + ('gsw', u'Swiss German'), + ('guj', 'gu', u'Gujarati'), + ('gwi', u"Gwich'in"), + ('hai', u'Haida'), + ('hat', 'ht', u'Haitian'), + ('hau', 'ha', u'Hausa'), + ('haw', u'Hawaiian'), + ('heb', 'he', u'Hebrew'), + ('her', 'hz', u'Herero'), + ('hil', u'Hiligaynon'), + ('him', u'Himachali'), + ('hin', 'hi', u'Hindi'), + ('hit', u'Hittite'), + ('hmn', u'Hmong'), + ('hmo', 'ho', u'Hiri Motu'), + ('hsb', u'Upper Sorbian'), + ('hun', 'hu', u'Hungarian'), + ('hup', u'Hupa'), + ('iba', u'Iban'), + ('ibo', 'ig', u'Igbo'), + ('ice', 'is', u'Icelandic'), + ('ido', 'io', u'Ido'), + ('iii', 'ii', u'Sichuan Yi'), + ('ijo', u'Ijo languages'), + ('iku', 'iu', u'Inuktitut'), + ('ile', 'ie', u'Interlingue'), + ('ilo', u'Iloko'), + ('ina', 'ia', u'Interlingua '), + ('inc', u'Indic '), + ('ind', 'id', u'Indonesian'), + ('ine', u'Indo-European '), + ('inh', u'Ingush'), + ('ipk', 'ik', u'Inupiaq'), + ('ira', u'Iranian '), + ('iro', u'Iroquoian languages'), + ('ita', 'it', u'Italian'), + ('jav', 'jv', u'Javanese'), + ('jbo', u'Lojban'), + ('jpn', 'ja', u'Japanese'), + ('jpr', u'Judeo-Persian'), + ('jrb', u'Judeo-Arabic'), + ('kaa', u'Kara-Kalpak'), + ('kab', u'Kabyle'), + ('kac', u'Kachin'), + ('kal', 'kl', u'Kalaallisut'), + ('kam', u'Kamba'), + ('kan', 'kn', u'Kannada'), + ('kar', u'Karen languages'), + ('kas', 'ks', u'Kashmiri'), + ('kau', 'kr', u'Kanuri'), + ('kaw', u'Kawi'), + ('kaz', 'kk', u'Kazakh'), + ('kbd', u'Kabardian'), + ('kha', u'Khasi'), + ('khi', u'Khoisan '), + ('khm', 'km', u'Central Khmer'), + ('kho', u'Khotanese'), + ('kik', 'ki', u'Kikuyu'), + ('kin', 'rw', u'Kinyarwanda'), + ('kir', 'ky', u'Kirghiz'), + ('kmb', u'Kimbundu'), + ('kok', u'Konkani'), + ('kom', 'kv', u'Komi'), + ('kon', 'kg', u'Kongo'), + ('kor', 'ko', u'Korean'), + ('kos', u'Kosraean'), + ('kpe', u'Kpelle'), + ('krc', u'Karachay-Balkar'), + ('krl', u'Karelian'), + ('kro', u'Kru languages'), + ('kru', u'Kurukh'), + ('kua', 'kj', u'Kuanyama'), + ('kum', u'Kumyk'), + ('kur', 'ku', u'Kurdish'), + ('kut', u'Kutenai'), + ('lad', u'Ladino'), + ('lah', u'Lahnda'), + ('lam', u'Lamba'), + ('lao', 'lo', u'Lao'), + ('lat', 'la', u'Latin'), + ('lav', 'lv', u'Latvian'), + ('lez', u'Lezghian'), + ('lim', 'li', u'Limburgan'), + ('lin', 'ln', u'Lingala'), + ('lit', 'lt', u'Lithuanian'), + ('lol', u'Mongo'), + ('loz', u'Lozi'), + ('ltz', 'lb', u'Luxembourgish'), + ('lua', u'Luba-Lulua'), + ('lub', 'lu', u'Luba-Katanga'), + ('lug', 'lg', u'Ganda'), + ('lui', u'Luiseno'), + ('lun', u'Lunda'), + ('luo', u'Luo '), + ('lus', u'Lushai'), + ('mac', 'mk', u'Macedonian'), + ('mad', u'Madurese'), + ('mag', u'Magahi'), + ('mah', 'mh', u'Marshallese'), + ('mai', u'Maithili'), + ('mak', u'Makasar'), + ('mal', 'ml', u'Malayalam'), + ('man', u'Mandingo'), + ('mao', 'mi', u'Maori'), + ('map', u'Austronesian '), + ('mar', 'mr', u'Marathi'), + ('mas', u'Masai'), + ('may', 'ms', u'Malay'), + ('mdf', u'Moksha'), + ('mdr', u'Mandar'), + ('men', u'Mende'), + ('mga', u'Irish, Middle '), + ('mic', u"Mi'kmaq"), + ('min', u'Minangkabau'), + ('mis', u'Uncoded languages'), + ('mkh', u'Mon-Khmer '), + ('mlg', 'mg', u'Malagasy'), + ('mlt', 'mt', u'Maltese'), + ('mnc', u'Manchu'), + ('mni', u'Manipuri'), + ('mno', u'Manobo languages'), + ('moh', u'Mohawk'), + ('mol', 'mo', u'Moldavian'), + ('mon', 'mn', u'Mongolian'), + ('mos', u'Mossi'), + ('mul', u'Multiple languages'), + ('mun', u'Munda languages'), + ('mus', u'Creek'), + ('mwl', u'Mirandese'), + ('mwr', u'Marwari'), + ('myn', u'Mayan languages'), + ('myv', u'Erzya'), + ('nah', u'Nahuatl languages'), + ('nai', u'North American Indian'), + ('nap', u'Neapolitan'), + ('nau', 'na', u'Nauru'), + ('nav', 'nv', u'Navajo'), + ('nbl', 'nr', u'Ndebele, South'), + ('nde', 'nd', u'Ndebele, North'), + ('ndo', 'ng', u'Ndonga'), + ('nds', u'Low German'), + ('nep', 'ne', u'Nepali'), + ('new', u'Nepal Bhasa'), + ('nia', u'Nias'), + ('nic', u'Niger-Kordofanian '), + ('niu', u'Niuean'), + ('nno', 'nn', u'Norwegian Nynorsk'), + ('nob', 'nb', u'Bokm\xe5l, Norwegian'), + ('nog', u'Nogai'), + ('non', u'Norse, Old'), + ('nor', 'no', u'Norwegian'), + ('nqo', u"N'Ko"), + ('nso', u'Pedi'), + ('nub', u'Nubian languages'), + ('nwc', u'Classical Newari'), + ('nya', 'ny', u'Chichewa'), + ('nym', u'Nyamwezi'), + ('nyn', u'Nyankole'), + ('nyo', u'Nyoro'), + ('nzi', u'Nzima'), + ('oci', 'oc', u'Occitan '), + ('oji', 'oj', u'Ojibwa'), + ('ori', 'or', u'Oriya'), + ('orm', 'om', u'Oromo'), + ('osa', u'Osage'), + ('oss', 'os', u'Ossetian'), + ('ota', u'Turkish, Ottoman '), + ('oto', u'Otomian languages'), + ('paa', u'Papuan '), + ('pag', u'Pangasinan'), + ('pal', u'Pahlavi'), + ('pam', u'Pampanga'), + ('pan', 'pa', u'Panjabi'), + ('pap', u'Papiamento'), + ('pau', u'Palauan'), + ('peo', u'Persian, Old '), + ('per', 'fa', u'Persian'), + ('phi', u'Philippine '), + ('phn', u'Phoenician'), + ('pli', 'pi', u'Pali'), + ('pol', 'pl', u'Polish'), + ('pon', u'Pohnpeian'), + ('por', 'pt', u'Portuguese'), + ('pra', u'Prakrit languages'), + ('pro', u'Proven\xe7al, Old '), + ('pus', 'ps', u'Pushto'), + ('qaa-qtz', u'Reserved for local use'), + ('que', 'qu', u'Quechua'), + ('raj', u'Rajasthani'), + ('rap', u'Rapanui'), + ('rar', u'Rarotongan'), + ('roa', u'Romance '), + ('roh', 'rm', u'Romansh'), + ('rom', u'Romany'), + ('rum', 'ro', u'Romanian'), + ('run', 'rn', u'Rundi'), + ('rup', u'Aromanian'), + ('rus', 'ru', u'Russian'), + ('sad', u'Sandawe'), + ('sag', 'sg', u'Sango'), + ('sah', u'Yakut'), + ('sai', u'South American Indian '), + ('sal', u'Salishan languages'), + ('sam', u'Samaritan Aramaic'), + ('san', 'sa', u'Sanskrit'), + ('sas', u'Sasak'), + ('sat', u'Santali'), + ('scc', 'sr', u'Serbian'), + ('scn', u'Sicilian'), + ('sco', u'Scots'), + ('scr', 'hr', u'Croatian'), + ('sel', u'Selkup'), + ('sem', u'Semitic '), + ('sga', u'Irish, Old '), + ('sgn', u'Sign Languages'), + ('shn', u'Shan'), + ('sid', u'Sidamo'), + ('sin', 'si', u'Sinhala'), + ('sio', u'Siouan languages'), + ('sit', u'Sino-Tibetan '), + ('sla', u'Slavic '), + ('slo', 'sk', u'Slovak'), + ('slv', 'sl', u'Slovenian'), + ('sma', u'Southern Sami'), + ('sme', 'se', u'Northern Sami'), + ('smi', u'Sami languages '), + ('smj', u'Lule Sami'), + ('smn', u'Inari Sami'), + ('smo', 'sm', u'Samoan'), + ('sms', u'Skolt Sami'), + ('sna', 'sn', u'Shona'), + ('snd', 'sd', u'Sindhi'), + ('snk', u'Soninke'), + ('sog', u'Sogdian'), + ('som', 'so', u'Somali'), + ('son', u'Songhai languages'), + ('sot', 'st', u'Sotho, Southern'), + ('spa', 'es', u'Spanish'), + ('srd', 'sc', u'Sardinian'), + ('srn', u'Sranan Tongo'), + ('srr', u'Serer'), + ('ssa', u'Nilo-Saharan '), + ('ssw', 'ss', u'Swati'), + ('suk', u'Sukuma'), + ('sun', 'su', u'Sundanese'), + ('sus', u'Susu'), + ('sux', u'Sumerian'), + ('swa', 'sw', u'Swahili'), + ('swe', 'sv', u'Swedish'), + ('syc', u'Classical Syriac'), + ('syr', u'Syriac'), + ('tah', 'ty', u'Tahitian'), + ('tai', u'Tai '), + ('tam', 'ta', u'Tamil'), + ('tat', 'tt', u'Tatar'), + ('tel', 'te', u'Telugu'), + ('tem', u'Timne'), + ('ter', u'Tereno'), + ('tet', u'Tetum'), + ('tgk', 'tg', u'Tajik'), + ('tgl', 'tl', u'Tagalog'), + ('tha', 'th', u'Thai'), + ('tib', 'bo', u'Tibetan'), + ('tig', u'Tigre'), + ('tir', 'ti', u'Tigrinya'), + ('tiv', u'Tiv'), + ('tkl', u'Tokelau'), + ('tlh', u'Klingon'), + ('tli', u'Tlingit'), + ('tmh', u'Tamashek'), + ('tog', u'Tonga '), + ('ton', 'to', u'Tonga '), + ('tpi', u'Tok Pisin'), + ('tsi', u'Tsimshian'), + ('tsn', 'tn', u'Tswana'), + ('tso', 'ts', u'Tsonga'), + ('tuk', 'tk', u'Turkmen'), + ('tum', u'Tumbuka'), + ('tup', u'Tupi languages'), + ('tur', 'tr', u'Turkish'), + ('tut', u'Altaic '), + ('tvl', u'Tuvalu'), + ('twi', 'tw', u'Twi'), + ('tyv', u'Tuvinian'), + ('udm', u'Udmurt'), + ('uga', u'Ugaritic'), + ('uig', 'ug', u'Uighur'), + ('ukr', 'uk', u'Ukrainian'), + ('umb', u'Umbundu'), + ('und', u'Undetermined'), + ('urd', 'ur', u'Urdu'), + ('uzb', 'uz', u'Uzbek'), + ('vai', u'Vai'), + ('ven', 've', u'Venda'), + ('vie', 'vi', u'Vietnamese'), + ('vol', 'vo', u'Volap\xfck'), + ('vot', u'Votic'), + ('wak', u'Wakashan languages'), + ('wal', u'Walamo'), + ('war', u'Waray'), + ('was', u'Washo'), + ('wel', 'cy', u'Welsh'), + ('wen', u'Sorbian languages'), + ('wln', 'wa', u'Walloon'), + ('wol', 'wo', u'Wolof'), + ('xal', u'Kalmyk'), + ('xho', 'xh', u'Xhosa'), + ('yao', u'Yao'), + ('yap', u'Yapese'), + ('yid', 'yi', u'Yiddish'), + ('yor', 'yo', u'Yoruba'), + ('ypk', u'Yupik languages'), + ('zap', u'Zapotec'), + ('zbl', u'Blissymbols'), + ('zen', u'Zenaga'), + ('zha', 'za', u'Zhuang'), + ('znd', u'Zande languages'), + ('zul', 'zu', u'Zulu'), + ('zun', u'Zuni'), + ('zxx', u'No linguistic content'), + ('zza', u'Zaza'), +) diff --git a/libs/enzyme/mkv.py b/libs/enzyme/mkv.py new file mode 100644 index 00000000..eac0a380 --- /dev/null +++ b/libs/enzyme/mkv.py @@ -0,0 +1,841 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# Copyright (C) 2003-2006 Jason Tackaberry +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +from datetime import datetime +from exceptions import * +from struct import unpack +import core +import logging +import re + +# get logging object +log = logging.getLogger(__name__) + +# Main IDs for the Matroska streams +MATROSKA_VIDEO_TRACK = 0x01 +MATROSKA_AUDIO_TRACK = 0x02 +MATROSKA_SUBTITLES_TRACK = 0x11 + +MATROSKA_HEADER_ID = 0x1A45DFA3 +MATROSKA_TRACKS_ID = 0x1654AE6B +MATROSKA_CUES_ID = 0x1C53BB6B +MATROSKA_SEGMENT_ID = 0x18538067 +MATROSKA_SEGMENT_INFO_ID = 0x1549A966 +MATROSKA_CLUSTER_ID = 0x1F43B675 +MATROSKA_VOID_ID = 0xEC +MATROSKA_CRC_ID = 0xBF +MATROSKA_TIMECODESCALE_ID = 0x2AD7B1 +MATROSKA_DURATION_ID = 0x4489 +MATROSKA_CRC32_ID = 0xBF +MATROSKA_TIMECODESCALE_ID = 0x2AD7B1 +MATROSKA_MUXING_APP_ID = 0x4D80 +MATROSKA_WRITING_APP_ID = 0x5741 +MATROSKA_CODEC_ID = 0x86 +MATROSKA_CODEC_PRIVATE_ID = 0x63A2 +MATROSKA_FRAME_DURATION_ID = 0x23E383 +MATROSKA_VIDEO_SETTINGS_ID = 0xE0 +MATROSKA_VIDEO_WIDTH_ID = 0xB0 +MATROSKA_VIDEO_HEIGHT_ID = 0xBA +MATROSKA_VIDEO_INTERLACED_ID = 0x9A +MATROSKA_VIDEO_DISPLAY_WIDTH_ID = 0x54B0 +MATROSKA_VIDEO_DISPLAY_HEIGHT_ID = 0x54BA +MATROSKA_AUDIO_SETTINGS_ID = 0xE1 +MATROSKA_AUDIO_SAMPLERATE_ID = 0xB5 +MATROSKA_AUDIO_CHANNELS_ID = 0x9F +MATROSKA_TRACK_UID_ID = 0x73C5 +MATROSKA_TRACK_NUMBER_ID = 0xD7 +MATROSKA_TRACK_TYPE_ID = 0x83 +MATROSKA_TRACK_LANGUAGE_ID = 0x22B59C +MATROSKA_TRACK_OFFSET = 0x537F +MATROSKA_TRACK_FLAG_DEFAULT_ID = 0x88 +MATROSKA_TRACK_FLAG_ENABLED_ID = 0xB9 +MATROSKA_TITLE_ID = 0x7BA9 +MATROSKA_DATE_UTC_ID = 0x4461 +MATROSKA_NAME_ID = 0x536E + +MATROSKA_CHAPTERS_ID = 0x1043A770 +MATROSKA_CHAPTER_UID_ID = 0x73C4 +MATROSKA_EDITION_ENTRY_ID = 0x45B9 +MATROSKA_CHAPTER_ATOM_ID = 0xB6 +MATROSKA_CHAPTER_TIME_START_ID = 0x91 +MATROSKA_CHAPTER_TIME_END_ID = 0x92 +MATROSKA_CHAPTER_FLAG_ENABLED_ID = 0x4598 +MATROSKA_CHAPTER_DISPLAY_ID = 0x80 +MATROSKA_CHAPTER_LANGUAGE_ID = 0x437C +MATROSKA_CHAPTER_STRING_ID = 0x85 + +MATROSKA_ATTACHMENTS_ID = 0x1941A469 +MATROSKA_ATTACHED_FILE_ID = 0x61A7 +MATROSKA_FILE_DESC_ID = 0x467E +MATROSKA_FILE_NAME_ID = 0x466E +MATROSKA_FILE_MIME_TYPE_ID = 0x4660 +MATROSKA_FILE_DATA_ID = 0x465C + +MATROSKA_SEEKHEAD_ID = 0x114D9B74 +MATROSKA_SEEK_ID = 0x4DBB +MATROSKA_SEEKID_ID = 0x53AB +MATROSKA_SEEK_POSITION_ID = 0x53AC + +MATROSKA_TAGS_ID = 0x1254C367 +MATROSKA_TAG_ID = 0x7373 +MATROSKA_TARGETS_ID = 0x63C0 +MATROSKA_TARGET_TYPE_VALUE_ID = 0x68CA +MATROSKA_TARGET_TYPE_ID = 0x63CA +MATRSOKA_TAGS_TRACK_UID_ID = 0x63C5 +MATRSOKA_TAGS_EDITION_UID_ID = 0x63C9 +MATRSOKA_TAGS_CHAPTER_UID_ID = 0x63C4 +MATRSOKA_TAGS_ATTACHMENT_UID_ID = 0x63C6 +MATROSKA_SIMPLE_TAG_ID = 0x67C8 +MATROSKA_TAG_NAME_ID = 0x45A3 +MATROSKA_TAG_LANGUAGE_ID = 0x447A +MATROSKA_TAG_STRING_ID = 0x4487 +MATROSKA_TAG_BINARY_ID = 0x4485 + + +# See mkv spec for details: +# http://www.matroska.org/technical/specs/index.html + +# Map to convert to well known codes +# http://haali.cs.msu.ru/mkv/codecs.pdf +FOURCCMap = { + 'V_THEORA': 'THEO', + 'V_SNOW': 'SNOW', + 'V_MPEG4/ISO/ASP': 'MP4V', + 'V_MPEG4/ISO/AVC': 'AVC1', + 'A_AC3': 0x2000, + 'A_MPEG/L3': 0x0055, + 'A_MPEG/L2': 0x0050, + 'A_MPEG/L1': 0x0050, + 'A_DTS': 0x2001, + 'A_PCM/INT/LIT': 0x0001, + 'A_PCM/FLOAT/IEEE': 0x003, + 'A_TTA1': 0x77a1, + 'A_WAVPACK4': 0x5756, + 'A_VORBIS': 0x6750, + 'A_FLAC': 0xF1AC, + 'A_AAC': 0x00ff, + 'A_AAC/': 0x00ff +} + + +def matroska_date_to_datetime(date): + """ + Converts a date in Matroska's date format to a python datetime object. + Returns the given date string if it could not be converted. + """ + # From the specs: + # The fields with dates should have the following format: YYYY-MM-DD + # HH:MM:SS.MSS [...] To store less accuracy, you remove items starting + # from the right. To store only the year, you would use, "2004". To store + # a specific day such as May 1st, 2003, you would use "2003-05-01". + format = re.split(r'([-:. ])', '%Y-%m-%d %H:%M:%S.%f') + while format: + try: + return datetime.strptime(date, ''.join(format)) + except ValueError: + format = format[:-2] + return date + + +def matroska_bps_to_bitrate(bps): + """ + Tries to convert a free-form bps string into a bitrate (bits per second). + """ + m = re.search('([\d.]+)\s*(\D.*)', bps) + if m: + bps, suffix = m.groups() + if 'kbit' in suffix: + return float(bps) * 1024 + elif 'kbyte' in suffix: + return float(bps) * 1024 * 8 + elif 'byte' in suffix: + return float(bps) * 8 + elif 'bps' in suffix or 'bit' in suffix: + return float(bps) + if bps.replace('.', '').isdigit(): + if float(bps) < 30000: + # Assume kilobits and convert to bps + return float(bps) * 1024 + return float(bps) + + +# Used to convert the official matroska tag names (only lower-cased) to core +# attributes. tag name -> attr, filter +TAGS_MAP = { + # From Media core + u'title': ('title', None), + u'subtitle': ('caption', None), + u'comment': ('comment', None), + u'url': ('url', None), + u'artist': ('artist', None), + u'keywords': ('keywords', lambda s: [word.strip() for word in s.split(',')]), + u'composer_nationality': ('country', None), + u'date_released': ('datetime', None), + u'date_recorded': ('datetime', None), + u'date_written': ('datetime', None), + + # From Video core + u'encoder': ('encoder', None), + u'bps': ('bitrate', matroska_bps_to_bitrate), + u'part_number': ('trackno', int), + u'total_parts': ('trackof', int), + u'copyright': ('copyright', None), + u'genre': ('genre', None), + u'actor': ('actors', None), + u'written_by': ('writer', None), + u'producer': ('producer', None), + u'production_studio': ('studio', None), + u'law_rating': ('rating', None), + u'summary': ('summary', None), + u'synopsis': ('synopsis', None), +} + + +class EbmlEntity: + """ + This is class that is responsible to handle one Ebml entity as described in + the Matroska/Ebml spec + """ + def __init__(self, inbuf): + # Compute the EBML id + # Set the CRC len to zero + self.crc_len = 0 + # Now loop until we find an entity without CRC + try: + self.build_entity(inbuf) + except IndexError: + raise ParseError() + while self.get_id() == MATROSKA_CRC32_ID: + self.crc_len += self.get_total_len() + inbuf = inbuf[self.get_total_len():] + self.build_entity(inbuf) + + def build_entity(self, inbuf): + self.compute_id(inbuf) + + if self.id_len == 0: + log.error("EBML entity not found, bad file format") + raise ParseError() + + self.entity_len, self.len_size = self.compute_len(inbuf[self.id_len:]) + self.entity_data = inbuf[self.get_header_len() : self.get_total_len()] + self.ebml_length = self.entity_len + self.entity_len = min(len(self.entity_data), self.entity_len) + + # if the data size is 8 or less, it could be a numeric value + self.value = 0 + if self.entity_len <= 8: + for pos, shift in zip(range(self.entity_len), range((self.entity_len-1)*8, -1, -8)): + self.value |= ord(self.entity_data[pos]) << shift + + + def add_data(self, data): + maxlen = self.ebml_length - len(self.entity_data) + if maxlen <= 0: + return + self.entity_data += data[:maxlen] + self.entity_len = len(self.entity_data) + + + def compute_id(self, inbuf): + self.id_len = 0 + if len(inbuf) < 1: + return 0 + first = ord(inbuf[0]) + if first & 0x80: + self.id_len = 1 + self.entity_id = first + elif first & 0x40: + if len(inbuf) < 2: + return 0 + self.id_len = 2 + self.entity_id = ord(inbuf[0])<<8 | ord(inbuf[1]) + elif first & 0x20: + if len(inbuf) < 3: + return 0 + self.id_len = 3 + self.entity_id = (ord(inbuf[0])<<16) | (ord(inbuf[1])<<8) | \ + (ord(inbuf[2])) + elif first & 0x10: + if len(inbuf) < 4: + return 0 + self.id_len = 4 + self.entity_id = (ord(inbuf[0])<<24) | (ord(inbuf[1])<<16) | \ + (ord(inbuf[2])<<8) | (ord(inbuf[3])) + self.entity_str = inbuf[0:self.id_len] + + + def compute_len(self, inbuf): + if not inbuf: + return 0, 0 + i = num_ffs = 0 + len_mask = 0x80 + len = ord(inbuf[0]) + while not len & len_mask: + i += 1 + len_mask >>= 1 + if i >= 8: + return 0, 0 + + len &= len_mask - 1 + if len == len_mask - 1: + num_ffs += 1 + for p in range(i): + len = (len << 8) | ord(inbuf[p + 1]) + if len & 0xff == 0xff: + num_ffs += 1 + if num_ffs == i + 1: + len = 0 + return len, i + 1 + + + def get_crc_len(self): + return self.crc_len + + + def get_value(self): + return self.value + + + def get_float_value(self): + if len(self.entity_data) == 4: + return unpack('!f', self.entity_data)[0] + elif len(self.entity_data) == 8: + return unpack('!d', self.entity_data)[0] + return 0.0 + + + def get_data(self): + return self.entity_data + + + def get_utf8(self): + return unicode(self.entity_data, 'utf-8', 'replace') + + + def get_str(self): + return unicode(self.entity_data, 'ascii', 'replace') + + + def get_id(self): + return self.entity_id + + + def get_str_id(self): + return self.entity_str + + + def get_len(self): + return self.entity_len + + + def get_total_len(self): + return self.entity_len + self.id_len + self.len_size + + + def get_header_len(self): + return self.id_len + self.len_size + + + +class Matroska(core.AVContainer): + """ + Matroska video and audio parser. If at least one video stream is + detected it will set the type to MEDIA_AV. + """ + def __init__(self, file): + core.AVContainer.__init__(self) + self.samplerate = 1 + + self.file = file + # Read enough that we're likely to get the full seekhead (FIXME: kludge) + buffer = file.read(2000) + if len(buffer) == 0: + # Regular File end + raise ParseError() + + # Check the Matroska header + header = EbmlEntity(buffer) + if header.get_id() != MATROSKA_HEADER_ID: + raise ParseError() + + log.debug("HEADER ID found %08X" % header.get_id() ) + self.mime = 'video/x-matroska' + self.type = 'Matroska' + self.has_idx = False + self.objects_by_uid = {} + + # Now get the segment + self.segment = segment = EbmlEntity(buffer[header.get_total_len():]) + # Record file offset of segment data for seekheads + self.segment.offset = header.get_total_len() + segment.get_header_len() + if segment.get_id() != MATROSKA_SEGMENT_ID: + log.debug("SEGMENT ID not found %08X" % segment.get_id()) + return + + log.debug("SEGMENT ID found %08X" % segment.get_id()) + try: + for elem in self.process_one_level(segment): + if elem.get_id() == MATROSKA_SEEKHEAD_ID: + self.process_elem(elem) + except ParseError: + pass + + if not self.has_idx: + log.warning('File has no index') + self._set('corrupt', True) + + def process_elem(self, elem): + elem_id = elem.get_id() + log.debug('BEGIN: process element %s' % hex(elem_id)) + if elem_id == MATROSKA_SEGMENT_INFO_ID: + duration = 0 + scalecode = 1000000.0 + + for ielem in self.process_one_level(elem): + ielem_id = ielem.get_id() + if ielem_id == MATROSKA_TIMECODESCALE_ID: + scalecode = ielem.get_value() + elif ielem_id == MATROSKA_DURATION_ID: + duration = ielem.get_float_value() + elif ielem_id == MATROSKA_TITLE_ID: + self.title = ielem.get_utf8() + elif ielem_id == MATROSKA_DATE_UTC_ID: + timestamp = unpack('!q', ielem.get_data())[0] / 10.0**9 + # Date is offset 2001-01-01 00:00:00 (timestamp 978307200.0) + self.timestamp = int(timestamp + 978307200) + + self.length = duration * scalecode / 1000000000.0 + + elif elem_id == MATROSKA_TRACKS_ID: + self.process_tracks(elem) + + elif elem_id == MATROSKA_CHAPTERS_ID: + self.process_chapters(elem) + + elif elem_id == MATROSKA_ATTACHMENTS_ID: + self.process_attachments(elem) + + elif elem_id == MATROSKA_SEEKHEAD_ID: + self.process_seekhead(elem) + + elif elem_id == MATROSKA_TAGS_ID: + self.process_tags(elem) + + elif elem_id == MATROSKA_CUES_ID: + self.has_idx = True + + log.debug('END: process element %s' % hex(elem_id)) + return True + + + def process_seekhead(self, elem): + for seek_elem in self.process_one_level(elem): + if seek_elem.get_id() != MATROSKA_SEEK_ID: + continue + for sub_elem in self.process_one_level(seek_elem): + if sub_elem.get_id() == MATROSKA_SEEKID_ID: + if sub_elem.get_value() == MATROSKA_CLUSTER_ID: + # Not interested in these. + return + + elif sub_elem.get_id() == MATROSKA_SEEK_POSITION_ID: + self.file.seek(self.segment.offset + sub_elem.get_value()) + buffer = self.file.read(100) + try: + elem = EbmlEntity(buffer) + except ParseError: + continue + + # Fetch all data necessary for this element. + elem.add_data(self.file.read(elem.ebml_length)) + self.process_elem(elem) + + + def process_tracks(self, tracks): + tracksbuf = tracks.get_data() + index = 0 + while index < tracks.get_len(): + trackelem = EbmlEntity(tracksbuf[index:]) + log.debug ("ELEMENT %X found" % trackelem.get_id()) + self.process_track(trackelem) + index += trackelem.get_total_len() + trackelem.get_crc_len() + + + def process_one_level(self, item): + buf = item.get_data() + index = 0 + while index < item.get_len(): + if len(buf[index:]) == 0: + break + elem = EbmlEntity(buf[index:]) + yield elem + index += elem.get_total_len() + elem.get_crc_len() + + def set_track_defaults(self, track): + track.language = 'eng' + + def process_track(self, track): + # Collapse generator into a list since we need to iterate over it + # twice. + elements = [x for x in self.process_one_level(track)] + track_type = [x.get_value() for x in elements if x.get_id() == MATROSKA_TRACK_TYPE_ID] + if not track_type: + log.debug('Bad track: no type id found') + return + + track_type = track_type[0] + track = None + + if track_type == MATROSKA_VIDEO_TRACK: + log.debug("Video track found") + track = self.process_video_track(elements) + elif track_type == MATROSKA_AUDIO_TRACK: + log.debug("Audio track found") + track = self.process_audio_track(elements) + elif track_type == MATROSKA_SUBTITLES_TRACK: + log.debug("Subtitle track found") + track = core.Subtitle() + self.set_track_defaults(track) + track.id = len(self.subtitles) + self.subtitles.append(track) + for elem in elements: + self.process_track_common(elem, track) + + + def process_track_common(self, elem, track): + elem_id = elem.get_id() + if elem_id == MATROSKA_TRACK_LANGUAGE_ID: + track.language = elem.get_str() + log.debug("Track language found: %s" % track.language) + elif elem_id == MATROSKA_NAME_ID: + track.title = elem.get_utf8() + elif elem_id == MATROSKA_TRACK_NUMBER_ID: + track.trackno = elem.get_value() + elif elem_id == MATROSKA_TRACK_FLAG_ENABLED_ID: + track.enabled = bool(elem.get_value()) + elif elem_id == MATROSKA_TRACK_FLAG_DEFAULT_ID: + track.default = bool(elem.get_value()) + elif elem_id == MATROSKA_CODEC_ID: + track.codec = elem.get_str() + elif elem_id == MATROSKA_CODEC_PRIVATE_ID: + track.codec_private = elem.get_data() + elif elem_id == MATROSKA_TRACK_UID_ID: + self.objects_by_uid[elem.get_value()] = track + + + def process_video_track(self, elements): + track = core.VideoStream() + # Defaults + track.codec = u'Unknown' + track.fps = 0 + self.set_track_defaults(track) + + for elem in elements: + elem_id = elem.get_id() + if elem_id == MATROSKA_CODEC_ID: + track.codec = elem.get_str() + + elif elem_id == MATROSKA_FRAME_DURATION_ID: + try: + track.fps = 1 / (pow(10, -9) * (elem.get_value())) + except ZeroDivisionError: + pass + + elif elem_id == MATROSKA_VIDEO_SETTINGS_ID: + d_width = d_height = None + for settings_elem in self.process_one_level(elem): + settings_elem_id = settings_elem.get_id() + if settings_elem_id == MATROSKA_VIDEO_WIDTH_ID: + track.width = settings_elem.get_value() + elif settings_elem_id == MATROSKA_VIDEO_HEIGHT_ID: + track.height = settings_elem.get_value() + elif settings_elem_id == MATROSKA_VIDEO_DISPLAY_WIDTH_ID: + d_width = settings_elem.get_value() + elif settings_elem_id == MATROSKA_VIDEO_DISPLAY_HEIGHT_ID: + d_height = settings_elem.get_value() + elif settings_elem_id == MATROSKA_VIDEO_INTERLACED_ID: + value = int(settings_elem.get_value()) + self._set('interlaced', value) + + if None not in [d_width, d_height]: + track.aspect = float(d_width) / d_height + + else: + self.process_track_common(elem, track) + + # convert codec information + # http://haali.cs.msu.ru/mkv/codecs.pdf + if track.codec in FOURCCMap: + track.codec = FOURCCMap[track.codec] + elif '/' in track.codec and track.codec.split('/')[0] + '/' in FOURCCMap: + track.codec = FOURCCMap[track.codec.split('/')[0] + '/'] + elif track.codec.endswith('FOURCC') and len(track.codec_private or '') == 40: + track.codec = track.codec_private[16:20] + elif track.codec.startswith('V_REAL/'): + track.codec = track.codec[7:] + elif track.codec.startswith('V_'): + # FIXME: add more video codecs here + track.codec = track.codec[2:] + + track.id = len(self.video) + self.video.append(track) + return track + + + def process_audio_track(self, elements): + track = core.AudioStream() + track.codec = u'Unknown' + self.set_track_defaults(track) + + for elem in elements: + elem_id = elem.get_id() + if elem_id == MATROSKA_CODEC_ID: + track.codec = elem.get_str() + elif elem_id == MATROSKA_AUDIO_SETTINGS_ID: + for settings_elem in self.process_one_level(elem): + settings_elem_id = settings_elem.get_id() + if settings_elem_id == MATROSKA_AUDIO_SAMPLERATE_ID: + track.samplerate = settings_elem.get_float_value() + elif settings_elem_id == MATROSKA_AUDIO_CHANNELS_ID: + track.channels = settings_elem.get_value() + else: + self.process_track_common(elem, track) + + + if track.codec in FOURCCMap: + track.codec = FOURCCMap[track.codec] + elif '/' in track.codec and track.codec.split('/')[0] + '/' in FOURCCMap: + track.codec = FOURCCMap[track.codec.split('/')[0] + '/'] + elif track.codec.startswith('A_'): + track.codec = track.codec[2:] + + track.id = len(self.audio) + self.audio.append(track) + return track + + + def process_chapters(self, chapters): + elements = self.process_one_level(chapters) + for elem in elements: + if elem.get_id() == MATROSKA_EDITION_ENTRY_ID: + buf = elem.get_data() + index = 0 + while index < elem.get_len(): + sub_elem = EbmlEntity(buf[index:]) + if sub_elem.get_id() == MATROSKA_CHAPTER_ATOM_ID: + self.process_chapter_atom(sub_elem) + index += sub_elem.get_total_len() + sub_elem.get_crc_len() + + + def process_chapter_atom(self, atom): + elements = self.process_one_level(atom) + chap = core.Chapter() + + for elem in elements: + elem_id = elem.get_id() + if elem_id == MATROSKA_CHAPTER_TIME_START_ID: + # Scale timecode to seconds (float) + chap.pos = elem.get_value() / 1000000 / 1000.0 + elif elem_id == MATROSKA_CHAPTER_FLAG_ENABLED_ID: + chap.enabled = elem.get_value() + elif elem_id == MATROSKA_CHAPTER_DISPLAY_ID: + # Matroska supports multiple (chapter name, language) pairs for + # each chapter, so chapter names can be internationalized. This + # logic will only take the last one in the list. + for display_elem in self.process_one_level(elem): + if display_elem.get_id() == MATROSKA_CHAPTER_STRING_ID: + chap.name = display_elem.get_utf8() + elif elem_id == MATROSKA_CHAPTER_UID_ID: + self.objects_by_uid[elem.get_value()] = chap + + log.debug('Chapter "%s" found', chap.name) + chap.id = len(self.chapters) + self.chapters.append(chap) + + + def process_attachments(self, attachments): + buf = attachments.get_data() + index = 0 + while index < attachments.get_len(): + elem = EbmlEntity(buf[index:]) + if elem.get_id() == MATROSKA_ATTACHED_FILE_ID: + self.process_attachment(elem) + index += elem.get_total_len() + elem.get_crc_len() + + + def process_attachment(self, attachment): + elements = self.process_one_level(attachment) + name = desc = mimetype = "" + data = None + + for elem in elements: + elem_id = elem.get_id() + if elem_id == MATROSKA_FILE_NAME_ID: + name = elem.get_utf8() + elif elem_id == MATROSKA_FILE_DESC_ID: + desc = elem.get_utf8() + elif elem_id == MATROSKA_FILE_MIME_TYPE_ID: + mimetype = elem.get_data() + elif elem_id == MATROSKA_FILE_DATA_ID: + data = elem.get_data() + + # Right now we only support attachments that could be cover images. + # Make a guess to see if this attachment is a cover image. + if mimetype.startswith("image/") and u"cover" in (name+desc).lower() and data: + self.thumbnail = data + + log.debug('Attachment "%s" found' % name) + + + def process_tags(self, tags): + # Tags spec: http://www.matroska.org/technical/specs/tagging/index.html + # Iterate over Tags children. Tags element children is a + # Tag element (whose children are SimpleTags) and a Targets element + # whose children specific what objects the tags apply to. + for tag_elem in self.process_one_level(tags): + # Start a new dict to hold all SimpleTag elements. + tags_dict = core.Tags() + # A list of target uids this tags dict applies too. If empty, + # tags are global. + targets = [] + for sub_elem in self.process_one_level(tag_elem): + if sub_elem.get_id() == MATROSKA_SIMPLE_TAG_ID: + self.process_simple_tag(sub_elem, tags_dict) + elif sub_elem.get_id() == MATROSKA_TARGETS_ID: + # Targets element: if there is no uid child (track uid, + # chapter uid, etc.) then the tags dict applies to the + # whole file (top-level Media object). + for target_elem in self.process_one_level(sub_elem): + target_elem_id = target_elem.get_id() + if target_elem_id in (MATRSOKA_TAGS_TRACK_UID_ID, MATRSOKA_TAGS_EDITION_UID_ID, + MATRSOKA_TAGS_CHAPTER_UID_ID, MATRSOKA_TAGS_ATTACHMENT_UID_ID): + targets.append(target_elem.get_value()) + elif target_elem_id == MATROSKA_TARGET_TYPE_VALUE_ID: + # Target types not supported for now. (Unclear how this + # would fit with kaa.metadata.) + pass + if targets: + # Assign tags to all listed uids + for target in targets: + try: + self.objects_by_uid[target].tags.update(tags_dict) + self.tags_to_attributes(self.objects_by_uid[target], tags_dict) + except KeyError: + log.warning('Tags assigned to unknown/unsupported target uid %d', target) + else: + self.tags.update(tags_dict) + self.tags_to_attributes(self, tags_dict) + + + def process_simple_tag(self, simple_tag_elem, tags_dict): + """ + Returns a dict representing the Tag element. + """ + name = lang = value = children = None + binary = False + for elem in self.process_one_level(simple_tag_elem): + elem_id = elem.get_id() + if elem_id == MATROSKA_TAG_NAME_ID: + name = elem.get_utf8().lower() + elif elem_id == MATROSKA_TAG_STRING_ID: + value = elem.get_utf8() + elif elem_id == MATROSKA_TAG_BINARY_ID: + value = elem.get_data() + binary = True + elif elem_id == MATROSKA_TAG_LANGUAGE_ID: + lang = elem.get_utf8() + elif elem_id == MATROSKA_SIMPLE_TAG_ID: + if children is None: + children = core.Tags() + self.process_simple_tag(elem, children) + + if children: + # Convert ourselves to a Tags object. + children.value = value + children.langcode = lang + value = children + else: + if name.startswith('date_'): + # Try to convert date to a datetime object. + value = matroska_date_to_datetime(value) + value = core.Tag(value, lang, binary) + + if name in tags_dict: + # Multiple items of this tag name. + if not isinstance(tags_dict[name], list): + # Convert to a list + tags_dict[name] = [tags_dict[name]] + # Append to list + tags_dict[name].append(value) + else: + tags_dict[name] = value + + + def tags_to_attributes(self, obj, tags): + # Convert tags to core attributes. + for name, tag in tags.items(): + if isinstance(tag, dict): + # Nested tags dict, recurse. + self.tags_to_attributes(obj, tag) + continue + elif name not in TAGS_MAP: + continue + + attr, filter = TAGS_MAP[name] + if attr not in obj._keys and attr not in self._keys: + # Tag is not in any core attribute for this object or global, + # so skip. + continue + + # Pull value out of Tag object or list of Tag objects. + value = [item.value for item in tag] if isinstance(tag, list) else tag.value + if filter: + try: + value = [filter(item) for item in value] if isinstance(value, list) else filter(value) + except Exception, e: + log.warning('Failed to convert tag to core attribute: %s', e) + # Special handling for tv series recordings. The 'title' tag + # can be used for both the series and the episode name. The + # same is true for trackno which may refer to the season + # and the episode number. Therefore, if we find these + # attributes already set we try some guessing. + if attr == 'trackno' and getattr(self, attr) is not None: + # delete trackno and save season and episode + self.season = self.trackno + self.episode = value + self.trackno = None + continue + if attr == 'title' and getattr(self, attr) is not None: + # store current value of title as series and use current + # value of title as title + self.series = self.title + if attr in obj._keys: + setattr(obj, attr, value) + else: + setattr(self, attr, value) + + +Parser = Matroska diff --git a/libs/enzyme/mp4.py b/libs/enzyme/mp4.py new file mode 100644 index 00000000..41bf43da --- /dev/null +++ b/libs/enzyme/mp4.py @@ -0,0 +1,476 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2007 Thomas Schueppel +# Copyright (C) 2003-2007 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +import zlib +import logging +import StringIO +import struct +from exceptions import * +import core + +# get logging object +log = logging.getLogger(__name__) + + +# http://developer.apple.com/documentation/QuickTime/QTFF/index.html +# http://developer.apple.com/documentation/QuickTime/QTFF/QTFFChap4/\ +# chapter_5_section_2.html#//apple_ref/doc/uid/TP40000939-CH206-BBCBIICE +# Note: May need to define custom log level to work like ATOM_DEBUG did here + +QTUDTA = { + 'nam': 'title', + 'aut': 'artist', + 'cpy': 'copyright' +} + +QTLANGUAGES = { + 0: "en", + 1: "fr", + 2: "de", + 3: "it", + 4: "nl", + 5: "sv", + 6: "es", + 7: "da", + 8: "pt", + 9: "no", + 10: "he", + 11: "ja", + 12: "ar", + 13: "fi", + 14: "el", + 15: "is", + 16: "mt", + 17: "tr", + 18: "hr", + 19: "Traditional Chinese", + 20: "ur", + 21: "hi", + 22: "th", + 23: "ko", + 24: "lt", + 25: "pl", + 26: "hu", + 27: "et", + 28: "lv", + 29: "Lappish", + 30: "fo", + 31: "Farsi", + 32: "ru", + 33: "Simplified Chinese", + 34: "Flemish", + 35: "ga", + 36: "sq", + 37: "ro", + 38: "cs", + 39: "sk", + 40: "sl", + 41: "yi", + 42: "sr", + 43: "mk", + 44: "bg", + 45: "uk", + 46: "be", + 47: "uz", + 48: "kk", + 49: "az", + 50: "AzerbaijanAr", + 51: "hy", + 52: "ka", + 53: "mo", + 54: "ky", + 55: "tg", + 56: "tk", + 57: "mn", + 58: "MongolianCyr", + 59: "ps", + 60: "ku", + 61: "ks", + 62: "sd", + 63: "bo", + 64: "ne", + 65: "sa", + 66: "mr", + 67: "bn", + 68: "as", + 69: "gu", + 70: "pa", + 71: "or", + 72: "ml", + 73: "kn", + 74: "ta", + 75: "te", + 76: "si", + 77: "my", + 78: "Khmer", + 79: "lo", + 80: "vi", + 81: "id", + 82: "tl", + 83: "MalayRoman", + 84: "MalayArabic", + 85: "am", + 86: "ti", + 87: "om", + 88: "so", + 89: "sw", + 90: "Ruanda", + 91: "Rundi", + 92: "Chewa", + 93: "mg", + 94: "eo", + 128: "cy", + 129: "eu", + 130: "ca", + 131: "la", + 132: "qu", + 133: "gn", + 134: "ay", + 135: "tt", + 136: "ug", + 137: "Dzongkha", + 138: "JavaneseRom", +} + +class MPEG4(core.AVContainer): + """ + Parser for the MP4 container format. This format is mostly + identical to Apple Quicktime and 3GP files. It maps to mp4, mov, + qt and some other extensions. + """ + table_mapping = {'QTUDTA': QTUDTA} + + def __init__(self, file): + core.AVContainer.__init__(self) + self._references = [] + + self.mime = 'video/quicktime' + self.type = 'Quicktime Video' + h = file.read(8) + try: + (size, type) = struct.unpack('>I4s',h) + except struct.error: + # EOF. + raise ParseError() + + if type == 'ftyp': + # file type information + if size >= 12: + # this should always happen + if file.read(4) != 'qt ': + # not a quicktime movie, it is a mpeg4 container + self.mime = 'video/mp4' + self.type = 'MPEG-4 Video' + size -= 4 + file.seek(size-8, 1) + h = file.read(8) + (size, type) = struct.unpack('>I4s',h) + + while type in ['mdat', 'skip']: + # movie data at the beginning, skip + file.seek(size-8, 1) + h = file.read(8) + (size, type) = struct.unpack('>I4s',h) + + if not type in ['moov', 'wide', 'free']: + log.debug('invalid header: %r' % type) + raise ParseError() + + # Extended size + if size == 1: + size = struct.unpack('>Q', file.read(8)) + + # Back over the atom header we just read, since _readatom expects the + # file position to be at the start of an atom. + file.seek(-8, 1) + while self._readatom(file): + pass + + if self._references: + self._set('references', self._references) + + + def _readatom(self, file): + s = file.read(8) + if len(s) < 8: + return 0 + + atomsize,atomtype = struct.unpack('>I4s', s) + if not str(atomtype).decode('latin1').isalnum(): + # stop at nonsense data + return 0 + + log.debug('%s [%X]' % (atomtype,atomsize)) + + if atomtype == 'udta': + # Userdata (Metadata) + pos = 0 + tabl = {} + i18ntabl = {} + atomdata = file.read(atomsize-8) + while pos < atomsize-12: + (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8]) + if ord(datatype[0]) == 169: + # i18n Metadata... + mypos = 8+pos + while mypos + 4 < datasize+pos: + # first 4 Bytes are i18n header + (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos+4]) + i18ntabl[lang] = i18ntabl.get(lang, {}) + l = atomdata[mypos+4:mypos+tlen+4] + i18ntabl[lang][datatype[1:]] = l + mypos += tlen+4 + elif datatype == 'WLOC': + # Drop Window Location + pass + else: + if ord(atomdata[pos+8:pos+datasize][0]) > 1: + tabl[datatype] = atomdata[pos+8:pos+datasize] + pos += datasize + if len(i18ntabl.keys()) > 0: + for k in i18ntabl.keys(): + if QTLANGUAGES.has_key(k) and QTLANGUAGES[k] == 'en': + self._appendtable('QTUDTA', i18ntabl[k]) + self._appendtable('QTUDTA', tabl) + else: + log.debug('NO i18') + self._appendtable('QTUDTA', tabl) + + elif atomtype == 'trak': + atomdata = file.read(atomsize-8) + pos = 0 + trackinfo = {} + tracktype = None + while pos < atomsize-8: + (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8]) + + if datatype == 'tkhd': + tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos+8:pos+datasize]) + trackinfo['width'] = tkhd[10] >> 16 + trackinfo['height'] = tkhd[11] >> 16 + trackinfo['id'] = tkhd[3] + + try: + # XXX Timestamp of Seconds is since January 1st 1904! + # XXX 2082844800 is the difference between Unix and + # XXX Apple time. FIXME to work on Apple, too + self.timestamp = int(tkhd[1]) - 2082844800 + except Exception, e: + log.exception('There was trouble extracting timestamp') + + elif datatype == 'mdia': + pos += 8 + datasize -= 8 + log.debug('--> mdia information') + + while datasize: + mdia = struct.unpack('>I4s', atomdata[pos:pos+8]) + if mdia[1] == 'mdhd': + # Parse based on version of mdhd header. See + # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd + ver = ord(atomdata[pos + 8]) + if ver == 0: + mdhd = struct.unpack('>IIIIIhh', atomdata[pos+8:pos+8+24]) + elif ver == 1: + mdhd = struct.unpack('>IQQIQhh', atomdata[pos+8:pos+8+36]) + else: + mdhd = None + + if mdhd: + # duration / time scale + trackinfo['length'] = mdhd[4] / mdhd[3] + if mdhd[5] in QTLANGUAGES: + trackinfo['language'] = QTLANGUAGES[mdhd[5]] + # mdhd[6] == quality + self.length = max(self.length, mdhd[4] / mdhd[3]) + elif mdia[1] == 'minf': + # minf has only atoms inside + pos -= (mdia[0] - 8) + datasize += (mdia[0] - 8) + elif mdia[1] == 'stbl': + # stbl has only atoms inside + pos -= (mdia[0] - 8) + datasize += (mdia[0] - 8) + elif mdia[1] == 'hdlr': + hdlr = struct.unpack('>I4s4s', atomdata[pos+8:pos+8+12]) + if hdlr[1] == 'mhlr': + if hdlr[2] == 'vide': + tracktype = 'video' + if hdlr[2] == 'soun': + tracktype = 'audio' + elif mdia[1] == 'stsd': + stsd = struct.unpack('>2I', atomdata[pos+8:pos+8+8]) + if stsd[1] > 0: + codec = atomdata[pos+16:pos+16+8] + codec = struct.unpack('>I4s', codec) + trackinfo['codec'] = codec[1] + if codec[1] == 'jpeg': + tracktype = 'image' + elif mdia[1] == 'dinf': + dref = struct.unpack('>I4s', atomdata[pos+8:pos+8+8]) + log.debug(' --> %s, %s (useless)' % mdia) + if dref[1] == 'dref': + num = struct.unpack('>I', atomdata[pos+20:pos+20+4])[0] + rpos = pos+20+4 + for ref in range(num): + # FIXME: do somthing if this references + ref = struct.unpack('>I3s', atomdata[rpos:rpos+7]) + data = atomdata[rpos+7:rpos+ref[0]] + rpos += ref[0] + else: + if mdia[1].startswith('st'): + log.debug(' --> %s, %s (sample)' % mdia) + elif mdia[1] == 'vmhd' and not tracktype: + # indicates that this track is video + tracktype = 'video' + elif mdia[1] in ['vmhd', 'smhd'] and not tracktype: + # indicates that this track is audio + tracktype = 'audio' + else: + log.debug(' --> %s, %s (unknown)' % mdia) + + pos += mdia[0] + datasize -= mdia[0] + + elif datatype == 'udta': + log.debug(struct.unpack('>I4s', atomdata[:8])) + else: + if datatype == 'edts': + log.debug('--> %s [%d] (edit list)' % \ + (datatype, datasize)) + else: + log.debug('--> %s [%d] (unknown)' % \ + (datatype, datasize)) + pos += datasize + + info = None + if tracktype == 'video': + info = core.VideoStream() + self.video.append(info) + if tracktype == 'audio': + info = core.AudioStream() + self.audio.append(info) + if info: + for key, value in trackinfo.items(): + setattr(info, key, value) + + elif atomtype == 'mvhd': + # movie header + mvhd = struct.unpack('>6I2h', file.read(28)) + self.length = max(self.length, mvhd[4] / mvhd[3]) + self.volume = mvhd[6] + file.seek(atomsize-8-28,1) + + + elif atomtype == 'cmov': + # compressed movie + datasize, atomtype = struct.unpack('>I4s', file.read(8)) + if not atomtype == 'dcom': + return atomsize + + method = struct.unpack('>4s', file.read(datasize-8))[0] + + datasize, atomtype = struct.unpack('>I4s', file.read(8)) + if not atomtype == 'cmvd': + return atomsize + + if method == 'zlib': + data = file.read(datasize-8) + try: + decompressed = zlib.decompress(data) + except Exception, e: + try: + decompressed = zlib.decompress(data[4:]) + except Exception, e: + log.exception('There was a proble decompressiong atom') + return atomsize + + decompressedIO = StringIO.StringIO(decompressed) + while self._readatom(decompressedIO): + pass + + else: + log.info('unknown compression %s' % method) + # unknown compression method + file.seek(datasize-8,1) + + elif atomtype == 'moov': + # decompressed movie info + while self._readatom(file): + pass + + elif atomtype == 'mdat': + pos = file.tell() + atomsize - 8 + # maybe there is data inside the mdat + log.info('parsing mdat') + while self._readatom(file): + pass + log.info('end of mdat') + file.seek(pos, 0) + + + elif atomtype == 'rmra': + # reference list + while self._readatom(file): + pass + + elif atomtype == 'rmda': + # reference + atomdata = file.read(atomsize-8) + pos = 0 + url = '' + quality = 0 + datarate = 0 + while pos < atomsize-8: + (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8]) + if datatype == 'rdrf': + rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos+8:pos+20]) + if rtype == 'url ': + url = atomdata[pos+20:pos+20+rlen] + if url.find('\0') > 0: + url = url[:url.find('\0')] + elif datatype == 'rmqu': + quality = struct.unpack('>I', atomdata[pos+8:pos+12])[0] + + elif datatype == 'rmdr': + datarate = struct.unpack('>I', atomdata[pos+12:pos+16])[0] + + pos += datasize + if url: + self._references.append((url, quality, datarate)) + + else: + if not atomtype in ['wide', 'free']: + log.info('unhandled base atom %s' % atomtype) + + # Skip unknown atoms + try: + file.seek(atomsize-8,1) + except IOError: + return 0 + + return atomsize + + +Parser = MPEG4 diff --git a/libs/enzyme/mpeg.py b/libs/enzyme/mpeg.py new file mode 100644 index 00000000..d7c44db2 --- /dev/null +++ b/libs/enzyme/mpeg.py @@ -0,0 +1,915 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +import os +import struct +import logging +import stat +from exceptions import * +import core + +# get logging object +log = logging.getLogger(__name__) + +##------------------------------------------------------------------------ +## START_CODE +## +## Start Codes, with 'slice' occupying 0x01..0xAF +##------------------------------------------------------------------------ +START_CODE = { + 0x00 : 'picture_start_code', + 0xB0 : 'reserved', + 0xB1 : 'reserved', + 0xB2 : 'user_data_start_code', + 0xB3 : 'sequence_header_code', + 0xB4 : 'sequence_error_code', + 0xB5 : 'extension_start_code', + 0xB6 : 'reserved', + 0xB7 : 'sequence end', + 0xB8 : 'group of pictures', +} +for i in range(0x01,0xAF): + START_CODE[i] = 'slice_start_code' + +##------------------------------------------------------------------------ +## START CODES +##------------------------------------------------------------------------ +PICTURE = 0x00 +USERDATA = 0xB2 +SEQ_HEAD = 0xB3 +SEQ_ERR = 0xB4 +EXT_START = 0xB5 +SEQ_END = 0xB7 +GOP = 0xB8 + +SEQ_START_CODE = 0xB3 +PACK_PKT = 0xBA +SYS_PKT = 0xBB +PADDING_PKT = 0xBE +AUDIO_PKT = 0xC0 +VIDEO_PKT = 0xE0 +PRIVATE_STREAM1 = 0xBD +PRIVATE_STREAM2 = 0xBf + +TS_PACKET_LENGTH = 188 +TS_SYNC = 0x47 + +##------------------------------------------------------------------------ +## FRAME_RATE +## +## A lookup table of all the standard frame rates. Some rates adhere to +## a particular profile that ensures compatibility with VLSI capabilities +## of the early to mid 1990s. +## +## CPB +## Constrained Parameters Bitstreams, an MPEG-1 set of sampling and +## bitstream parameters designed to normalize decoder computational +## complexity, buffer size, and memory bandwidth while still addressing +## the widest possible range of applications. +## +## Main Level +## MPEG-2 Video Main Profile and Main Level is analogous to MPEG-1's +## CPB, with sampling limits at CCIR 601 parameters (720x480x30 Hz or +## 720x576x24 Hz). +## +##------------------------------------------------------------------------ +FRAME_RATE = [ + 0, + 24000.0/1001, ## 3-2 pulldown NTSC (CPB/Main Level) + 24, ## Film (CPB/Main Level) + 25, ## PAL/SECAM or 625/60 video + 30000.0/1001, ## NTSC (CPB/Main Level) + 30, ## drop-frame NTSC or component 525/60 (CPB/Main Level) + 50, ## double-rate PAL + 60000.0/1001, ## double-rate NTSC + 60, ## double-rate, drop-frame NTSC/component 525/60 video + ] + +##------------------------------------------------------------------------ +## ASPECT_RATIO -- INCOMPLETE? +## +## This lookup table maps the header aspect ratio index to a float value. +## These are just the defined ratios for CPB I believe. As I understand +## it, a stream that doesn't adhere to one of these aspect ratios is +## technically considered non-compliant. +##------------------------------------------------------------------------ +ASPECT_RATIO = ( None, # Forbidden + 1.0, # 1/1 (VGA) + 4.0 / 3, # 4/3 (TV) + 16.0 / 9, # 16/9 (Widescreen) + 2.21 # (Cinema) + ) + + +class MPEG(core.AVContainer): + """ + Parser for various MPEG files. This includes MPEG-1 and MPEG-2 + program streams, elementary streams and transport streams. The + reported length differs from the length reported by most video + players but the provides length here is correct. An MPEG file has + no additional metadata like title, etc; only codecs, length and + resolution is reported back. + """ + def __init__(self,file): + core.AVContainer.__init__(self) + self.sequence_header_offset = 0 + self.mpeg_version = 2 + + # detect TS (fast scan) + if not self.isTS(file): + # detect system mpeg (many infos) + if not self.isMPEG(file): + # detect PES + if not self.isPES(file): + # Maybe it's MPEG-ES + if self.isES(file): + # If isES() succeeds, we needn't do anything further. + return + if file.name.lower().endswith('mpeg') or \ + file.name.lower().endswith('mpg'): + # This has to be an mpeg file. It could be a bad + # recording from an ivtv based hardware encoder with + # same bytes missing at the beginning. + # Do some more digging... + if not self.isMPEG(file, force=True) or \ + not self.video or not self.audio: + # does not look like an mpeg at all + raise ParseError() + else: + # no mpeg at all + raise ParseError() + + self.mime = 'video/mpeg' + if not self.video: + self.video.append(core.VideoStream()) + + if self.sequence_header_offset <= 0: + return + + self.progressive(file) + + for vi in self.video: + vi.width, vi.height = self.dxy(file) + vi.fps, vi.aspect = self.framerate_aspect(file) + vi.bitrate = self.bitrate(file) + if self.length: + vi.length = self.length + + if not self.type: + self.type = 'MPEG Video' + + # set fourcc codec for video and audio + vc, ac = 'MP2V', 'MP2A' + if self.mpeg_version == 1: + vc, ac = 'MPEG', 0x0050 + for v in self.video: + v.codec = vc + for a in self.audio: + if not a.codec: + a.codec = ac + + + def dxy(self,file): + """ + get width and height of the video + """ + file.seek(self.sequence_header_offset+4,0) + v = file.read(4) + x = struct.unpack('>H',v[:2])[0] >> 4 + y = struct.unpack('>H',v[1:3])[0] & 0x0FFF + return (x,y) + + + def framerate_aspect(self,file): + """ + read framerate and aspect ratio + """ + file.seek(self.sequence_header_offset+7,0) + v = struct.unpack( '>B', file.read(1) )[0] + try: + fps = FRAME_RATE[v&0xf] + except IndexError: + fps = None + if v>>4 < len(ASPECT_RATIO): + aspect = ASPECT_RATIO[v>>4] + else: + aspect = None + return (fps, aspect) + + + def progressive(self, file): + """ + Try to find out with brute force if the mpeg is interlaced or not. + Search for the Sequence_Extension in the extension header (01B5) + """ + file.seek(0) + buffer = '' + count = 0 + while 1: + if len(buffer) < 1000: + count += 1 + if count > 1000: + break + buffer += file.read(1024) + if len(buffer) < 1000: + break + pos = buffer.find('\x00\x00\x01\xb5') + if pos == -1 or len(buffer) - pos < 5: + buffer = buffer[-10:] + continue + ext = (ord(buffer[pos+4]) >> 4) + if ext == 8: + pass + elif ext == 1: + if (ord(buffer[pos+5]) >> 3) & 1: + self._set('progressive', True) + else: + self._set('interlaced', True) + return True + else: + log.debug('ext', ext) + buffer = buffer[pos+4:] + return False + + + ##------------------------------------------------------------------------ + ## bitrate() + ## + ## From the MPEG-2.2 spec: + ## + ## bit_rate -- This is a 30-bit integer. The lower 18 bits of the + ## integer are in bit_rate_value and the upper 12 bits are in + ## bit_rate_extension. The 30-bit integer specifies the bitrate of the + ## bitstream measured in units of 400 bits/second, rounded upwards. + ## The value zero is forbidden. + ## + ## So ignoring all the variable bitrate stuff for now, this 30 bit integer + ## multiplied times 400 bits/sec should give the rate in bits/sec. + ## + ## TODO: Variable bitrates? I need one that implements this. + ## + ## Continued from the MPEG-2.2 spec: + ## + ## If the bitstream is a constant bitrate stream, the bitrate specified + ## is the actual rate of operation of the VBV specified in annex C. If + ## the bitstream is a variable bitrate stream, the STD specifications in + ## ISO/IEC 13818-1 supersede the VBV, and the bitrate specified here is + ## used to dimension the transport stream STD (2.4.2 in ITU-T Rec. xxx | + ## ISO/IEC 13818-1), or the program stream STD (2.4.5 in ITU-T Rec. xxx | + ## ISO/IEC 13818-1). + ## + ## If the bitstream is not a constant rate bitstream the vbv_delay + ## field shall have the value FFFF in hexadecimal. + ## + ## Given the value encoded in the bitrate field, the bitstream shall be + ## generated so that the video encoding and the worst case multiplex + ## jitter do not cause STD buffer overflow or underflow. + ## + ## + ##------------------------------------------------------------------------ + + + ## Some parts in the code are based on mpgtx (mpgtx.sf.net) + + def bitrate(self,file): + """ + read the bitrate (most of the time broken) + """ + file.seek(self.sequence_header_offset+8,0) + t,b = struct.unpack( '>HB', file.read(3) ) + vrate = t << 2 | b >> 6 + return vrate * 400 + + + def ReadSCRMpeg2(self, buffer): + """ + read SCR (timestamp) for MPEG2 at the buffer beginning (6 Bytes) + """ + if len(buffer) < 6: + return None + + highbit = (ord(buffer[0])&0x20)>>5 + + low4Bytes= ((long(ord(buffer[0])) & 0x18) >> 3) << 30 + low4Bytes |= (ord(buffer[0]) & 0x03) << 28 + low4Bytes |= ord(buffer[1]) << 20 + low4Bytes |= (ord(buffer[2]) & 0xF8) << 12 + low4Bytes |= (ord(buffer[2]) & 0x03) << 13 + low4Bytes |= ord(buffer[3]) << 5 + low4Bytes |= (ord(buffer[4])) >> 3 + + sys_clock_ref=(ord(buffer[4]) & 0x3) << 7 + sys_clock_ref|=(ord(buffer[5]) >> 1) + + return (long(highbit * (1<<16) * (1<<16)) + low4Bytes) / 90000 + + + def ReadSCRMpeg1(self, buffer): + """ + read SCR (timestamp) for MPEG1 at the buffer beginning (5 Bytes) + """ + if len(buffer) < 5: + return None + + highbit = (ord(buffer[0]) >> 3) & 0x01 + + low4Bytes = ((long(ord(buffer[0])) >> 1) & 0x03) << 30 + low4Bytes |= ord(buffer[1]) << 22; + low4Bytes |= (ord(buffer[2]) >> 1) << 15; + low4Bytes |= ord(buffer[3]) << 7; + low4Bytes |= ord(buffer[4]) >> 1; + + return (long(highbit) * (1<<16) * (1<<16) + low4Bytes) / 90000; + + + def ReadPTS(self, buffer): + """ + read PTS (PES timestamp) at the buffer beginning (5 Bytes) + """ + high = ((ord(buffer[0]) & 0xF) >> 1) + med = (ord(buffer[1]) << 7) + (ord(buffer[2]) >> 1) + low = (ord(buffer[3]) << 7) + (ord(buffer[4]) >> 1) + return ((long(high) << 30 ) + (med << 15) + low) / 90000 + + + def ReadHeader(self, buffer, offset): + """ + Handle MPEG header in buffer on position offset + Return None on error, new offset or 0 if the new offset can't be scanned + """ + if buffer[offset:offset+3] != '\x00\x00\x01': + return None + + id = ord(buffer[offset+3]) + + if id == PADDING_PKT: + return offset + (ord(buffer[offset+4]) << 8) + \ + ord(buffer[offset+5]) + 6 + + if id == PACK_PKT: + if ord(buffer[offset+4]) & 0xF0 == 0x20: + self.type = 'MPEG-1 Video' + self.get_time = self.ReadSCRMpeg1 + self.mpeg_version = 1 + return offset + 12 + elif (ord(buffer[offset+4]) & 0xC0) == 0x40: + self.type = 'MPEG-2 Video' + self.get_time = self.ReadSCRMpeg2 + return offset + (ord(buffer[offset+13]) & 0x07) + 14 + else: + # I have no idea what just happened, but for some DVB + # recordings done with mencoder this points to a + # PACK_PKT describing something odd. Returning 0 here + # (let's hope there are no extensions in the header) + # fixes it. + return 0 + + if 0xC0 <= id <= 0xDF: + # code for audio stream + for a in self.audio: + if a.id == id: + break + else: + self.audio.append(core.AudioStream()) + self.audio[-1]._set('id', id) + return 0 + + if 0xE0 <= id <= 0xEF: + # code for video stream + for v in self.video: + if v.id == id: + break + else: + self.video.append(core.VideoStream()) + self.video[-1]._set('id', id) + return 0 + + if id == SEQ_HEAD: + # sequence header, remember that position for later use + self.sequence_header_offset = offset + return 0 + + if id in [PRIVATE_STREAM1, PRIVATE_STREAM2]: + # private stream. we don't know, but maybe we can guess later + add = ord(buffer[offset+8]) + # if (ord(buffer[offset+6]) & 4) or 1: + # id = ord(buffer[offset+10+add]) + if buffer[offset+11+add:offset+15+add].find('\x0b\x77') != -1: + # AC3 stream + for a in self.audio: + if a.id == id: + break + else: + self.audio.append(core.AudioStream()) + self.audio[-1]._set('id', id) + self.audio[-1].codec = 0x2000 # AC3 + return 0 + + if id == SYS_PKT: + return 0 + + if id == EXT_START: + return 0 + + return 0 + + + # Normal MPEG (VCD, SVCD) ======================================== + + def isMPEG(self, file, force=False): + """ + This MPEG starts with a sequence of 0x00 followed by a PACK Header + http://dvd.sourceforge.net/dvdinfo/packhdr.html + """ + file.seek(0,0) + buffer = file.read(10000) + offset = 0 + + # seek until the 0 byte stop + while offset < len(buffer)-100 and buffer[offset] == '\0': + offset += 1 + offset -= 2 + + # test for mpeg header 0x00 0x00 0x01 + header = '\x00\x00\x01%s' % chr(PACK_PKT) + if offset < 0 or not buffer[offset:offset+4] == header: + if not force: + return 0 + # brute force and try to find the pack header in the first + # 10000 bytes somehow + offset = buffer.find(header) + if offset < 0: + return 0 + + # scan the 100000 bytes of data + buffer += file.read(100000) + + # scan first header, to get basic info about + # how to read a timestamp + self.ReadHeader(buffer, offset) + + # store first timestamp + self.start = self.get_time(buffer[offset+4:]) + while len(buffer) > offset + 1000 and \ + buffer[offset:offset+3] == '\x00\x00\x01': + # read the mpeg header + new_offset = self.ReadHeader(buffer, offset) + + # header scanning detected error, this is no mpeg + if new_offset == None: + return 0 + + if new_offset: + # we have a new offset + offset = new_offset + + # skip padding 0 before a new header + while len(buffer) > offset + 10 and \ + not ord(buffer[offset+2]): + offset += 1 + + else: + # seek to new header by brute force + offset += buffer[offset+4:].find('\x00\x00\x01') + 4 + + # fill in values for support functions: + self.__seek_size__ = 1000000 + self.__sample_size__ = 10000 + self.__search__ = self._find_timer_ + self.filename = file.name + + # get length of the file + self.length = self.get_length() + return 1 + + + def _find_timer_(self, buffer): + """ + Return position of timer in buffer or None if not found. + This function is valid for 'normal' mpeg files + """ + pos = buffer.find('\x00\x00\x01%s' % chr(PACK_PKT)) + if pos == -1: + return None + return pos + 4 + + + + # PES ============================================================ + + + def ReadPESHeader(self, offset, buffer, id=0): + """ + Parse a PES header. + Since it starts with 0x00 0x00 0x01 like 'normal' mpegs, this + function will return (0, None) when it is no PES header or + (packet length, timestamp position (maybe None)) + + http://dvd.sourceforge.net/dvdinfo/pes-hdr.html + """ + if not buffer[0:3] == '\x00\x00\x01': + return 0, None + + packet_length = (ord(buffer[4]) << 8) + ord(buffer[5]) + 6 + align = ord(buffer[6]) & 4 + header_length = ord(buffer[8]) + + # PES ID (starting with 001) + if ord(buffer[3]) & 0xE0 == 0xC0: + id = id or ord(buffer[3]) & 0x1F + for a in self.audio: + if a.id == id: + break + else: + self.audio.append(core.AudioStream()) + self.audio[-1]._set('id', id) + + elif ord(buffer[3]) & 0xF0 == 0xE0: + id = id or ord(buffer[3]) & 0xF + for v in self.video: + if v.id == id: + break + else: + self.video.append(core.VideoStream()) + self.video[-1]._set('id', id) + + # new mpeg starting + if buffer[header_length+9:header_length+13] == \ + '\x00\x00\x01\xB3' and not self.sequence_header_offset: + # yes, remember offset for later use + self.sequence_header_offset = offset + header_length+9 + elif ord(buffer[3]) == 189 or ord(buffer[3]) == 191: + # private stream. we don't know, but maybe we can guess later + id = id or ord(buffer[3]) & 0xF + if align and \ + buffer[header_length+9:header_length+11] == '\x0b\x77': + # AC3 stream + for a in self.audio: + if a.id == id: + break + else: + self.audio.append(core.AudioStream()) + self.audio[-1]._set('id', id) + self.audio[-1].codec = 0x2000 # AC3 + + else: + # unknown content + pass + + ptsdts = ord(buffer[7]) >> 6 + + if ptsdts and ptsdts == ord(buffer[9]) >> 4: + if ord(buffer[9]) >> 4 != ptsdts: + log.warning('WARNING: bad PTS/DTS, please contact us') + return packet_length, None + + # timestamp = self.ReadPTS(buffer[9:14]) + high = ((ord(buffer[9]) & 0xF) >> 1) + med = (ord(buffer[10]) << 7) + (ord(buffer[11]) >> 1) + low = (ord(buffer[12]) << 7) + (ord(buffer[13]) >> 1) + return packet_length, 9 + + return packet_length, None + + + + def isPES(self, file): + log.info('trying mpeg-pes scan') + file.seek(0,0) + buffer = file.read(3) + + # header (also valid for all mpegs) + if not buffer == '\x00\x00\x01': + return 0 + + self.sequence_header_offset = 0 + buffer += file.read(10000) + + offset = 0 + while offset + 1000 < len(buffer): + pos, timestamp = self.ReadPESHeader(offset, buffer[offset:]) + if not pos: + return 0 + if timestamp != None and not hasattr(self, 'start'): + self.get_time = self.ReadPTS + bpos = buffer[offset+timestamp:offset+timestamp+5] + self.start = self.get_time(bpos) + if self.sequence_header_offset and hasattr(self, 'start'): + # we have all informations we need + break + + offset += pos + if offset + 1000 < len(buffer) and len(buffer) < 1000000 or 1: + # looks like a pes, read more + buffer += file.read(10000) + + if not self.video and not self.audio: + # no video and no audio? + return 0 + + self.type = 'MPEG-PES' + + # fill in values for support functions: + self.__seek_size__ = 10000000 # 10 MB + self.__sample_size__ = 500000 # 500 k scanning + self.__search__ = self._find_timer_PES_ + self.filename = file.name + + # get length of the file + self.length = self.get_length() + return 1 + + + def _find_timer_PES_(self, buffer): + """ + Return position of timer in buffer or -1 if not found. + This function is valid for PES files + """ + pos = buffer.find('\x00\x00\x01') + offset = 0 + if pos == -1 or offset + 1000 >= len(buffer): + return None + + retpos = -1 + ackcount = 0 + while offset + 1000 < len(buffer): + pos, timestamp = self.ReadPESHeader(offset, buffer[offset:]) + if timestamp != None and retpos == -1: + retpos = offset + timestamp + if pos == 0: + # Oops, that was a mpeg header, no PES header + offset += buffer[offset:].find('\x00\x00\x01') + retpos = -1 + ackcount = 0 + else: + offset += pos + if retpos != -1: + ackcount += 1 + if ackcount > 10: + # looks ok to me + return retpos + return None + + + # Elementary Stream =============================================== + + def isES(self, file): + file.seek(0, 0) + try: + header = struct.unpack('>LL', file.read(8)) + except (struct.error, IOError): + return False + + if header[0] != 0x1B3: + return False + + # Is an mpeg video elementary stream + + self.mime = 'video/mpeg' + video = core.VideoStream() + video.width = header[1] >> 20 + video.height = (header[1] >> 8) & 0xfff + if header[1] & 0xf < len(FRAME_RATE): + video.fps = FRAME_RATE[header[1] & 0xf] + if (header[1] >> 4) & 0xf < len(ASPECT_RATIO): + # FIXME: Empirically the aspect looks like PAR rather than DAR + video.aspect = ASPECT_RATIO[(header[1] >> 4) & 0xf] + self.video.append(video) + return True + + + # Transport Stream =============================================== + + def isTS(self, file): + file.seek(0,0) + + buffer = file.read(TS_PACKET_LENGTH * 2) + c = 0 + + while c + TS_PACKET_LENGTH < len(buffer): + if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC: + break + c += 1 + else: + return 0 + + buffer += file.read(10000) + self.type = 'MPEG-TS' + + while c + TS_PACKET_LENGTH < len(buffer): + start = ord(buffer[c+1]) & 0x40 + # maybe load more into the buffer + if c + 2 * TS_PACKET_LENGTH > len(buffer) and c < 500000: + buffer += file.read(10000) + + # wait until the ts payload contains a payload header + if not start: + c += TS_PACKET_LENGTH + continue + + tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2]) + adapt = (ord(buffer[c+3]) & 0x30) >> 4 + + offset = 4 + if adapt & 0x02: + # meta info present, skip it for now + adapt_len = ord(buffer[c+offset]) + offset += adapt_len + 1 + + if not ord(buffer[c+1]) & 0x40: + # no new pes or psi in stream payload starting + pass + elif adapt & 0x01: + # PES + timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], + tsid)[1] + if timestamp != None: + if not hasattr(self, 'start'): + self.get_time = self.ReadPTS + timestamp = c + offset + timestamp + self.start = self.get_time(buffer[timestamp:timestamp+5]) + elif not hasattr(self, 'audio_ok'): + timestamp = c + offset + timestamp + start = self.get_time(buffer[timestamp:timestamp+5]) + if start is not None and self.start is not None and \ + abs(start - self.start) < 10: + # looks ok + self.audio_ok = True + else: + # timestamp broken + del self.start + log.warning('Timestamp error, correcting') + + if hasattr(self, 'start') and self.start and \ + self.sequence_header_offset and self.video and self.audio: + break + + c += TS_PACKET_LENGTH + + + if not self.sequence_header_offset: + return 0 + + # fill in values for support functions: + self.__seek_size__ = 10000000 # 10 MB + self.__sample_size__ = 100000 # 100 k scanning + self.__search__ = self._find_timer_TS_ + self.filename = file.name + + # get length of the file + self.length = self.get_length() + return 1 + + + def _find_timer_TS_(self, buffer): + c = 0 + + while c + TS_PACKET_LENGTH < len(buffer): + if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC: + break + c += 1 + else: + return None + + while c + TS_PACKET_LENGTH < len(buffer): + start = ord(buffer[c+1]) & 0x40 + if not start: + c += TS_PACKET_LENGTH + continue + + tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2]) + adapt = (ord(buffer[c+3]) & 0x30) >> 4 + + offset = 4 + if adapt & 0x02: + # meta info present, skip it for now + offset += ord(buffer[c+offset]) + 1 + + if adapt & 0x01: + timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], tsid)[1] + if timestamp is None: + # this should not happen + log.error('bad TS') + return None + return c + offset + timestamp + c += TS_PACKET_LENGTH + return None + + + + # Support functions ============================================== + + def get_endpos(self): + """ + get the last timestamp of the mpeg, return -1 if this is not possible + """ + if not hasattr(self, 'filename') or not hasattr(self, 'start'): + return None + + length = os.stat(self.filename)[stat.ST_SIZE] + if length < self.__sample_size__: + return + + file = open(self.filename) + file.seek(length - self.__sample_size__) + buffer = file.read(self.__sample_size__) + + end = None + while 1: + pos = self.__search__(buffer) + if pos == None: + break + end = self.get_time(buffer[pos:]) or end + buffer = buffer[pos+100:] + + file.close() + return end + + + def get_length(self): + """ + get the length in seconds, return -1 if this is not possible + """ + end = self.get_endpos() + if end == None or self.start == None: + return None + if self.start > end: + return int(((long(1) << 33) - 1 ) / 90000) - self.start + end + return end - self.start + + + def seek(self, end_time): + """ + Return the byte position in the file where the time position + is 'pos' seconds. Return 0 if this is not possible + """ + if not hasattr(self, 'filename') or not hasattr(self, 'start'): + return 0 + + file = open(self.filename) + seek_to = 0 + + while 1: + file.seek(self.__seek_size__, 1) + buffer = file.read(self.__sample_size__) + if len(buffer) < 10000: + break + pos = self.__search__(buffer) + if pos != None: + # found something + nt = self.get_time(buffer[pos:]) + if nt is not None and nt >= end_time: + # too much, break + break + # that wasn't enough + seek_to = file.tell() + + file.close() + return seek_to + + + def __scan__(self): + """ + scan file for timestamps (may take a long time) + """ + if not hasattr(self, 'filename') or not hasattr(self, 'start'): + return 0 + + file = open(self.filename) + log.debug('scanning file...') + while 1: + file.seek(self.__seek_size__ * 10, 1) + buffer = file.read(self.__sample_size__) + if len(buffer) < 10000: + break + pos = self.__search__(buffer) + if pos == None: + continue + log.debug('buffer position: %s' % self.get_time(buffer[pos:])) + + file.close() + log.debug('done scanning file') + + +Parser = MPEG diff --git a/libs/enzyme/ogm.py b/libs/enzyme/ogm.py new file mode 100644 index 00000000..6857cb66 --- /dev/null +++ b/libs/enzyme/ogm.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +import struct +import re +import stat +import os +import logging +from exceptions import * +import core + +# get logging object +log = logging.getLogger(__name__) + +PACKET_TYPE_HEADER = 0x01 +PACKED_TYPE_METADATA = 0x03 +PACKED_TYPE_SETUP = 0x05 +PACKET_TYPE_BITS = 0x07 +PACKET_IS_SYNCPOINT = 0x08 + +#VORBIS_VIDEO_PACKET_INFO = 'video' + +STREAM_HEADER_VIDEO = '<4sIQQIIHII' +STREAM_HEADER_AUDIO = '<4sIQQIIHHHI' + +VORBISCOMMENT = { 'TITLE': 'title', + 'ALBUM': 'album', + 'ARTIST': 'artist', + 'COMMENT': 'comment', + 'ENCODER': 'encoder', + 'TRACKNUMBER': 'trackno', + 'LANGUAGE': 'language', + 'GENRE': 'genre', + } + +# FIXME: check VORBISCOMMENT date and convert to timestamp +# Deactived tag: 'DATE': 'date', + +MAXITERATIONS = 30 + +class Ogm(core.AVContainer): + + table_mapping = { 'VORBISCOMMENT' : VORBISCOMMENT } + + def __init__(self, file): + core.AVContainer.__init__(self) + self.samplerate = 1 + self.all_streams = [] # used to add meta data to streams + self.all_header = [] + + for i in range(MAXITERATIONS): + granule, nextlen = self._parseOGGS(file) + if granule == None: + if i == 0: + # oops, bad file + raise ParseError() + break + elif granule > 0: + # ok, file started + break + + # seek to the end of the stream, to avoid scanning the whole file + if (os.stat(file.name)[stat.ST_SIZE] > 50000): + file.seek(os.stat(file.name)[stat.ST_SIZE]-49000) + + # read the rest of the file into a buffer + h = file.read() + + # find last OggS to get length info + if len(h) > 200: + idx = h.find('OggS') + pos = -49000 + idx + if idx: + file.seek(os.stat(file.name)[stat.ST_SIZE] + pos) + while 1: + granule, nextlen = self._parseOGGS(file) + if not nextlen: + break + + # Copy metadata to the streams + if len(self.all_header) == len(self.all_streams): + for i in range(len(self.all_header)): + + # get meta info + for key in self.all_streams[i].keys(): + if self.all_header[i].has_key(key): + self.all_streams[i][key] = self.all_header[i][key] + del self.all_header[i][key] + if self.all_header[i].has_key(key.upper()): + asi = self.all_header[i][key.upper()] + self.all_streams[i][key] = asi + del self.all_header[i][key.upper()] + + # Chapter parser + if self.all_header[i].has_key('CHAPTER01') and \ + not self.chapters: + while 1: + s = 'CHAPTER%02d' % (len(self.chapters) + 1) + if self.all_header[i].has_key(s) and \ + self.all_header[i].has_key(s + 'NAME'): + pos = self.all_header[i][s] + try: + pos = int(pos) + except ValueError: + new_pos = 0 + for v in pos.split(':'): + new_pos = new_pos * 60 + float(v) + pos = int(new_pos) + + c = self.all_header[i][s + 'NAME'] + c = core.Chapter(c, pos) + del self.all_header[i][s + 'NAME'] + del self.all_header[i][s] + self.chapters.append(c) + else: + break + + # If there are no video streams in this ogg container, it + # must be an audio file. Raise an exception to cause the + # factory to fall back to audio.ogg. + if len(self.video) == 0: + raise ParseError + + # Copy Metadata from tables into the main set of attributes + for header in self.all_header: + self._appendtable('VORBISCOMMENT', header) + + + def _parseOGGS(self,file): + h = file.read(27) + if len(h) == 0: + # Regular File end + return None, None + elif len(h) < 27: + log.debug("%d Bytes of Garbage found after End." % len(h)) + return None, None + if h[:4] != "OggS": + log.debug("Invalid Ogg") + raise ParseError() + + version = ord(h[4]) + if version != 0: + log.debug("Unsupported OGG/OGM Version %d." % version) + return None, None + + head = struct.unpack(' serial: + stream = self.all_streams[serial] + if hasattr(stream, 'samplerate') and \ + stream.samplerate: + stream.length = granulepos / stream.samplerate + elif hasattr(stream, 'bitrate') and \ + stream.bitrate: + stream.length = granulepos / stream.bitrate + + return granulepos, nextlen + 27 + pageSegCount + + + def _parseMeta(self,h): + flags = ord(h[0]) + headerlen = len(h) + if headerlen >= 7 and h[1:7] == 'vorbis': + header = {} + nextlen, self.encoder = self._extractHeaderString(h[7:]) + numItems = struct.unpack('= 30 and header[1:7] == 'vorbis': + ai = core.AudioStream() + ai.version, ai.channels, ai.samplerate, bitrate_max, ai.bitrate, \ + bitrate_min, blocksize, framing = \ + struct.unpack('= 7 and header[1:7] == 'theora': + # Theora Header + # XXX Finish Me + vi = core.VideoStream() + vi.codec = 'theora' + self.video.append(vi) + self.all_streams.append(vi) + + elif headerlen >= 142 and \ + header[1:36] == 'Direct Show Samples embedded in Ogg': + # Old Directshow format + # XXX Finish Me + vi = core.VideoStream() + vi.codec = 'dshow' + self.video.append(vi) + self.all_streams.append(vi) + + elif flags & PACKET_TYPE_BITS == PACKET_TYPE_HEADER and \ + headerlen >= struct.calcsize(STREAM_HEADER_VIDEO)+1: + # New Directshow Format + htype = header[1:9] + + if htype[:5] == 'video': + sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO)+9] + streamheader = struct.unpack(STREAM_HEADER_VIDEO, sh) + vi = core.VideoStream() + (type, ssize, timeunit, samplerate, vi.length, buffersize, \ + vi.bitrate, vi.width, vi.height) = streamheader + + vi.width /= 65536 + vi.height /= 65536 + # XXX length, bitrate are very wrong + vi.codec = type + vi.fps = 10000000 / timeunit + self.video.append(vi) + self.all_streams.append(vi) + + elif htype[:5] == 'audio': + sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO)+9] + streamheader = struct.unpack(STREAM_HEADER_AUDIO, sha) + ai = core.AudioStream() + (type, ssize, timeunit, ai.samplerate, ai.length, buffersize, \ + ai.bitrate, ai.channels, bloc, ai.bitrate) = streamheader + self.samplerate = ai.samplerate + log.debug("Samplerate %d" % self.samplerate) + self.audio.append(ai) + self.all_streams.append(ai) + + elif htype[:4] == 'text': + subtitle = core.Subtitle() + # FIXME: add more info + self.subtitles.append(subtitle) + self.all_streams.append(subtitle) + + else: + log.debug("Unknown Header") + + + def _extractHeaderString(self,header): + len = struct.unpack(' +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +import struct +import logging +from exceptions import * +import core + +# http://www.pcisys.net/~melanson/codecs/rmff.htm +# http://www.pcisys.net/~melanson/codecs/ + +# get logging object +log = logging.getLogger(__name__) + +class RealVideo(core.AVContainer): + def __init__(self,file): + core.AVContainer.__init__(self) + self.mime = 'video/real' + self.type = 'Real Video' + h = file.read(10) + try: + (object_id,object_size,object_version) = struct.unpack('>4sIH',h) + except struct.error: + # EOF. + raise ParseError() + + if not object_id == '.RMF': + raise ParseError() + + file_version, num_headers = struct.unpack('>II', file.read(8)) + log.debug("size: %d, ver: %d, headers: %d" % \ + (object_size, file_version,num_headers)) + for i in range(0,num_headers): + try: + oi = struct.unpack('>4sIH',file.read(10)) + except (struct.error, IOError): + # Header data we expected wasn't there. File may be + # only partially complete. + break + + if object_id == 'DATA' and oi[0] != 'INDX': + log.debug('INDX chunk expected after DATA but not found -- file corrupt') + break + + (object_id,object_size,object_version) = oi + if object_id == 'DATA': + # Seek over the data chunk rather than reading it in. + file.seek(object_size - 10, 1) + else: + self._read_header(object_id, file.read(object_size-10)) + log.debug("%s [%d]" % (object_id,object_size-10)) + # Read all the following headers + + + def _read_header(self,object_id,s): + if object_id == 'PROP': + prop = struct.unpack('>9IHH', s) + log.debug(prop) + if object_id == 'MDPR': + mdpr = struct.unpack('>H7I', s[:30]) + log.debug(mdpr) + self.length = mdpr[7]/1000.0 + (stream_name_size,) = struct.unpack('>B', s[30:31]) + stream_name = s[31:31+stream_name_size] + pos = 31+stream_name_size + (mime_type_size,) = struct.unpack('>B', s[pos:pos+1]) + mime = s[pos+1:pos+1+mime_type_size] + pos += mime_type_size+1 + (type_specific_len,) = struct.unpack('>I', s[pos:pos+4]) + type_specific = s[pos+4:pos+4+type_specific_len] + pos += 4+type_specific_len + if mime[:5] == 'audio': + ai = core.AudioStream() + ai.id = mdpr[0] + ai.bitrate = mdpr[2] + self.audio.append(ai) + elif mime[:5] == 'video': + vi = core.VideoStream() + vi.id = mdpr[0] + vi.bitrate = mdpr[2] + self.video.append(vi) + else: + log.debug("Unknown: %s" % mime) + if object_id == 'CONT': + pos = 0 + (title_len,) = struct.unpack('>H', s[pos:pos+2]) + self.title = s[2:title_len+2] + pos += title_len+2 + (author_len,) = struct.unpack('>H', s[pos:pos+2]) + self.artist = s[pos+2:pos+author_len+2] + pos += author_len+2 + (copyright_len,) = struct.unpack('>H', s[pos:pos+2]) + self.copyright = s[pos+2:pos+copyright_len+2] + pos += copyright_len+2 + (comment_len,) = struct.unpack('>H', s[pos:pos+2]) + self.comment = s[pos+2:pos+comment_len+2] + + +Parser = RealVideo diff --git a/libs/enzyme/riff.py b/libs/enzyme/riff.py new file mode 100644 index 00000000..221a5fc7 --- /dev/null +++ b/libs/enzyme/riff.py @@ -0,0 +1,568 @@ +# -*- coding: utf-8 -*- +# enzyme - Video metadata parser +# Copyright (C) 2011 Antoine Bertin +# Copyright (C) 2003-2006 Thomas Schueppel +# Copyright (C) 2003-2006 Dirk Meyer +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['Parser'] + +import os +import struct +import string +import logging +import time +from exceptions import * +import core + +# get logging object +log = logging.getLogger(__name__) + +# List of tags +# http://kibus1.narod.ru/frames_eng.htm?sof/abcavi/infotags.htm +# http://www.divx-digest.com/software/avitags_dll.html +# File Format: google for odmlff2.pdf + +AVIINFO = { + 'INAM': 'title', + 'IART': 'artist', + 'IPRD': 'product', + 'ISFT': 'software', + 'ICMT': 'comment', + 'ILNG': 'language', + 'IKEY': 'keywords', + 'IPRT': 'trackno', + 'IFRM': 'trackof', + 'IPRO': 'producer', + 'IWRI': 'writer', + 'IGNR': 'genre', + 'ICOP': 'copyright' +} + +# Taken from libavcodec/mpeg4data.h (pixel_aspect struct) +PIXEL_ASPECT = { + 1: (1, 1), + 2: (12, 11), + 3: (10, 11), + 4: (16, 11), + 5: (40, 33) +} + + +class Riff(core.AVContainer): + """ + AVI parser also parsing metadata like title, languages, etc. + """ + table_mapping = { 'AVIINFO' : AVIINFO } + + def __init__(self,file): + core.AVContainer.__init__(self) + # read the header + h = file.read(12) + if h[:4] != "RIFF" and h[:4] != 'SDSS': + raise ParseError() + + self.has_idx = False + self.header = {} + self.junkStart = None + self.infoStart = None + self.type = h[8:12] + if self.type == 'AVI ': + self.mime = 'video/avi' + elif self.type == 'WAVE': + self.mime = 'audio/wav' + try: + while self._parseRIFFChunk(file): + pass + except IOError: + log.exception('error in file, stop parsing') + + self._find_subtitles(file.name) + + if not self.has_idx and isinstance(self, core.AVContainer): + log.debug('WARNING: avi has no index') + self._set('corrupt', True) + + + def _find_subtitles(self, filename): + """ + Search for subtitle files. Right now only VobSub is supported + """ + base = os.path.splitext(filename)[0] + if os.path.isfile(base+'.idx') and \ + (os.path.isfile(base+'.sub') or os.path.isfile(base+'.rar')): + file = open(base+'.idx') + if file.readline().find('VobSub index file') > 0: + for line in file.readlines(): + if line.find('id') == 0: + sub = core.Subtitle() + sub.language = line[4:6] + sub.trackno = base + '.idx' # Maybe not? + self.subtitles.append(sub) + file.close() + + + def _parseAVIH(self,t): + retval = {} + v = struct.unpack('> 16) / (r & 0xFFFF) + retval['FrameAspectRatio'] = r + if self.video: + map(lambda v: setattr(v, 'aspect', r), self.video) + return (retval, v[0]) + + + def _parseLISTmovi(self, size, file): + """ + Digs into movi list, looking for a Video Object Layer header in an + mpeg4 stream in order to determine aspect ratio. + """ + i = 0 + n_dc = 0 + done = False + # If the VOL header doesn't appear within 5MB or 5 video chunks, + # give up. The 5MB limit is not likely to apply except in + # pathological cases. + while i < min(1024*1024*5, size - 8) and n_dc < 5: + data = file.read(8) + if ord(data[0]) == 0: + # Eat leading nulls. + data = data[1:] + file.read(1) + i += 1 + + key, sz = struct.unpack('<4sI', data) + if key[2:] != 'dc' or sz > 1024*500: + # This chunk is not video or is unusually big (> 500KB); + # skip it. + file.seek(sz, 1) + i += 8 + sz + continue + + n_dc += 1 + # Read video chunk into memory + data = file.read(sz) + + #for p in range(0,min(80, sz)): + # print "%02x " % ord(data[p]), + #print "\n\n" + + # Look through the picture header for VOL startcode. The basic + # logic for this is taken from libavcodec, h263.c + pos = 0 + startcode = 0xff + def bits(v, o, n): + # Returns n bits in v, offset o bits. + return (v & 2**n-1 << (64-n-o)) >> 64-n-o + + while pos < sz: + startcode = ((startcode << 8) | ord(data[pos])) & 0xffffffff + pos += 1 + if startcode & 0xFFFFFF00 != 0x100: + # No startcode found yet + continue + + if startcode >= 0x120 and startcode <= 0x12F: + # We have the VOL startcode. Pull 64 bits of it and treat + # as a bitstream + v = struct.unpack(">Q", data[pos : pos+8])[0] + offset = 10 + if bits(v, 9, 1): + # is_ol_id, skip over vo_ver_id and vo_priority + offset += 7 + ar_info = bits(v, offset, 4) + if ar_info == 15: + # Extended aspect + num = bits(v, offset + 4, 8) + den = bits(v, offset + 12, 8) + else: + # A standard pixel aspect + num, den = PIXEL_ASPECT.get(ar_info, (0, 0)) + + # num/den indicates pixel aspect; convert to video aspect, + # so we need frame width and height. + if 0 not in [num, den]: + width, height = self.video[-1].width, self.video[-1].height + self.video[-1].aspect = num / float(den) * width / height + + done = True + break + + startcode = 0xff + + i += 8 + len(data) + + if done: + # We have the aspect, no need to continue parsing the movi + # list, so break out of the loop. + break + + + if i < size: + # Seek past whatever might be remaining of the movi list. + file.seek(size-i,1) + + + + def _parseLIST(self,t): + retval = {} + i = 0 + size = len(t) + + while i < size-8: + # skip zero + if ord(t[i]) == 0: i += 1 + key = t[i:i+4] + sz = 0 + + if key == 'LIST': + sz = struct.unpack(' 0: + # works for Casio S500 camera videos + value = value[:value.find('\0')] + value = value.replace('\0', '').lstrip().rstrip() + value = value.replace('\0', '').lstrip().rstrip() + if value: + retval[key] = value + if key in ['IDIT', 'ICRD']: + # Timestamp the video was created. Spec says it + # should be a format like "Wed Jan 02 02:03:55 1990" + # Casio S500 uses "2005/12/24/ 14:11", but I've + # also seen "December 24, 2005" + specs = ('%a %b %d %H:%M:%S %Y', '%Y/%m/%d/ %H:%M', '%B %d, %Y') + for tmspec in specs: + try: + tm = time.strptime(value, tmspec) + # save timestamp as int + self.timestamp = int(time.mktime(tm)) + break + except ValueError: + pass + else: + log.debug('no support for time format %s', value) + i+=sz + return retval + + + def _parseRIFFChunk(self,file): + h = file.read(8) + if len(h) < 8: + return False + name = h[:4] + size = struct.unpack(' 80000: + log.debug('RIFF LIST "%s" too long to parse: %s bytes' % (key, size)) + t = file.seek(size-4,1) + return True + elif size < 5: + log.debug('RIFF LIST "%s" too short: %s bytes' % (key, size)) + return True + + t = file.read(size-4) + log.debug('parse RIFF LIST "%s": %d bytes' % (key, size)) + value = self._parseLIST(t) + self.header[key] = value + if key == 'INFO': + self.infoStart = pos + self._appendtable('AVIINFO', value) + elif key == 'MID ': + self._appendtable('AVIMID', value) + elif key == 'hdrl': + # no need to add this info to a table + pass + else: + log.debug('Skipping table info %s' % key) + + elif name == 'JUNK': + self.junkStart = file.tell() - 8 + self.junkSize = size + file.seek(size, 1) + elif name == 'idx1': + self.has_idx = True + log.debug('idx1: %s bytes' % size) + # no need to parse this + t = file.seek(size,1) + elif name == 'RIFF': + log.debug("New RIFF chunk, extended avi [%i]" % size) + type = file.read(4) + if type != 'AVIX': + log.debug("Second RIFF chunk is %s, not AVIX, skipping", type) + file.seek(size-4, 1) + # that's it, no new informations should be in AVIX + return False + elif name == 'fmt ' and size <= 50: + # This is a wav file. + data = file.read(size) + fmt = struct.unpack(" +# Copyright (C) 2006-2009 Dirk Meyer +# Copyright (C) 2006-2009 Jason Tackaberry +# +# This file is part of enzyme. +# +# enzyme is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# enzyme is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +__all__ = ['ENCODING', 'str_to_unicode', 'unicode_to_str'] + +import locale + +# find the correct encoding +try: + ENCODING = locale.getdefaultlocale()[1] + ''.encode(ENCODING) +except (UnicodeError, TypeError): + ENCODING = 'latin-1' + + +def str_to_unicode(s, encoding=None): + """ + Attempts to convert a string of unknown character set to a unicode + string. First it tries to decode the string based on the locale's + preferred encoding, and if that fails, fall back to UTF-8 and then + latin-1. If all fails, it will force encoding to the preferred + charset, replacing unknown characters. If the given object is no + string, this function will return the given object. + """ + if not type(s) == str: + return s + + if not encoding: + encoding = ENCODING + + for c in [encoding, "utf-8", "latin-1"]: + try: + return s.decode(c) + except UnicodeDecodeError: + pass + + return s.decode(encoding, "replace") + + +def unicode_to_str(s, encoding=None): + """ + Attempts to convert a unicode string of unknown character set to a + string. First it tries to encode the string based on the locale's + preferred encoding, and if that fails, fall back to UTF-8 and then + latin-1. If all fails, it will force encoding to the preferred + charset, replacing unknown characters. If the given object is no + unicode string, this function will return the given object. + """ + if not type(s) == unicode: + return s + + if not encoding: + encoding = ENCODING + + for c in [encoding, "utf-8", "latin-1"]: + try: + return s.encode(c) + except UnicodeDecodeError: + pass + + return s.encode(encoding, "replace") diff --git a/libs/guessit/ISO-639-2_utf-8.txt b/libs/guessit/ISO-639-2_utf-8.txt new file mode 100644 index 00000000..2961d219 --- /dev/null +++ b/libs/guessit/ISO-639-2_utf-8.txt @@ -0,0 +1,485 @@ +aar||aa|Afar|afar +abk||ab|Abkhazian|abkhaze +ace|||Achinese|aceh +ach|||Acoli|acoli +ada|||Adangme|adangme +ady|||Adyghe; Adygei|adyghé +afa|||Afro-Asiatic languages|afro-asiatiques, langues +afh|||Afrihili|afrihili +afr||af|Afrikaans|afrikaans +ain|||Ainu|aïnou +aka||ak|Akan|akan +akk|||Akkadian|akkadien +alb|sqi|sq|Albanian|albanais +ale|||Aleut|aléoute +alg|||Algonquian languages|algonquines, langues +alt|||Southern Altai|altai du Sud +amh||am|Amharic|amharique +ang|||English, Old (ca.450-1100)|anglo-saxon (ca.450-1100) +anp|||Angika|angika +apa|||Apache languages|apaches, langues +ara||ar|Arabic|arabe +arc|||Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)|araméen d'empire (700-300 BCE) +arg||an|Aragonese|aragonais +arm|hye|hy|Armenian|arménien +arn|||Mapudungun; Mapuche|mapudungun; mapuche; mapuce +arp|||Arapaho|arapaho +art|||Artificial languages|artificielles, langues +arw|||Arawak|arawak +asm||as|Assamese|assamais +ast|||Asturian; Bable; Leonese; Asturleonese|asturien; bable; léonais; asturoléonais +ath|||Athapascan languages|athapascanes, langues +aus|||Australian languages|australiennes, langues +ava||av|Avaric|avar +ave||ae|Avestan|avestique +awa|||Awadhi|awadhi +aym||ay|Aymara|aymara +aze||az|Azerbaijani|azéri +bad|||Banda languages|banda, langues +bai|||Bamileke languages|bamiléké, langues +bak||ba|Bashkir|bachkir +bal|||Baluchi|baloutchi +bam||bm|Bambara|bambara +ban|||Balinese|balinais +baq|eus|eu|Basque|basque +bas|||Basa|basa +bat|||Baltic languages|baltes, langues +bej|||Beja; Bedawiyet|bedja +bel||be|Belarusian|biélorusse +bem|||Bemba|bemba +ben||bn|Bengali|bengali +ber|||Berber languages|berbères, langues +bho|||Bhojpuri|bhojpuri +bih||bh|Bihari languages|langues biharis +bik|||Bikol|bikol +bin|||Bini; Edo|bini; edo +bis||bi|Bislama|bichlamar +bla|||Siksika|blackfoot +bnt|||Bantu (Other)|bantoues, autres langues +bos||bs|Bosnian|bosniaque +bra|||Braj|braj +bre||br|Breton|breton +btk|||Batak languages|batak, langues +bua|||Buriat|bouriate +bug|||Buginese|bugi +bul||bg|Bulgarian|bulgare +bur|mya|my|Burmese|birman +byn|||Blin; Bilin|blin; bilen +cad|||Caddo|caddo +cai|||Central American Indian languages|amérindiennes de L'Amérique centrale, langues +car|||Galibi Carib|karib; galibi; carib +cat||ca|Catalan; Valencian|catalan; valencien +cau|||Caucasian languages|caucasiennes, langues +ceb|||Cebuano|cebuano +cel|||Celtic languages|celtiques, langues; celtes, langues +cha||ch|Chamorro|chamorro +chb|||Chibcha|chibcha +che||ce|Chechen|tchétchène +chg|||Chagatai|djaghataï +chi|zho|zh|Chinese|chinois +chk|||Chuukese|chuuk +chm|||Mari|mari +chn|||Chinook jargon|chinook, jargon +cho|||Choctaw|choctaw +chp|||Chipewyan; Dene Suline|chipewyan +chr|||Cherokee|cherokee +chu||cu|Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic|slavon d'église; vieux slave; slavon liturgique; vieux bulgare +chv||cv|Chuvash|tchouvache +chy|||Cheyenne|cheyenne +cmc|||Chamic languages|chames, langues +cop|||Coptic|copte +cor||kw|Cornish|cornique +cos||co|Corsican|corse +cpe|||Creoles and pidgins, English based|créoles et pidgins basés sur l'anglais +cpf|||Creoles and pidgins, French-based |créoles et pidgins basés sur le français +cpp|||Creoles and pidgins, Portuguese-based |créoles et pidgins basés sur le portugais +cre||cr|Cree|cree +crh|||Crimean Tatar; Crimean Turkish|tatar de Crimé +crp|||Creoles and pidgins |créoles et pidgins +csb|||Kashubian|kachoube +cus|||Cushitic languages|couchitiques, langues +cze|ces|cs|Czech|tchèque +dak|||Dakota|dakota +dan||da|Danish|danois +dar|||Dargwa|dargwa +day|||Land Dayak languages|dayak, langues +del|||Delaware|delaware +den|||Slave (Athapascan)|esclave (athapascan) +dgr|||Dogrib|dogrib +din|||Dinka|dinka +div||dv|Divehi; Dhivehi; Maldivian|maldivien +doi|||Dogri|dogri +dra|||Dravidian languages|dravidiennes, langues +dsb|||Lower Sorbian|bas-sorabe +dua|||Duala|douala +dum|||Dutch, Middle (ca.1050-1350)|néerlandais moyen (ca. 1050-1350) +dut|nld|nl|Dutch; Flemish|néerlandais; flamand +dyu|||Dyula|dioula +dzo||dz|Dzongkha|dzongkha +efi|||Efik|efik +egy|||Egyptian (Ancient)|égyptien +eka|||Ekajuk|ekajuk +elx|||Elamite|élamite +eng||en|English|anglais +enm|||English, Middle (1100-1500)|anglais moyen (1100-1500) +epo||eo|Esperanto|espéranto +est||et|Estonian|estonien +ewe||ee|Ewe|éwé +ewo|||Ewondo|éwondo +fan|||Fang|fang +fao||fo|Faroese|féroïen +fat|||Fanti|fanti +fij||fj|Fijian|fidjien +fil|||Filipino; Pilipino|filipino; pilipino +fin||fi|Finnish|finnois +fiu|||Finno-Ugrian languages|finno-ougriennes, langues +fon|||Fon|fon +fre|fra|fr|French|français +frm|||French, Middle (ca.1400-1600)|français moyen (1400-1600) +fro|||French, Old (842-ca.1400)|français ancien (842-ca.1400) +frr|||Northern Frisian|frison septentrional +frs|||Eastern Frisian|frison oriental +fry||fy|Western Frisian|frison occidental +ful||ff|Fulah|peul +fur|||Friulian|frioulan +gaa|||Ga|ga +gay|||Gayo|gayo +gba|||Gbaya|gbaya +gem|||Germanic languages|germaniques, langues +geo|kat|ka|Georgian|géorgien +ger|deu|de|German|allemand +gez|||Geez|guèze +gil|||Gilbertese|kiribati +gla||gd|Gaelic; Scottish Gaelic|gaélique; gaélique écossais +gle||ga|Irish|irlandais +glg||gl|Galician|galicien +glv||gv|Manx|manx; mannois +gmh|||German, Middle High (ca.1050-1500)|allemand, moyen haut (ca. 1050-1500) +goh|||German, Old High (ca.750-1050)|allemand, vieux haut (ca. 750-1050) +gon|||Gondi|gond +gor|||Gorontalo|gorontalo +got|||Gothic|gothique +grb|||Grebo|grebo +grc|||Greek, Ancient (to 1453)|grec ancien (jusqu'à 1453) +gre|ell|el|Greek, Modern (1453-)|grec moderne (après 1453) +grn||gn|Guarani|guarani +gsw|||Swiss German; Alemannic; Alsatian|suisse alémanique; alémanique; alsacien +guj||gu|Gujarati|goudjrati +gwi|||Gwich'in|gwich'in +hai|||Haida|haida +hat||ht|Haitian; Haitian Creole|haïtien; créole haïtien +hau||ha|Hausa|haoussa +haw|||Hawaiian|hawaïen +heb||he|Hebrew|hébreu +her||hz|Herero|herero +hil|||Hiligaynon|hiligaynon +him|||Himachali languages; Western Pahari languages|langues himachalis; langues paharis occidentales +hin||hi|Hindi|hindi +hit|||Hittite|hittite +hmn|||Hmong; Mong|hmong +hmo||ho|Hiri Motu|hiri motu +hrv||hr|Croatian|croate +hsb|||Upper Sorbian|haut-sorabe +hun||hu|Hungarian|hongrois +hup|||Hupa|hupa +iba|||Iban|iban +ibo||ig|Igbo|igbo +ice|isl|is|Icelandic|islandais +ido||io|Ido|ido +iii||ii|Sichuan Yi; Nuosu|yi de Sichuan +ijo|||Ijo languages|ijo, langues +iku||iu|Inuktitut|inuktitut +ile||ie|Interlingue; Occidental|interlingue +ilo|||Iloko|ilocano +ina||ia|Interlingua (International Auxiliary Language Association)|interlingua (langue auxiliaire internationale) +inc|||Indic languages|indo-aryennes, langues +ind||id|Indonesian|indonésien +ine|||Indo-European languages|indo-européennes, langues +inh|||Ingush|ingouche +ipk||ik|Inupiaq|inupiaq +ira|||Iranian languages|iraniennes, langues +iro|||Iroquoian languages|iroquoises, langues +ita||it|Italian|italien +jav||jv|Javanese|javanais +jbo|||Lojban|lojban +jpn||ja|Japanese|japonais +jpr|||Judeo-Persian|judéo-persan +jrb|||Judeo-Arabic|judéo-arabe +kaa|||Kara-Kalpak|karakalpak +kab|||Kabyle|kabyle +kac|||Kachin; Jingpho|kachin; jingpho +kal||kl|Kalaallisut; Greenlandic|groenlandais +kam|||Kamba|kamba +kan||kn|Kannada|kannada +kar|||Karen languages|karen, langues +kas||ks|Kashmiri|kashmiri +kau||kr|Kanuri|kanouri +kaw|||Kawi|kawi +kaz||kk|Kazakh|kazakh +kbd|||Kabardian|kabardien +kha|||Khasi|khasi +khi|||Khoisan languages|khoïsan, langues +khm||km|Central Khmer|khmer central +kho|||Khotanese; Sakan|khotanais; sakan +kik||ki|Kikuyu; Gikuyu|kikuyu +kin||rw|Kinyarwanda|rwanda +kir||ky|Kirghiz; Kyrgyz|kirghiz +kmb|||Kimbundu|kimbundu +kok|||Konkani|konkani +kom||kv|Komi|kom +kon||kg|Kongo|kongo +kor||ko|Korean|coréen +kos|||Kosraean|kosrae +kpe|||Kpelle|kpellé +krc|||Karachay-Balkar|karatchai balkar +krl|||Karelian|carélien +kro|||Kru languages|krou, langues +kru|||Kurukh|kurukh +kua||kj|Kuanyama; Kwanyama|kuanyama; kwanyama +kum|||Kumyk|koumyk +kur||ku|Kurdish|kurde +kut|||Kutenai|kutenai +lad|||Ladino|judéo-espagnol +lah|||Lahnda|lahnda +lam|||Lamba|lamba +lao||lo|Lao|lao +lat||la|Latin|latin +lav||lv|Latvian|letton +lez|||Lezghian|lezghien +lim||li|Limburgan; Limburger; Limburgish|limbourgeois +lin||ln|Lingala|lingala +lit||lt|Lithuanian|lituanien +lol|||Mongo|mongo +loz|||Lozi|lozi +ltz||lb|Luxembourgish; Letzeburgesch|luxembourgeois +lua|||Luba-Lulua|luba-lulua +lub||lu|Luba-Katanga|luba-katanga +lug||lg|Ganda|ganda +lui|||Luiseno|luiseno +lun|||Lunda|lunda +luo|||Luo (Kenya and Tanzania)|luo (Kenya et Tanzanie) +lus|||Lushai|lushai +mac|mkd|mk|Macedonian|macédonien +mad|||Madurese|madourais +mag|||Magahi|magahi +mah||mh|Marshallese|marshall +mai|||Maithili|maithili +mak|||Makasar|makassar +mal||ml|Malayalam|malayalam +man|||Mandingo|mandingue +mao|mri|mi|Maori|maori +map|||Austronesian languages|austronésiennes, langues +mar||mr|Marathi|marathe +mas|||Masai|massaï +may|msa|ms|Malay|malais +mdf|||Moksha|moksa +mdr|||Mandar|mandar +men|||Mende|mendé +mga|||Irish, Middle (900-1200)|irlandais moyen (900-1200) +mic|||Mi'kmaq; Micmac|mi'kmaq; micmac +min|||Minangkabau|minangkabau +mis|||Uncoded languages|langues non codées +mkh|||Mon-Khmer languages|môn-khmer, langues +mlg||mg|Malagasy|malgache +mlt||mt|Maltese|maltais +mnc|||Manchu|mandchou +mni|||Manipuri|manipuri +mno|||Manobo languages|manobo, langues +moh|||Mohawk|mohawk +mon||mn|Mongolian|mongol +mos|||Mossi|moré +mul|||Multiple languages|multilingue +mun|||Munda languages|mounda, langues +mus|||Creek|muskogee +mwl|||Mirandese|mirandais +mwr|||Marwari|marvari +myn|||Mayan languages|maya, langues +myv|||Erzya|erza +nah|||Nahuatl languages|nahuatl, langues +nai|||North American Indian languages|nord-amérindiennes, langues +nap|||Neapolitan|napolitain +nau||na|Nauru|nauruan +nav||nv|Navajo; Navaho|navaho +nbl||nr|Ndebele, South; South Ndebele|ndébélé du Sud +nde||nd|Ndebele, North; North Ndebele|ndébélé du Nord +ndo||ng|Ndonga|ndonga +nds|||Low German; Low Saxon; German, Low; Saxon, Low|bas allemand; bas saxon; allemand, bas; saxon, bas +nep||ne|Nepali|népalais +new|||Nepal Bhasa; Newari|nepal bhasa; newari +nia|||Nias|nias +nic|||Niger-Kordofanian languages|nigéro-kordofaniennes, langues +niu|||Niuean|niué +nno||nn|Norwegian Nynorsk; Nynorsk, Norwegian|norvégien nynorsk; nynorsk, norvégien +nob||nb|Bokmål, Norwegian; Norwegian Bokmål|norvégien bokmål +nog|||Nogai|nogaï; nogay +non|||Norse, Old|norrois, vieux +nor||no|Norwegian|norvégien +nqo|||N'Ko|n'ko +nso|||Pedi; Sepedi; Northern Sotho|pedi; sepedi; sotho du Nord +nub|||Nubian languages|nubiennes, langues +nwc|||Classical Newari; Old Newari; Classical Nepal Bhasa|newari classique +nya||ny|Chichewa; Chewa; Nyanja|chichewa; chewa; nyanja +nym|||Nyamwezi|nyamwezi +nyn|||Nyankole|nyankolé +nyo|||Nyoro|nyoro +nzi|||Nzima|nzema +oci||oc|Occitan (post 1500); Provençal|occitan (après 1500); provençal +oji||oj|Ojibwa|ojibwa +ori||or|Oriya|oriya +orm||om|Oromo|galla +osa|||Osage|osage +oss||os|Ossetian; Ossetic|ossète +ota|||Turkish, Ottoman (1500-1928)|turc ottoman (1500-1928) +oto|||Otomian languages|otomi, langues +paa|||Papuan languages|papoues, langues +pag|||Pangasinan|pangasinan +pal|||Pahlavi|pahlavi +pam|||Pampanga; Kapampangan|pampangan +pan||pa|Panjabi; Punjabi|pendjabi +pap|||Papiamento|papiamento +pau|||Palauan|palau +peo|||Persian, Old (ca.600-400 B.C.)|perse, vieux (ca. 600-400 av. J.-C.) +per|fas|fa|Persian|persan +phi|||Philippine languages|philippines, langues +phn|||Phoenician|phénicien +pli||pi|Pali|pali +pol||pl|Polish|polonais +pon|||Pohnpeian|pohnpei +por||pt|Portuguese|portugais +pra|||Prakrit languages|prâkrit, langues +pro|||Provençal, Old (to 1500)|provençal ancien (jusqu'à 1500) +pus||ps|Pushto; Pashto|pachto +qaa-qtz|||Reserved for local use|réservée à l'usage local +que||qu|Quechua|quechua +raj|||Rajasthani|rajasthani +rap|||Rapanui|rapanui +rar|||Rarotongan; Cook Islands Maori|rarotonga; maori des îles Cook +roa|||Romance languages|romanes, langues +roh||rm|Romansh|romanche +rom|||Romany|tsigane +rum|ron|ro|Romanian; Moldavian; Moldovan|roumain; moldave +run||rn|Rundi|rundi +rup|||Aromanian; Arumanian; Macedo-Romanian|aroumain; macédo-roumain +rus||ru|Russian|russe +sad|||Sandawe|sandawe +sag||sg|Sango|sango +sah|||Yakut|iakoute +sai|||South American Indian (Other)|indiennes d'Amérique du Sud, autres langues +sal|||Salishan languages|salishennes, langues +sam|||Samaritan Aramaic|samaritain +san||sa|Sanskrit|sanskrit +sas|||Sasak|sasak +sat|||Santali|santal +scn|||Sicilian|sicilien +sco|||Scots|écossais +sel|||Selkup|selkoupe +sem|||Semitic languages|sémitiques, langues +sga|||Irish, Old (to 900)|irlandais ancien (jusqu'à 900) +sgn|||Sign Languages|langues des signes +shn|||Shan|chan +sid|||Sidamo|sidamo +sin||si|Sinhala; Sinhalese|singhalais +sio|||Siouan languages|sioux, langues +sit|||Sino-Tibetan languages|sino-tibétaines, langues +sla|||Slavic languages|slaves, langues +slo|slk|sk|Slovak|slovaque +slv||sl|Slovenian|slovène +sma|||Southern Sami|sami du Sud +sme||se|Northern Sami|sami du Nord +smi|||Sami languages|sames, langues +smj|||Lule Sami|sami de Lule +smn|||Inari Sami|sami d'Inari +smo||sm|Samoan|samoan +sms|||Skolt Sami|sami skolt +sna||sn|Shona|shona +snd||sd|Sindhi|sindhi +snk|||Soninke|soninké +sog|||Sogdian|sogdien +som||so|Somali|somali +son|||Songhai languages|songhai, langues +sot||st|Sotho, Southern|sotho du Sud +spa||es|Spanish; Castilian|espagnol; castillan +srd||sc|Sardinian|sarde +srn|||Sranan Tongo|sranan tongo +srp||sr|Serbian|serbe +srr|||Serer|sérère +ssa|||Nilo-Saharan languages|nilo-sahariennes, langues +ssw||ss|Swati|swati +suk|||Sukuma|sukuma +sun||su|Sundanese|soundanais +sus|||Susu|soussou +sux|||Sumerian|sumérien +swa||sw|Swahili|swahili +swe||sv|Swedish|suédois +syc|||Classical Syriac|syriaque classique +syr|||Syriac|syriaque +tah||ty|Tahitian|tahitien +tai|||Tai languages|tai, langues +tam||ta|Tamil|tamoul +tat||tt|Tatar|tatar +tel||te|Telugu|télougou +tem|||Timne|temne +ter|||Tereno|tereno +tet|||Tetum|tetum +tgk||tg|Tajik|tadjik +tgl||tl|Tagalog|tagalog +tha||th|Thai|thaï +tib|bod|bo|Tibetan|tibétain +tig|||Tigre|tigré +tir||ti|Tigrinya|tigrigna +tiv|||Tiv|tiv +tkl|||Tokelau|tokelau +tlh|||Klingon; tlhIngan-Hol|klingon +tli|||Tlingit|tlingit +tmh|||Tamashek|tamacheq +tog|||Tonga (Nyasa)|tonga (Nyasa) +ton||to|Tonga (Tonga Islands)|tongan (Îles Tonga) +tpi|||Tok Pisin|tok pisin +tsi|||Tsimshian|tsimshian +tsn||tn|Tswana|tswana +tso||ts|Tsonga|tsonga +tuk||tk|Turkmen|turkmène +tum|||Tumbuka|tumbuka +tup|||Tupi languages|tupi, langues +tur||tr|Turkish|turc +tut|||Altaic languages|altaïques, langues +tvl|||Tuvalu|tuvalu +twi||tw|Twi|twi +tyv|||Tuvinian|touva +udm|||Udmurt|oudmourte +uga|||Ugaritic|ougaritique +uig||ug|Uighur; Uyghur|ouïgour +ukr||uk|Ukrainian|ukrainien +umb|||Umbundu|umbundu +und|||Undetermined|indéterminée +urd||ur|Urdu|ourdou +uzb||uz|Uzbek|ouszbek +vai|||Vai|vaï +ven||ve|Venda|venda +vie||vi|Vietnamese|vietnamien +vol||vo|Volapük|volapük +vot|||Votic|vote +wak|||Wakashan languages|wakashanes, langues +wal|||Walamo|walamo +war|||Waray|waray +was|||Washo|washo +wel|cym|cy|Welsh|gallois +wen|||Sorbian languages|sorabes, langues +wln||wa|Walloon|wallon +wol||wo|Wolof|wolof +xal|||Kalmyk; Oirat|kalmouk; oïrat +xho||xh|Xhosa|xhosa +yao|||Yao|yao +yap|||Yapese|yapois +yid||yi|Yiddish|yiddish +yor||yo|Yoruba|yoruba +ypk|||Yupik languages|yupik, langues +zap|||Zapotec|zapotèque +zbl|||Blissymbols; Blissymbolics; Bliss|symboles Bliss; Bliss +zen|||Zenaga|zenaga +zha||za|Zhuang; Chuang|zhuang; chuang +znd|||Zande languages|zandé, langues +zul||zu|Zulu|zoulou +zun|||Zuni|zuni +zxx|||No linguistic content; Not applicable|pas de contenu linguistique; non applicable +zza|||Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki|zaza; dimili; dimli; kirdki; kirmanjki; zazaki \ No newline at end of file diff --git a/libs/guessit/__init__.py b/libs/guessit/__init__.py new file mode 100644 index 00000000..7c58efa5 --- /dev/null +++ b/libs/guessit/__init__.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +__version__ = '0.2' +__all__ = [ 'Guess', 'Language', + 'guess_file_info', 'guess_video_info', + 'guess_movie_info', 'guess_episode_info' ] + + +from guessit.guess import Guess, merge_all +from guessit.language import Language +from guessit.matcher import IterativeMatcher +import logging + +log = logging.getLogger("guessit") + +class NullHandler(logging.Handler): + def emit(self, record): + pass + +# let's be a nicely behaving library +h = NullHandler() +log.addHandler(h) + + + + +def guess_file_info(filename, filetype, info = [ 'filename' ]): + """info can contain the names of the various plugins, such as 'filename' to + detect filename info, or 'hash_md5' to get the md5 hash of the file. + + >>> guess_file_info('test/dummy.srt', 'autodetect', info = ['hash_md5', 'hash_sha1']) + {'hash_md5': 'e781de9b94ba2753a8e2945b2c0a123d', 'hash_sha1': 'bfd18e2f4e5d59775c2bc14d80f56971891ed620'} + """ + result = [] + hashers = [] + + for infotype in info: + if infotype == 'filename': + m = IterativeMatcher(filename, filetype = filetype) + result.append(m.matched()) + + elif infotype == 'hash_mpc': + import hash_mpc + try: + result.append(Guess({ 'hash_mpc': hash_mpc.hash_file(filename) }, + confidence = 1.0)) + except Exception, e: + log.warning('Could not compute MPC-style hash because: %s' % e) + + elif infotype == 'hash_ed2k': + import hash_ed2k + try: + result.append(Guess({ 'hash_ed2k': hash_ed2k.hash_file(filename) }, + confidence = 1.0)) + except Exception, e: + log.warning('Could not compute ed2k hash because: %s' % e) + + elif infotype.startswith('hash_'): + import hashlib + hashname = infotype[5:] + try: + hasher = getattr(hashlib, hashname)() + hashers.append((infotype, hasher)) + except AttributeError: + log.warning('Could not compute %s hash because it is not available from python\'s hashlib module' % hashname) + + else: + log.warning('Invalid infotype: %s' % infotype) + + + """For plugins which depend on some optional library, import them like that: + + if infotype == 'plugin_name': + try: + import optional_lib + except ImportError: + raise Exception, 'The plugin module cannot be loaded because the optional_lib lib is missing' + + # do some stuff + """ + + # do all the hashes now, but on a single pass + if hashers: + try: + blocksize = 8192 + hasherobjs = dict(hashers).values() + + with open(filename, 'rb') as f: + for chunk in iter(lambda: f.read(blocksize), ''): + for hasher in hasherobjs: + hasher.update(chunk) + + for infotype, hasher in hashers: + result.append(Guess({ infotype: hasher.hexdigest() }, + confidence = 1.0)) + except Exception, e: + log.warning('Could not compute hash because: %s' % e) + + + return merge_all(result) + + +def guess_video_info(filename, info = [ 'filename' ]): + return guess_file_info(filename, 'autodetect', info) + +def guess_movie_info(filename, info = [ 'filename' ]): + return guess_file_info(filename, 'movie', info) + +def guess_episode_info(filename, info = [ 'filename' ]): + return guess_file_info(filename, 'episode', info) + + diff --git a/libs/guessit/autodetect.py b/libs/guessit/autodetect.py new file mode 100644 index 00000000..fe05b68b --- /dev/null +++ b/libs/guessit/autodetect.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +#from guessit import movie, episode +import os, os.path +import logging + +log = logging.getLogger('guessit.autodetect') + +def within(x, nrange): + """Return whether a number is inside a given range, specified as a list or tuple + of the lower and upper bounds.""" + low, high = nrange + return low <= x <= high + +def guess_filename_info(filename): + log.debug('Trying to guess info for file: ' + filename) + + # try to guess info as if it were an episode + episode_info = episode.guess_episode_filename(filename) + + # 1- if we found either season/episodeNumber, then we're pretty sure it must + # be an episode + if 'season' in episode_info or 'episodeNumber' in episode_info: + log.debug('Likely an episode as it contains season and/or episodeNumber: ' + filename) + episode_info.update({ 'type': 'episode' }, confidence = 0.9) + return episode_info + + # try to guess info as if it were a movie + movie_info = movie.guess_movie_filename(filename) + + # 2- if the file exists, try to guess its type using its size + if os.path.exists(filename): + size = os.stat(filename).st_size / (1024 * 1024) + + # if size <= 1/2 of 1CD -> episode (very unlikely a movie so small) + if size < 400: + log.debug('Likely an episode due to its small size (%dMB): %s' % (size, filename)) + episode_info.update({ 'type': 'episode' }, confidence = 0.8) + return episode_info + + # if size > 2G -> movie (even fullHD eps aren't that big yet) + if size > 2048: + log.debug('Likely a movie due to its big size (%dMB): %s' % (size, filename)) + movie_info.update({ 'type': 'movie' }, confidence = 0.8) + return movie_info + + # if size == 1CD or 2CDs -> movie + if within(size, [690, 710]) or within(size, [1380, 1420]): + log.debug('Likely a movie due to its size close to a CD size (%dMB): %s' % (size, filename)) + movie_info.update({ 'type': 'movie' }, confidence = 0.8) + return movie_info + + + # 3- if all else fails, assume it's a movie + log.debug('Couldn\'t make an informed guess... Assuming file is a movie: %s' % filename) + movie_info.update({ 'type': 'movie' }, confidence = 0.5) + return movie_info diff --git a/libs/guessit/date.py b/libs/guessit/date.py new file mode 100644 index 00000000..9f79daec --- /dev/null +++ b/libs/guessit/date.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +import datetime +import re + +def search_year(string): + """Looks for year patterns, and if found return the year and group span. + Assumes there are sentinels at the beginning and end of the string that + always allow matching a non-digit delimiting the date. + + Note this only looks for valid production years, that is between 1920 + and now + 5 years, so for instance 2000 would be returned as a valid + year but 1492 would not. + + >>> search_year('in the year 2000...') + (2000, (12, 16)) + + >>> search_year('they arrived in 1492.') + (None, None) + """ + match = re.search(r'[^0-9]([0-9]{4})[^0-9]', string) + if match: + year = int(match.group(1)) + if 1920 < year < datetime.date.today().year + 5: + return (year, match.span(1)) + + return (None, None) + + +def search_date(string): + """Looks for date patterns, and if found return the date and group span. + Assumes there are sentinels at the beginning and end of the string that + always allow matching a non-digit delimiting the date. + + >>> search_date('This happened on 2002-04-22.') + (datetime.date(2002, 4, 22), (17, 27)) + + >>> search_date('And this on 17-06-1998.') + (datetime.date(1998, 6, 17), (12, 22)) + + >>> search_date('no date in here') + (None, None) + """ + + dsep = r'[-/ \.]' + + date_rexps = [ # 20010823 + r'[^0-9]' + + r'(?P[0-9]{4})' + + r'(?P[0-9]{2})' + + r'(?P[0-9]{2})' + + r'[^0-9]', + + # 2001-08-23 + r'[^0-9]' + + r'(?P[0-9]{4})' + dsep + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + + r'[^0-9]', + + # 23-08-2001 + r'[^0-9]' + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{4})' + + r'[^0-9]', + + # 23-08-01 + r'[^0-9]' + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + + r'[^0-9]', + ] + + for drexp in date_rexps: + match = re.search(drexp, string) + if match: + d = match.groupdict() + year, month, day = int(d['year']), int(d['month']), int(d['day']) + # years specified as 2 digits should be adjusted here + if year < 100: + if year > (datetime.date.today().year % 100)+ 5: + year = 1900 + year + else: + year = 2000 + year + + date = None + try: + date = datetime.date(year, month, day) + except ValueError: + try: + date = datetime.date(year, day, month) + except ValueError: + pass + + if date is None: + continue + + # check date plausibility + if not 1900 < date.year < datetime.date.today().year + 5: + continue + + # looks like we have a valid date + # note: span is [+1,-1] because we don't want to include the non-digit char + start, end = match.span() + return (date, (start+1, end-1)) + + return None, None diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py new file mode 100644 index 00000000..7c07af71 --- /dev/null +++ b/libs/guessit/fileutils.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +import ntpath +import os.path + + +def split_path(path): + r"""Splits the given path into the list of folders and the filename (or the + last folder if you gave it a folder path. + + If the given path was an absolute path, the first element will always be: + - the '/' root folder on Unix systems + - the drive letter on Windows systems (eg: r'C:\') + + >>> split_path('/usr/bin/smewt') + ['/', 'usr', 'bin', 'smewt'] + + >>> split_path('relative_path/to/my_folder/') + ['relative_path', 'to', 'my_folder'] + + >>> split_path(r'C:\Program Files\Smewt\smewt.exe') + ['C:\\', 'Program Files', 'Smewt', 'smewt.exe'] + + >>> split_path(r'Documents and Settings\User\config\\') + ['Documents and Settings', 'User', 'config'] + + """ + result = [] + while True: + head, tail = ntpath.split(path) + + # on Unix systems, the root folder is '/' + if head == '/' and tail == '': + return [ '/' ] + result + + # on Windows, the root folder is a drive letter (eg: 'C:\') + if len(head) == 3 and head[1:] == ':\\' and tail == '': + return [ head ] + result + + if head == '' and tail == '': + return result + + # we just split a directory ending with '/', so tail is empty + if not tail: + path = head + continue + + result = [ tail ] + result + path = head + + +def split_path_components(filename): + """Returns the filename split into [ dir*, basename, ext ].""" + result = split_path(filename) + basename = result.pop(-1) + return result + list(os.path.splitext(basename)) + + +def file_in_same_dir(ref_file, desired_file): + """Return the path for a file in the same dir as a given reference file. + + >>> file_in_same_dir('~/smewt/smewt.db', 'smewt.settings') + '~/smewt/smewt.settings' + + """ + return os.path.join(*(split_path(ref_file)[:-1] + [ desired_file ])) diff --git a/libs/guessit/guess.py b/libs/guessit/guess.py new file mode 100644 index 00000000..92ec4bea --- /dev/null +++ b/libs/guessit/guess.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +import json +import datetime +import logging + +log = logging.getLogger("guessit.guess") + + +class Guess(dict): + """A Guess is a dictionary which has an associated confidence for each of its values. + + As it is a subclass of dict, you can use it everywhere you expect a simple dict""" + def __init__(self, *args, **kwargs): + try: + confidence = kwargs.pop('confidence') + except KeyError: + confidence = 0 + + dict.__init__(self, *args, **kwargs) + + self._confidence = {} + for prop in self: + self._confidence[prop] = confidence + + def to_utf8_dict(self): + from guessit.language import Language + data = dict(self) + for prop, value in data.items(): + if isinstance(value, datetime.date): + data[prop] = value.isoformat() + elif isinstance(value, Language): + data[prop] = str(value) + elif isinstance(value, unicode): + data[prop] = value.encode('utf-8') + elif isinstance(value, list): + data[prop] = [ str(x) for x in value ] + + return data + + def nice_string(self): + data = self.to_utf8_dict() + + parts = json.dumps(data, indent = 4).split('\n') + for i, p in enumerate(parts): + if p[:5] != ' "': + continue + + prop = p.split('"')[1] + parts[i] = (' [%.2f] "' % (self._confidence.get(prop) or -1)) + p[5:] + + return '\n'.join(parts) + + def __str__(self): + return str(self.to_utf8_dict()) + + def confidence(self, prop): + return self._confidence[prop] + + def set(self, prop, value, confidence = None): + self[prop] = value + if confidence is not None: + self._confidence[prop] = confidence + + def set_confidence(self, prop, value): + self._confidence[prop] = value + + def update(self, other, confidence = None): + dict.update(self, other) + if isinstance(other, Guess): + for prop in other: + self._confidence[prop] = other.confidence(prop) + + if confidence is not None: + for prop in other: + self._confidence[prop] = confidence + + def update_highest_confidence(self, other): + """Update this guess with the values from the given one. In case there is + property present in both, only the one with the highest one is kept.""" + if not isinstance(other, Guess): + raise ValueError, 'Can only call this function on Guess instances' + + for prop in other: + if prop in self and self._confidence[prop] >= other._confidence[prop]: + continue + self[prop] = other[prop] + self._confidence[prop] = other._confidence[prop] + + + + +def choose_int(g1, g2): + """Function used by merge_similar_guesses to choose between 2 possible properties + when they are integers.""" + v1, c1 = g1 # value, confidence + v2, c2 = g2 + if (v1 == v2): + return (v1, 1 - (1-c1)*(1-c2)) + else: + if c1 > c2: + return (v1, c1 - c2) + else: + return (v2, c2 - c1) + +def choose_string(g1, g2): + """Function used by merge_similar_guesses to choose between 2 possible properties + when they are strings. + + If the 2 strings are similar, or one is contained in the other, the latter is returned + with an increased confidence. + + If the 2 strings are dissimilar, the one with the higher confidence is returned, with + a weaker confidence. + + Note that here, 'similar' means that 2 strings are either equal, or that they + differ very little, such as one string being the other one with the 'the' word + prepended to it. + + >>> choose_string(('Hello', 0.75), ('World', 0.5)) + ('Hello', 0.25) + + >>> choose_string(('Hello', 0.5), ('hello', 0.5)) + ('Hello', 0.75) + + >>> choose_string(('Hello', 0.4), ('Hello World', 0.4)) + ('Hello', 0.64000000000000001) + + >>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5)) + ('The Simpsons', 0.75) + + """ + v1, c1 = g1 # value, confidence + v2, c2 = g2 + + if not v1: + return g2 + elif not v2: + return g1 + + v1, v2 = v1.strip(), v2.strip() + v1l, v2l = v1.lower(), v2.lower() + + combined_prob = 1 - (1-c1)*(1-c2) + + if v1l == v2l: + return (v1, combined_prob) + + # check for common patterns + elif v1l == 'the ' + v2l: + return (v1, combined_prob) + elif v2l == 'the ' + v1l: + return (v2, combined_prob) + + # if one string is contained in the other, return the shortest one + elif v2l in v1l: + return (v2, combined_prob) + elif v1l in v2l: + return (v1, combined_prob) + + # in case of conflict, return the one with highest priority + else: + if c1 > c2: + return (v1, c1 - c2) + else: + return (v2, c2 - c1) + + +def _merge_similar_guesses_nocheck(guesses, prop, choose): + """Take a list of guesses and merge those which have the same properties, + increasing or decreasing the confidence depending on whether their values + are similar. + + This function assumes there are at least 2 valid guesses.""" + + similar = [ guess for guess in guesses if prop in guess ] + + g1, g2 = similar[0], similar[1] + + other_props = set(g1) & set(g2) - set([prop]) + if other_props: + for prop in other_props: + if g1[prop] != g2[prop]: + log.warning('both guesses to be merged have more than one different property in common, bailing out...') + return + + # merge all props of s2 into s1, updating the confidence for the considered property + v1, v2 = g1[prop], g2[prop] + c1, c2 = g1.confidence(prop), g2.confidence(prop) + + new_value, new_confidence = choose((v1, c1), (v2, c2)) + if new_confidence >= c1: + log.debug("Updating matching property '%s' with confidence %.2f" % (prop, new_confidence)) + else: + log.debug("Updating non-matching property '%s' with confidence %.2f" % (prop, new_confidence)) + + g2[prop] = new_value + g2.set_confidence(prop, new_confidence) + + g1.update(g2) + guesses.remove(g2) + +def merge_similar_guesses(guesses, prop, choose): + """Take a list of guesses and merge those which have the same properties, + increasing or decreasing the confidence depending on whether their values + are similar.""" + + similar = [ guess for guess in guesses if prop in guess ] + if len(similar) < 2: + # nothing to merge + return + + if len(similar) == 2: + _merge_similar_guesses_nocheck(guesses, prop, choose) + + if len(similar) > 2: + log.debug('complex merge, trying our best...') + _merge_similar_guesses_nocheck(guesses, prop, choose) + merge_similar_guesses(guesses, prop, choose) + return + + +def merge_append_guesses(guesses, prop): + """Take a list of guesses and merge those which have the same properties by + appending them in a list. + + DEPRECATED, remove with old guessers + + """ + + + similar = [ guess for guess in guesses if prop in guess ] + if not similar: + return + + merged = similar[0] + merged[prop] = [ merged[prop] ] + # TODO: what to do with global confidence? mean of them all? + + for m in similar[1:]: + for prop2 in m: + if prop == prop2: + merged[prop].append(m[prop]) + else: + if prop2 in m: + log.warning('overwriting property "%s" with value ' % (prop2, m[prop2])) + merged[prop2] = m[prop2] + # TODO: confidence also + + guesses.remove(m) + + +def merge_all(guesses, append = []): + """Merges all the guesses in a single result, removes very unlikely values, and returns it. + You can specify a list of properties that should be appended into a list instead of being + merged. + + >>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6), + ... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ]) + {'season': 2, 'episodeNumber': 13} + + >>> merge_all([ Guess({ 'episodeNumber': 27 }, confidence = 0.02), + ... Guess({ 'season': 1 }, confidence = 0.2) ]) + {'season': 1} + + """ + if not guesses: + return Guess() + + result = guesses[0] + + for g in guesses[1:]: + # first append our appendable properties + for prop in append: + if prop in g: + result.set(prop, result.get(prop, []) + [ g[prop] ], + # TODO: what to do with confidence here? maybe an arithmetic mean... + confidence = g.confidence(prop)) + + del g[prop] + + # then merge the remaining ones + if set(result) & set(g): + log.warning('duplicate properties %s in merged result...' % (set(result) & set(g))) + + result.update_highest_confidence(g) + + # delete very unlikely values + for p in result.keys(): + if result.confidence(p) < 0.05: + del result[p] + + # make sure our appendable properties contain unique values + for prop in append: + if prop in result: + result[prop] = list(set(result[prop])) + + return result + diff --git a/libs/guessit/hash_ed2k.py b/libs/guessit/hash_ed2k.py new file mode 100644 index 00000000..eb1a4ea6 --- /dev/null +++ b/libs/guessit/hash_ed2k.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +import hashlib, os.path + +def hash_file(filename): + """Returns the ed2k hash of a given file. + + >>> hash_file('test/dummy.srt') + 'ed2k://|file|dummy.srt|44|1CA0B9DED3473B926AA93A0A546138BB|/' + """ + return 'ed2k://|file|%s|%d|%s|/' % (os.path.basename(filename), + os.path.getsize(filename), + hash_filehash(filename).upper()) + +def hash_filehash(filename): + """Returns the ed2k hash of a given file. + + This function is taken from: + http://www.radicand.org/blog/orz/2010/2/21/edonkey2000-hash-in-python/ + """ + md4 = hashlib.new('md4').copy + + def gen(f): + while True: + x = f.read(9728000) + if x: yield x + else: return + + def md4_hash(data): + m = md4() + m.update(data) + return m + + with open(filename, 'rb') as f: + a = gen(f) + hashes = [md4_hash(data).digest() for data in a] + if len(hashes) == 1: + return hashes[0].encode("hex") + else: return md4_hash(reduce(lambda a,d: a + d, hashes, "")).hexd diff --git a/libs/guessit/hash_mpc.py b/libs/guessit/hash_mpc.py new file mode 100644 index 00000000..2023f3d5 --- /dev/null +++ b/libs/guessit/hash_mpc.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +import struct, os + +def hash_file(filename): + """This function is taken from: + http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes + and is licensed under the GPL.""" + + longlongformat = 'q' # long long + bytesize = struct.calcsize(longlongformat) + + f = open(filename, "rb") + + filesize = os.path.getsize(filename) + hash = filesize + + if filesize < 65536 * 2: + raise Exception, "SizeError: size is %d, should be > 132K..." % filesize + + for x in range(65536/bytesize): + buffer = f.read(bytesize) + (l_value,)= struct.unpack(longlongformat, buffer) + hash += l_value + hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number + + + f.seek(max(0,filesize-65536),0) + for x in range(65536/bytesize): + buffer = f.read(bytesize) + (l_value,)= struct.unpack(longlongformat, buffer) + hash += l_value + hash = hash & 0xFFFFFFFFFFFFFFFF + + f.close() + returnedhash = "%016x" % hash + return returnedhash diff --git a/libs/guessit/language.py b/libs/guessit/language.py new file mode 100644 index 00000000..c251f62e --- /dev/null +++ b/libs/guessit/language.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import fileutils +import os.path +import re +import logging + +log = logging.getLogger('guessit.language') + + + +# downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt +# +# Description of the fields: +# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given), +# an alpha-2 code (when given), an English name, and a French name of a language +# are all separated by pipe (|) characters." +language_matrix = [ l.strip().decode('utf-8').split('|') for l in open(fileutils.file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt')) ] + +lng3 = frozenset(filter(bool, (l[0] for l in language_matrix))) +lng3term = frozenset(filter(bool, (l[1] for l in language_matrix))) +lng2 = frozenset(filter(bool, (l[2] for l in language_matrix))) +lng_en_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[3].lower().split('; ')))) +lng_fr_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[4].lower().split('; ')))) +lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name + +lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1]) +lng3term_to_lng3 = dict((l[1], l[0]) for l in language_matrix if l[1]) + +lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2]) +lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2]) + +# we only return the first given english name, hoping it is the most used one +lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) for l in language_matrix if l[3]) +lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) for l in language_matrix if l[3] for en_name in l[3].split('; ')) + +# we only return the first given french name, hoping it is the most used one +lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) for l in language_matrix if l[4]) +lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) for l in language_matrix if l[4] for fr_name in l[4].split('; ')) + + +def is_language(language): + return language.lower() in lng_all_names + +class Language(object): + """This class represents a human language. + + You can initialize it with pretty much everything, as it knows conversion from + ISO-639 2-letter and 3-letter codes, English and French names. + + >>> Language('fr') + Language(French) + + >>> Language('eng').french_name() + u'anglais' + """ + def __init__(self, language): + lang = None + language = language.lower() + if len(language) == 2: + lang = lng2_to_lng3.get(language) + elif len(language) == 3: + lang = language if language in lng3 else lng3term_to_lng3.get(language) + else: + lang = lng_en_name_to_lng3.get(language) or lng_fr_name_to_lng3.get(language) + + if lang is None: + raise ValueError, 'The given string "%s" could not be identified as a language' % language + + self.lang = lang + + def lng2(self): + return lng3_to_lng2[self.lang] + + def lng3(self): + return self.lang + + def lng3term(self): + return lng3_to_lng3term[self.lang] + + def english_name(self): + return lng3_to_lng_en_name[self.lang] + + def french_name(self): + return lng3_to_lng_fr_name[self.lang] + + + def __hash__(self): + return hash(self.lang) + + def __eq__(self, other): + if isinstance(other, Language): + return self.lang == other.lang + + if isinstance(other, basestring): + try: + return self == Language(other) + except ValueError: + return False + + return False + + def __ne__(self, other): + return not self == other + + def __unicode__(self): + return lng3_to_lng_en_name[self.lang] + + def __str__(self): + return unicode(self).encode('utf-8') + + def __repr__(self): + return 'Language(%s)' % self + + + +def search_language(string, lang_filter = None): + """Looks for language patterns, and if found return the language object, + its group span and an associated confidence. + + you can specify a list of allowed languages using the lang_filter argument, + as in lang_filter = [ 'fr', 'eng', 'spanish' ] + + Assumes there are sentinels at the beginning and end of the string that + always allow matching a non-letter delimiting the language. + + >>> search_language('movie [en].avi') + (Language(English), (7, 9), 0.80000000000000004) + + >>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es']) + (None, None, None) + """ + + # list of common words which could be interpreted as languages, but which + # are far too common to be able to say they represent a language in the + # middle of a string (where they most likely carry their commmon meaning) + lng_common_words = frozenset([ # english words + 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to', + 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan', + 'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as', + 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr', + # french words + 'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que', + 'mal', 'est', 'vol', 'or', 'mon', 'se', + # spanish words + 'la', 'el', 'del', 'por', 'mar', + # other + 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san' + ]) + sep = r'[](){} \._-+' + + if lang_filter: + lang_filter = set(Language(l) for l in lang_filter) + + slow = string.lower() + confidence = 1.0 # for all of them + for lang in lng_all_names: + + if lang in lng_common_words: + continue + + pos = slow.find(lang) + + if pos != -1: + end = pos + len(lang) + # make sure our word is always surrounded by separators + if slow[pos-1] not in sep or slow[end] not in sep: + continue + + language = Language(slow[pos:end]) + if lang_filter and language not in lang_filter: + continue + + # only allow those languages that have a 2-letter code, those who + # don't are too esoteric and probably false matches + if language.lang not in lng3_to_lng2: + continue + + # confidence depends on lng2, lng3, english name, ... + if len(lang) == 2: + confidence = 0.8 + elif len(lang) == 3: + confidence = 0.9 + else: + # Note: we could either be really confident that we found a language + # or assume that full language names are too common words + confidence = 0.3 # going with the low-confidence route here + + return language, (pos, end), confidence + + return None, None, None diff --git a/libs/guessit/matcher.py b/libs/guessit/matcher.py new file mode 100644 index 00000000..1fe3dcf9 --- /dev/null +++ b/libs/guessit/matcher.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import fileutils, textutils +from guessit.guess import Guess, merge_similar_guesses, merge_all, choose_int, choose_string +from guessit.date import search_date, search_year +from guessit.language import search_language +from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, properties, canonical_form +from guessit.matchtree import get_group, find_group, leftover_valid_groups, tree_to_string +from guessit.textutils import find_first_level_groups, split_on_groups, blank_region, clean_string, to_utf8 +from guessit.fileutils import split_path_components +import datetime +import os.path +import re +import copy +import logging + +log = logging.getLogger("guessit.matcher") + + + +def split_explicit_groups(string): + """return the string split into explicit groups, that is, those either + between parenthese, square brackets or curly braces, and those separated + by a dash.""" + result = find_first_level_groups(string, '()') + result = reduce(lambda l, x: l + find_first_level_groups(x, '[]'), result, []) + result = reduce(lambda l, x: l + find_first_level_groups(x, '{}'), result, []) + # do not do this at this moment, it is not strong enough and can break other + # patterns, such as dates, etc... + #result = reduce(lambda l, x: l + x.split('-'), result, []) + + return result + + +def format_guess(guess): + """Format all the found values to their natural type. + For instance, a year would be stored as an int value, etc... + + Note that this modifies the dictionary given as input. + """ + for prop, value in guess.items(): + if prop in ('season', 'episodeNumber', 'year', 'cdNumber', 'cdNumberTotal'): + guess[prop] = int(guess[prop]) + elif isinstance(value, basestring): + if prop in ('edition',): + value = clean_string(value) + guess[prop] = canonical_form(value) + + return guess + + +def guess_groups(string, result, filetype): + # add sentinels so we can match a separator char at either end of + # our groups, even when they are at the beginning or end of the string + # we will adjust the span accordingly later + # + # filetype can either be movie, moviesubtitle, episode, episodesubtitle + current = ' ' + string + ' ' + + regions = [] # list of (start, end) of matched regions + + def guessed(match_dict, confidence): + guess = format_guess(Guess(match_dict, confidence = confidence)) + result.append(guess) + log.debug('Found with confidence %.2f: %s' % (confidence, guess)) + return guess + + def update_found(string, guess, span, span_adjust = (0,0)): + span = (span[0] + span_adjust[0], + span[1] + span_adjust[1]) + regions.append((span, guess)) + return blank_region(string, span) + + # try to find dates first, as they are very specific + date, span = search_date(current) + if date: + guess = guessed({ 'date': date }, confidence = 1.0) + current = update_found(current, guess, span) + + # for non episodes only, look for year information + if filetype not in ('episode', 'episodesubtitle'): + year, span = search_year(current) + if year: + guess = guessed({ 'year': year }, confidence = 1.0) + current = update_found(current, guess, span) + + # specific regexps (ie: cd number, season X episode, ...) + for rexp, confidence, span_adjust in video_rexps: + match = re.search(rexp, current, re.IGNORECASE) + if match: + metadata = match.groupdict() + # is this the better place to put it? (maybe, as it is at least the soonest that we can catch it) + if 'cdNumberTotal' in metadata and metadata['cdNumberTotal'] is None: + del metadata['cdNumberTotal'] + + guess = guessed(metadata, confidence = confidence) + current = update_found(current, guess, match.span(), span_adjust) + + if filetype in ('episode', 'episodesubtitle'): + for rexp, confidence, span_adjust in episode_rexps: + match = re.search(rexp, current, re.IGNORECASE) + if match: + metadata = match.groupdict() + guess = guessed(metadata, confidence = confidence) + current = update_found(current, guess, match.span(), span_adjust) + + + # Now websites, but as exact string instead of regexps + clow = current.lower() + for site in websites: + pos = clow.find(site.lower()) + if pos != -1: + guess = guessed({ 'website': site }, confidence = confidence) + current = update_found(current, guess, (pos, pos+len(site))) + clow = current.lower() + + + # release groups have certain constraints, cannot be included in the previous general regexps + group_names = [ r'\.(Xvid)-(?P.*?)[ \.]', + r'\.(DivX)-(?P.*?)[\. ]', + r'\.(DVDivX)-(?P.*?)[\. ]', + ] + for rexp in group_names: + match = re.search(rexp, current, re.IGNORECASE) + if match: + metadata = match.groupdict() + metadata.update({ 'videoCodec': match.group(1) }) + guess = guessed(metadata, confidence = 0.8) + current = update_found(current, guess, match.span(), span_adjust = (1, -1)) + + + # common well-defined words and regexps + clow = current.lower() + confidence = 1.0 # for all of them + for prop, values in properties.items(): + for value in values: + pos = clow.find(value.lower()) + if pos != -1: + end = pos + len(value) + # make sure our word is always surrounded by separators + if clow[pos-1] not in sep or clow[end] not in sep: + # note: sep is a regexp, but in this case using it as + # a sequence achieves the same goal + continue + + guess = guessed({ prop: value }, confidence = confidence) + current = update_found(current, guess, (pos, end)) + clow = current.lower() + + # weak guesses for episode number, only run it if we don't have an estimate already + if filetype in ('episode', 'episodesubtitle'): + if not any('episodeNumber' in match for match in result): + for rexp, _, span_adjust in weak_episode_rexps: + match = re.search(rexp, current, re.IGNORECASE) + if match: + metadata = match.groupdict() + epnum = int(metadata['episodeNumber']) + if epnum > 100: + guess = guessed({ 'season': epnum // 100, + 'episodeNumber': epnum % 100 }, confidence = 0.6) + else: + guess = guessed(metadata, confidence = 0.3) + current = update_found(current, guess, match.span(), span_adjust) + + # try to find languages now + language, span, confidence = search_language(current) + while language: + # is it a subtitle language? + if 'sub' in clean_string(current[:span[0]]).lower().split(' '): + guess = guessed({ 'subtitleLanguage': language }, confidence = confidence) + else: + guess = guessed({ 'language': language }, confidence = confidence) + current = update_found(current, guess, span) + + language, span, confidence = search_language(current) + + + # remove our sentinels now and ajust spans accordingly + assert(current[0] == ' ' and current[-1] == ' ') + current = current[1:-1] + regions = [ ((start-1, end-1), guess) for (start, end), guess in regions ] + + # split into '-' separated subgroups (with required separator chars + # around the dash) + didx = current.find('-') + while didx > 0: + regions.append(((didx, didx), None)) + didx = current.find('-', didx+1) + + # cut our final groups, and rematch the guesses to the group that created + # id, None if it is a leftover group + region_spans = [ span for span, guess in regions ] + string_groups = split_on_groups(string, region_spans) + remaining_groups = split_on_groups(current, region_spans) + guesses = [] + + pos = 0 + for group in string_groups: + found = False + for span, guess in regions: + if span[0] == pos: + guesses.append(guess) + found = True + if not found: + guesses.append(None) + + pos += len(group) + + return zip(string_groups, + remaining_groups, + guesses) + + +def match_from_epnum_position(match_tree, epnum_pos, guessed, update_found): + """guessed is a callback function to call with the guessed group + update_found is a callback to update the match group and returns leftover groups.""" + pidx, eidx, gidx = epnum_pos + + # a few helper functions to be able to filter using high-level semantics + def same_pgroup_before(group): + _, (ppidx, eeidx, ggidx) = group + return ppidx == pidx and (eeidx, ggidx) < (eidx, gidx) + + def same_pgroup_after(group): + _, (ppidx, eeidx, ggidx) = group + return ppidx == pidx and (eeidx, ggidx) > (eidx, gidx) + + def same_egroup_before(group): + _, (ppidx, eeidx, ggidx) = group + return ppidx == pidx and eeidx == eidx and ggidx < gidx + + def same_egroup_after(group): + _, (ppidx, eeidx, ggidx) = group + return ppidx == pidx and eeidx == eidx and ggidx > gidx + + leftover = leftover_valid_groups(match_tree) + + # if we have at least 1 valid group before the episodeNumber, then it's probably + # the series name + series_candidates = filter(same_pgroup_before, leftover) + if len(series_candidates) >= 1: + guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.7) + leftover = update_found(leftover, series_candidates[0][1], guess) + + # only 1 group after (in the same path group) and it's probably the episode title + title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, + filter(same_pgroup_after, leftover)) + if len(title_candidates) == 1: + guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.5) + leftover = update_found(leftover, title_candidates[0][1], guess) + else: + # try in the same explicit group, with lower confidence + title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, + filter(same_egroup_after, leftover)) + if len(title_candidates) == 1: + guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.4) + leftover = update_found(leftover, title_candidates[0][1], guess) + + # epnumber is the first group and there are only 2 after it in same path group + # -> season title - episode title + already_has_title = (find_group(match_tree, 'title') != []) + + title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, + filter(same_pgroup_after, leftover)) + if (not already_has_title and # no title + not filter(same_pgroup_before, leftover) and # no groups before + len(title_candidates) == 2): # only 2 groups after + + guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4) + leftover = update_found(leftover, title_candidates[0][1], guess) + guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4) + leftover = update_found(leftover, title_candidates[1][1], guess) + + + # if we only have 1 remaining valid group in the pathpart before the filename, + # then it's likely that it is the series name + series_candidates = [ group for group in leftover if group[1][0] == pidx-1 ] + if len(series_candidates) == 1: + guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.5) + leftover = update_found(leftover, series_candidates[0][1], guess) + + return match_tree + + + +class IterativeMatcher(object): + def __init__(self, filename, filetype = 'autodetect'): + """An iterative matcher tries to match different patterns that appear + in the filename. + + The 'filetype' argument indicates which type of file you want to match. + If it is 'autodetect', the matcher will try to see whether it can guess + that the file corresponds to an episode, or otherwise will assume it is + a movie. + + The recognized 'filetype' values are: + [ autodetect, subtitle, movie, moviesubtitle, episode, episodesubtitle ] + + + The IterativeMatcher works mainly in 2 steps: + + First, it splits the filename into a match_tree, which is a tree of groups + which have a semantic meaning, such as episode number, movie title, + etc... + + The match_tree created looks like the following: + + 0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111 + 0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000 + 0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000 + __________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___ + xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc + [XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv + + The first 3 lines indicates the group index in which a char in the + filename is located. So for instance, x264 is the group (0, 4, 1), and + it corresponds to a video codec, denoted by the letter'v' in the 4th line. + (for more info, see guess.matchtree.tree_to_string) + + + Second, it tries to merge all this information into a single object + containing all the found properties, and does some (basic) conflict + resolution when they arise. + """ + + if filetype not in ('autodetect', 'subtitle', 'movie', 'moviesubtitle', + 'episode', 'episodesubtitle'): + raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')" + if not isinstance(filename, unicode): + log.debug('WARNING: given filename to matcher is not unicode...') + + match_tree = [] + result = [] # list of found metadata + + def guessed(match_dict, confidence): + guess = format_guess(Guess(match_dict, confidence = confidence)) + result.append(guess) + log.debug('Found with confidence %.2f: %s' % (confidence, guess)) + return guess + + def update_found(leftover, group_pos, guess): + pidx, eidx, gidx = group_pos + group = match_tree[pidx][eidx][gidx] + match_tree[pidx][eidx][gidx] = (group[0], + deleted * len(group[0]), + guess) + return [ g for g in leftover if g[1] != group_pos ] + + + # 1- first split our path into dirs + basename + ext + match_tree = split_path_components(filename) + + fileext = match_tree.pop(-1)[1:].lower() + if fileext in subtitle_exts: + if 'movie' in filetype: + filetype = 'moviesubtitle' + elif 'episode' in filetype: + filetype = 'episodesubtitle' + else: + filetype = 'subtitle' + extguess = guessed({ 'container': fileext }, confidence = 1.0) + elif fileext in video_exts: + extguess = guessed({ 'container': fileext }, confidence = 1.0) + else: + extguess = guessed({ 'extension': fileext}, confidence = 1.0) + + # TODO: depending on the extension, we could already grab some info and maybe specialized + # guessers, eg: a lang parser for idx files, an automatic detection of the language + # for srt files, a video metadata extractor for avi, mkv, ... + + # if we are on autodetect, try to do it now so we can tell the + # guess_groups function what type of info it should be looking for + if filetype in ('autodetect', 'subtitle'): + for rexp, confidence, span_adjust in episode_rexps: + match = re.search(rexp, filename, re.IGNORECASE) + if match: + if filetype == 'autodetect': + filetype = 'episode' + elif filetype == 'subtitle': + filetype = 'episodesubtitle' + break + + # if no episode info found, assume it's a movie + if filetype == 'autodetect': + filetype = 'movie' + elif filetype == 'subtitle': + filetype = 'moviesubtitle' + + guessed({ 'type': filetype }, confidence = 1.0) + + + # 2- split each of those into explicit groups, if any + # note: be careful, as this might split some regexps with more confidence such as + # Alfleni-Team, or [XCT] or split a date such as (14-01-2008) + match_tree = [ split_explicit_groups(part) for part in match_tree ] + + + # 3- try to match information in decreasing order of confidence and + # blank the matching group in the string if we found something + for pathpart in match_tree: + for gidx, explicit_group in enumerate(pathpart): + pathpart[gidx] = guess_groups(explicit_group, result, filetype = filetype) + + # 4- try to identify the remaining unknown groups by looking at their position + # relative to other known elements + + if filetype in ('episode', 'episodesubtitle'): + eps = find_group(match_tree, 'episodeNumber') + if eps: + match_tree = match_from_epnum_position(match_tree, eps[0], guessed, update_found) + + leftover = leftover_valid_groups(match_tree) + + if not eps: + # if we don't have the episode number, but at least 2 groups in the + # last path group, then it's probably series - eptitle + title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, + filter(lambda g: g[1][0] == len(match_tree)-1, + leftover_valid_groups(match_tree))) + if len(title_candidates) >= 2: + guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4) + leftover = update_found(leftover, title_candidates[0][1], guess) + guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4) + leftover = update_found(leftover, title_candidates[1][1], guess) + + + # if there's a path group that only contains the season info, then the previous one + # is most likely the series title (ie: .../series/season X/...) + eps = [ gpos for gpos in find_group(match_tree, 'season') + if 'episodeNumber' not in get_group(match_tree, gpos)[2] ] + + if eps: + pidx, eidx, gidx = eps[0] + previous = [ group for group in leftover if group[1][0] == pidx - 1 ] + if len(previous) == 1: + guess = guessed({ 'series': previous[0][0] }, confidence = 0.5) + leftover = update_found(leftover, previous[0][1], guess) + + + elif filetype in ('movie', 'moviesubtitle'): + leftover_all = leftover_valid_groups(match_tree) + + # specific cases: + # - movies/tttttt (yyyy)/tttttt.ccc + try: + if match_tree[-3][0][0][0].lower() == 'movies': + # Note:too generic, might solve all the unittests as they all contain 'movies' + # in their path + # + #if len(match_tree[-2][0]) == 1: + # title = match_tree[-2][0][0] + # guess = guessed({ 'title': clean_string(title[0]) }, confidence = 0.7) + # update_found(leftover_all, title, guess) + + year_group = filter(lambda gpos: gpos[0] == len(match_tree)-2, + find_group(match_tree, 'year'))[0] + leftover = leftover_valid_groups(match_tree, + valid = lambda g: ((g[0] and g[0][0] not in sep) and + g[1][0] == len(match_tree) - 2)) + if len(match_tree[-2]) == 2 and year_group[1] == 1: + title = leftover[0] + guess = guessed({ 'title': clean_string(title[0]) }, + confidence = 0.8) + update_found(leftover_all, title[1], guess) + raise Exception # to exit the try catch now + + leftover = [ g for g in leftover_all if (g[1][0] == year_group[0] and + g[1][1] < year_group[1] and + g[1][2] < year_group[2]) ] + leftover = sorted(leftover, key = lambda x:x[1]) + title = leftover[0] + guess = guessed({ 'title': title[0] }, confidence = 0.8) + leftover = update_found(leftover, title[1], guess) + except: + pass + + # if we have either format or videoCodec in the folder containing the file + # or one of its parents, then we should probably look for the title in + # there rather than in the basename + props = filter(lambda g: g[0] <= len(match_tree) - 2, + find_group(match_tree, 'videoCodec') + + find_group(match_tree, 'format') + + find_group(match_tree, 'language')) + leftover = None + if props and all(g[0] == props[0][0] for g in props): + leftover = [ g for g in leftover_all if g[1][0] == props[0][0] ] + + if props and leftover: + guess = guessed({ 'title': leftover[0][0] }, confidence = 0.7) + leftover = update_found(leftover, leftover[0][1], guess) + + else: + # first leftover group in the last path part sounds like a good candidate for title, + # except if it's only one word and that the first group before has at least 3 words in it + # (case where the filename contains an 8 chars short name and the movie title is + # actually in the parent directory name) + leftover = [ g for g in leftover_all if g[1][0] == len(match_tree)-1 ] + if leftover: + title, (pidx, eidx, gidx) = leftover[0] + previous_pgroup_leftover = filter(lambda g: g[1][0] == pidx-1, leftover_all) + + if (title.count(' ') == 0 and + previous_pgroup_leftover and + previous_pgroup_leftover[0][0].count(' ') >= 2): + + guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6) + leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess) + + else: + guess = guessed({ 'title': title }, confidence = 0.6) + leftover = update_found(leftover, leftover[0][1], guess) + else: + # if there were no leftover groups in the last path part, look in the one before that + previous_pgroup_leftover = filter(lambda g: g[1][0] == len(match_tree)-2, leftover_all) + if previous_pgroup_leftover: + guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6) + leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess) + + + + + + + # 5- perform some post-processing steps + + # 5.1- try to promote language to subtitle language where it makes sense + for pidx, eidx, gidx in find_group(match_tree, 'language'): + string, remaining, guess = get_group(match_tree, (pidx, eidx, gidx)) + + def promote_subtitle(): + guess.set('subtitleLanguage', guess['language'], confidence = guess.confidence('language')) + del guess['language'] + + # - if we matched a language in a file with a sub extension and that the group + # is the last group of the filename, it is probably the language of the subtitle + # (eg: 'xxx.english.srt') + if (fileext in subtitle_exts and + pidx == len(match_tree) - 1 and + eidx == len(match_tree[pidx]) - 1): + promote_subtitle() + + # - if a language is in an explicit group just preceded by "st", it is a subtitle + # language (eg: '...st[fr-eng]...') + if eidx > 0: + previous = get_group(match_tree, (pidx, eidx-1, -1)) + if previous[0][-2:].lower() == 'st': + promote_subtitle() + + + + # re-append the extension now + match_tree.append([[(fileext, deleted*len(fileext), extguess)]]) + + self.parts = result + self.match_tree = match_tree + + if filename.startswith('/'): + filename = ' ' + filename + + log.debug('Found match tree:\n%s\n%s' % (to_utf8(tree_to_string(match_tree)), + to_utf8(filename))) + + + def matched(self): + # we need to make a copy here, as the merge functions work in place and + # calling them on the match tree would modify it + parts = copy.deepcopy(self.parts) + + # 1- start by doing some common preprocessing tasks + + # 1.1- ", the" at the end of a series title should be prepended to it + for part in parts: + if 'series' not in part: + continue + + series = part['series'] + lseries = series.lower() + + if lseries[-4:] == ',the': + part['series'] = 'The ' + series[:-4] + + if lseries[-5:] == ', the': + part['series'] = 'The ' + series[:-5] + + + # 2- try to merge similar information together and give it a higher confidence + for int_part in ('year', 'season', 'episodeNumber'): + merge_similar_guesses(parts, int_part, choose_int) + + for string_part in ('title', 'series', 'container', 'format', 'releaseGroup', 'website', + 'audioCodec', 'videoCodec', 'screenSize', 'episodeFormat'): + merge_similar_guesses(parts, string_part, choose_string) + + result = merge_all(parts, append = ['language', 'subtitleLanguage', 'other']) + + # 3- some last minute post-processing + if (result['type'] == 'episode' and + 'season' not in result and + result.get('episodeFormat', '') == 'Minisode'): + result['season'] = 0 + + log.debug('Final result: ' + result.nice_string()) + return result diff --git a/libs/guessit/matchtree.py b/libs/guessit/matchtree.py new file mode 100644 index 00000000..944cab03 --- /dev/null +++ b/libs/guessit/matchtree.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.patterns import deleted +from guessit.textutils import clean_string +import logging + +log = logging.getLogger("guessit.matchtree") + + + +def tree_to_string(tree): + """Return a string representation for the given tree. + + The lines convey the following information: + - line 1: path idx + - line 2: explicit group idx + - line 3: group index + - line 4: remaining info + - line 5: meaning conveyed + + Meaning is a letter indicating what type of info was matched by this group, + for instance 't' = title, 'f' = format, 'l' = language, etc... + + An example is the following: + + 0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111 + 0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000 + 0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000 + __________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___ + xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc + [XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv + + (note: the last line representing the filename is not pat of the tree representation) + """ + m_tree = [ '', # path level index + '', # explicit group index + '', # matched regexp and dash-separated + '', # groups leftover that couldn't be matched + '', # meaning conveyed: E = episodenumber, S = season, ... + ] + + def add_char(pidx, eidx, gidx, remaining, meaning = None): + nr = len(remaining) + def to_hex(x): + if isinstance(x, int): + return str(x) if x < 10 else chr(55+x) + return x + m_tree[0] = m_tree[0] + to_hex(pidx) * nr + m_tree[1] = m_tree[1] + to_hex(eidx) * nr + m_tree[2] = m_tree[2] + to_hex(gidx) * nr + m_tree[3] = m_tree[3] + remaining + m_tree[4] = m_tree[4] + str(meaning or ' ') * nr + + def meaning(result): + mmap = { 'episodeNumber': 'E', + 'season': 'S', + 'extension': 'e', + 'format': 'f', + 'language': 'l', + 'videoCodec': 'v', + 'audioCodec': 'a', + 'website': 'w', + 'container': 'c', + 'series': 'T', + 'title': 't', + 'date': 'd', + 'year': 'y', + 'releaseGroup': 'r', + 'screenSize': 's' + } + + if result is None: + return ' ' + + for prop, l in mmap.items(): + if prop in result: + return l + + return 'x' + + for pidx, pathpart in enumerate(tree): + for eidx, explicit_group in enumerate(pathpart): + for gidx, (group, remaining, result) in enumerate(explicit_group): + add_char(pidx, eidx, gidx, remaining, meaning(result)) + + # special conditions for the path separator + if pidx < len(tree) - 2: + add_char(' ', ' ', ' ', '/') + elif pidx == len(tree) - 2: + add_char(' ', ' ', ' ', '.') + + return '\n'.join(m_tree) + + + +def iterate_groups(match_tree): + """Iterate over all the groups in a match_tree and return them as pairs + of (group_pos, group) where: + - group_pos = (pidx, eidx, gidx) + - group = (string, remaining, guess) + """ + for pidx, pathpart in enumerate(match_tree): + for eidx, explicit_group in enumerate(pathpart): + for gidx, group in enumerate(explicit_group): + yield (pidx, eidx, gidx), group + + +def find_group(match_tree, prop): + """Find the list of groups that resulted in a guess that contains the + asked property.""" + result = [] + for gpos, (string, remaining, guess) in iterate_groups(match_tree): + if guess and prop in guess: + result.append(gpos) + return result + +def get_group(match_tree, gpos): + pidx, eidx, gidx = gpos + return match_tree[pidx][eidx][gidx] + + +def leftover_valid_groups(match_tree, valid = lambda s: len(s[0]) > 3): + """Return the list of valid string groups (eg: len(s) > 3) that could not be + matched to anything as a list of pairs (cleaned_str, group_pos).""" + leftover = [] + for gpos, (group, remaining, guess) in iterate_groups(match_tree): + if not guess: + clean_str = clean_string(remaining) + if valid((clean_str, gpos)): + leftover.append((clean_str, gpos)) + + return leftover + + + diff --git a/libs/guessit/patterns.py b/libs/guessit/patterns.py new file mode 100644 index 00000000..dec27fe6 --- /dev/null +++ b/libs/guessit/patterns.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2011 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + + +subtitle_exts = [ 'srt', 'idx', 'sub' ] + +video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'mov', 'ogg', 'ogm', 'ogv', 'wmv' ] + +# separator character regexp +sep = r'[][)(}{+ \._-]' # regexp art, hehe :D + +# character used to represent a deleted char (when matching groups) +deleted = '_' + +# format: [ (regexp, confidence, span_adjust) ] +episode_rexps = [ # ... Season 2 ... + (r'season (?P[0-9]+)', 1.0, (0, 0)), + (r'saison (?P[0-9]+)', 1.0, (0, 0)), + + # ... s02e13 ... + (r'[Ss](?P[0-9]{1,2}).{,3}[EeXx](?P[0-9]{1,2})[^0-9]', 1.0, (0, -1)), + + # ... 2x13 ... + (r'[^0-9](?P[0-9]{1,2})[x\.](?P[0-9]{2})[^0-9]', 0.8, (1, -1)), + + # ... s02 ... + (sep + r's(?P[0-9]{1,2})' + sep + '?', 0.6, (0, 0)), + + # v2 or v3 for some mangas which have multiples rips + (sep + r'(?P[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)), + ] + + +weak_episode_rexps = [ # ... 213 or 0106 ... + (sep + r'(?P[0-9]{1,4})' + sep, 0.3, (1, -1)), + ] + +non_episode_title = [ 'extras' ] + + +video_rexps = [ # cd number + (r'cd ?(?P[0-9])( ?of ?(?P[0-9]))?', 1.0, (0, 0)), + (r'(?P[1-9]) cds?', 0.9, (0, 0)), + + # special editions + (r'edition' + sep + r'(?Pcollector)', 1.0, (0, 0)), + (r'(?Pcollector)' + sep + 'edition', 1.0, (0, 0)), + (r'(?Pspecial)' + sep + 'edition', 1.0, (0, 0)), + (r'(?Pcriterion)' + sep + 'edition', 1.0, (0, 0)), + + # director's cut + (r"(?Pdirector'?s?" + sep + "cut)", 1.0, (0, 0)), + + # video size + (r'(?P[0-9]{3,4})x(?P[0-9]{3,4})', 0.9, (0, 0)), + + # website + (r'(?Pwww(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0)) + ] + +websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com', 'sharethefiles.com' ] + +properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'Blu-ray', 'BDRip', 'BRRip', + 'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'WEBRip', 'DVDSCR', 'Screener', 'VHS', + 'VIDEO_TS' ], + + 'container': [ 'avi', 'mkv', 'ogv', 'ogm', 'wmv', 'mp4', 'mov' ], + + 'screenSize': [ '720p', '720' ], + + 'videoCodec': [ 'XviD', 'DivX', 'x264', 'h264', 'Rv10' ], + + 'audioCodec': [ 'AC3', 'DTS', 'He-AAC', 'AAC-He', 'AAC' ], + + 'audioChannels': [ '5.1' ], + + 'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', '[XCT]', 'iNT', 'PUKKA', + 'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE', + 'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe', + 'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL' ], + + 'episodeFormat': [ 'Minisode', 'Minisodes' ], + + 'other': [ '5ch', 'PROPER', 'REPACK', 'LIMITED', 'DualAudio', 'iNTERNAL', 'Audiofixed', 'R5', + 'complete', 'classic', # not so sure about these ones, could appear in a title + 'ws', # widescreen + #'SE', # special edition + # TODO: director's cut + ], + } + + +property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ], + 'HD-DVD': [ 'HDDVD', 'HDDVDRip' ], + 'BluRay': [ 'BDRip', 'BRRip', 'Blu-ray' ], + 'Screener': [ 'DVDSCR' ], + 'DivX': [ 'DVDivX' ], + 'h264': [ 'x264' ], + '720p': [ '720' ], + 'AAC': [ 'He-AAC', 'AAC-He' ], + 'Special Edition': [ 'Special' ], + 'Collector Edition': [ 'Collector' ], + 'Criterion Edition': [ 'Criterion' ], + 'Minisode': [ 'Minisodes' ] + } + + +reverse_synonyms = {} +for canonical, synonyms in property_synonyms.items(): + for synonym in synonyms: + reverse_synonyms[synonym.lower()] = canonical + +def canonical_form(string): + return reverse_synonyms.get(string.lower(), string) diff --git a/libs/guessit/slogging.py b/libs/guessit/slogging.py new file mode 100644 index 00000000..308ef887 --- /dev/null +++ b/libs/guessit/slogging.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Smewt - A smart collection manager +# Copyright (c) 2011 Nicolas Wack +# +# Smewt is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Smewt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import logging +import sys + +GREEN_FONT = "\x1B[0;32m" +YELLOW_FONT = "\x1B[0;33m" +BLUE_FONT = "\x1B[0;34m" +RED_FONT = "\x1B[0;31m" +RESET_FONT = "\x1B[0m" + + +def setupLogging(colored = True): + """Sets up a nice colored logger as the main application logger (not only smewt itself).""" + + class SimpleFormatter(logging.Formatter): + def __init__(self): + self.fmt = '%(levelname)-8s %(module)s:%(funcName)s -- %(message)s' + logging.Formatter.__init__(self, self.fmt) + + class ColoredFormatter(logging.Formatter): + def __init__(self): + self.fmt = '%(levelname)-8s ' + BLUE_FONT + '%(module)s:%(funcName)s' + RESET_FONT + ' -- %(message)s' + logging.Formatter.__init__(self, self.fmt) + + def format(self, record): + result = logging.Formatter.format(self, record) + if record.levelno in (logging.DEBUG, logging.INFO): + return GREEN_FONT + result + elif record.levelno == logging.WARNING: + return YELLOW_FONT + result + else: + return RED_FONT + result + + + ch = logging.StreamHandler() + if colored and sys.platform != 'win32': + ch.setFormatter(ColoredFormatter()) + else: + ch.setFormatter(SimpleFormatter()) + logging.getLogger().addHandler(ch) + diff --git a/libs/guessit/textutils.py b/libs/guessit/textutils.py new file mode 100644 index 00000000..2ab88815 --- /dev/null +++ b/libs/guessit/textutils.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Smewt - A smart collection manager +# Copyright (c) 2008 Nicolas Wack +# +# Smewt is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Smewt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +from guessit.patterns import sep, deleted +import copy + +# string-related functions + +def strip_brackets(s): + if not s: + return s + if s[0] == '[' and s[-1] == ']': return s[1:-1] + if s[0] == '(' and s[-1] == ')': return s[1:-1] + if s[0] == '{' and s[-1] == '}': return s[1:-1] + return s + + +def clean_string(s): + for c in sep: + s = s.replace(c, ' ') + parts = s.split() + return ' '.join(p for p in parts if p != '') + + +def str_replace(string, pos, c): + return string[:pos] + c + string[pos+1:] + +def blank_region(string, region, blank_sep = deleted): + start, end = region + return string[:start] + blank_sep * (end - start) + string[end:] + + +def between(s, left, right): + return s.split(left)[1].split(right)[0] + +def to_utf8(o): + '''converts all unicode strings found in the given object to utf-8 strings''' + + if isinstance(o, unicode): + return o.encode('utf-8') + elif isinstance(o, list): + return [ to_utf8(i) for i in o ] + elif isinstance(o, dict): + result = copy.deepcopy(o) # need to do it like that to handle Guess instances correctly + for key, value in o.items(): + result[to_utf8(key)] = to_utf8(value) + return result + + else: + return o + + +def levenshtein(a, b): + if not a: return len(b) + if not b: return len(a) + + m = len(a) + n = len(b) + d = [] + for i in range(m+1): + d.append([0] * (n+1)) + + for i in range(m+1): + d[i][0] = i + + for j in range(n+1): + d[0][j] = j + + for i in range(1, m+1): + for j in range(1, n+1): + if a[i-1] == b[j-1]: + cost = 0 + else: + cost = 1 + + d[i][j] = min(d[i-1][j] + 1, # deletion + d[i][j-1] + 1, # insertion + d[i-1][j-1] + cost # substitution + ) + + return d[m][n] + + +# group-related functions + +def find_first_level_groups_span(string, enclosing): + """Return a list of pairs (start, end) for the groups delimited by the given + enclosing characters. + This does not return nested groups, ie: '(ab(c)(d))' will return a single group + containing the whole string. + + >>> find_first_level_groups_span('abcd', '()') + [] + + >>> find_first_level_groups_span('abc(de)fgh', '()') + [(3, 7)] + + >>> find_first_level_groups_span('(ab(c)(d))', '()') + [(0, 10)] + + >>> find_first_level_groups_span('ab[c]de[f]gh(i)', '[]') + [(2, 5), (7, 10)] + """ + opening, closing = enclosing + depth = [] # depth is a stack of indices where we opened a group + result = [] + for i, c, in enumerate(string): + if c == opening: + depth.append(i) + elif c == closing: + try: + start = depth.pop() + end = i + if not depth: + # we emptied our stack, so we have a 1st level group + result.append((start, end+1)) + except IndexError: + # we closed a group which was not opened before + pass + + return result + + +def split_on_groups(string, groups): + """Split the given string using the different known groups for boundaries. + + >>> split_on_groups('0123456789', [ (2, 4) ]) + ['01', '23', '456789'] + + >>> split_on_groups('0123456789', [ (2, 4), (4, 6) ]) + ['01', '23', '45', '6789'] + + >>> split_on_groups('0123456789', [ (5, 7), (2, 4) ]) + ['01', '23', '4', '56', '789'] + + """ + if not groups: + return [ string ] + + boundaries = sorted(set(reduce(lambda l, x: l + list(x), groups, []))) + if boundaries[0] != 0: + boundaries.insert(0, 0) + if boundaries[-1] != len(string): + boundaries.append(len(string)) + + groups = [ string[start:end] for start, end in zip(boundaries[:-1], boundaries[1:]) ] + + return filter(bool, groups) # return only non-empty groups + + + + +def find_first_level_groups(string, enclosing, blank_sep = None): + """Return a list of groups that could be split because of explicit grouping. + The groups are delimited by the given enclosing characters. + + You can also specify if you want to blank the separator chars in the returned + list of groups by specifying a character for it. None means it won't be replaced. + + This does not return nested groups, ie: '(ab(c)(d))' will return a single group + containing the whole string. + + >>> find_first_level_groups('', '()') + [''] + + >>> find_first_level_groups('abcd', '()') + ['abcd'] + + >>> find_first_level_groups('abc(de)fgh', '()') + ['abc', '(de)', 'fgh'] + + >>> find_first_level_groups('(ab(c)(d))', '()', blank_sep = '_') + ['_ab(c)(d)_'] + + >>> find_first_level_groups('ab[c]de[f]gh(i)', '[]') + ['ab', '[c]', 'de', '[f]', 'gh(i)'] + + >>> find_first_level_groups('()[]()', '()', blank_sep = '-') + ['--', '[]', '--'] + + """ + groups = find_first_level_groups_span(string, enclosing) + if blank_sep: + for start, end in groups: + string = str_replace(string, start, blank_sep) + string = str_replace(string, end-1, blank_sep) + + return split_on_groups(string, groups) + + + + + diff --git a/libs/requests/__init__.py b/libs/requests/__init__.py new file mode 100644 index 00000000..ee3173e1 --- /dev/null +++ b/libs/requests/__init__.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# __ +# /__) _ _ _ _ _/ _ +# / ( (- (/ (/ (- _) / _) +# / + +""" +requests +~~~~~~~~ + +:copyright: (c) 2011 by Kenneth Reitz. +:license: ISC, see LICENSE for more details. + +""" + +__title__ = 'requests' +__version__ = '0.8.6' +__build__ = 0x000806 +__author__ = 'Kenneth Reitz' +__license__ = 'ISC' +__copyright__ = 'Copyright 2011 Kenneth Reitz' + + + +from . import utils +from .models import Request, Response +from .api import request, get, head, post, patch, put, delete, options +from .sessions import session, Session +from .status_codes import codes +from .exceptions import ( + RequestException, Timeout, URLRequired, + TooManyRedirects, HTTPError, ConnectionError +) diff --git a/libs/requests/api.py b/libs/requests/api.py new file mode 100644 index 00000000..066d4287 --- /dev/null +++ b/libs/requests/api.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +""" +requests.api +~~~~~~~~~~~~ + +This module implements the Requests API. + +:copyright: (c) 2011 by Kenneth Reitz. +:license: ISC, see LICENSE for more details. + +""" + +from . import sessions + +def request(method, url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=False, + proxies=None, + hooks=None, + return_response=True, + prefetch=False, + session=None, + config=None): + """Constructs and sends a :class:`Request `. + Returns :class:`Response ` object. + + :param method: method for the new :class:`Request` object. + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload. + :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param return_response: (optional) If False, an un-sent Request object will returned. + :param session: (optional) A :class:`Session` object to be used for the request. + :param config: (optional) A configuration dictionary. + """ + + s = session or sessions.session() + return s.request( + method=method, + url=url, + params=params, + data=data, + headers=headers, + cookies=cookies, + files=files, + auth=auth, + timeout=timeout, + allow_redirects=allow_redirects, + proxies=proxies, + hooks=hooks, + return_response=return_response, + config=config, + prefetch=prefetch + ) + + + +def get(url, **kwargs): + """Sends a GET request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return request('get', url, **kwargs) + + +def options(url, **kwargs): + """Sends a OPTIONS request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return request('options', url, **kwargs) + + +def head(url, **kwargs): + """Sends a HEAD request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return request('head', url, **kwargs) + + +def post(url, data=None, **kwargs): + """Sends a POST request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return request('post', url, data=data, **kwargs) + + +def put(url, data=None, **kwargs): + """Sends a PUT request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return request('put', url, data=data, **kwargs) + + +def patch(url, data=None, **kwargs): + """Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return request('patch', url, data=data, **kwargs) + + +def delete(url, **kwargs): + """Sends a DELETE request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return request('delete', url, **kwargs) diff --git a/libs/requests/async.py b/libs/requests/async.py new file mode 100644 index 00000000..c91025fc --- /dev/null +++ b/libs/requests/async.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- + +""" +requests.async +~~~~~~~~~~~~~~ + +This module contains an asynchronous replica of ``requests.api``, powered +by gevent. All API methods return a ``Request`` instance (as opposed to +``Response``). A list of requests can be sent with ``map()``. +""" + +try: + import gevent + from gevent import monkey as curious_george + from gevent.pool import Pool +except ImportError: + raise RuntimeError('Gevent is required for requests.async.') + +# Monkey-patch. +curious_george.patch_all(thread=False) + +from . import api + + +__all__ = ( + 'map', + 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request' +) + + +def patched(f): + """Patches a given API function to not send.""" + + def wrapped(*args, **kwargs): + + kwargs['return_response'] = False + kwargs['prefetch'] = True + + return f(*args, **kwargs) + + return wrapped + + +def send(r, pools=None): + """Sends a given Request object.""" + + if pools: + r._pools = pools + + r.send() + + return r.response + + +# Patched requests.api functions. +get = patched(api.get) +options = patched(api.options) +head = patched(api.head) +post = patched(api.post) +put = patched(api.put) +patch = patched(api.patch) +delete = patched(api.delete) +request = patched(api.request) + + +def map(requests, prefetch=True, size=None): + """Concurrently converts a list of Requests to Responses. + + :param requests: a collection of Request objects. + :param prefetch: If False, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. If None, no throttling occurs. + """ + + requests = list(requests) + + if size: + pool = Pool(size) + pool.map(send, requests) + pool.join() + else: + jobs = [gevent.spawn(send, r) for r in requests] + gevent.joinall(jobs) + + if prefetch: + [r.response.content for r in requests] + + return [r.response for r in requests] + + + + diff --git a/libs/requests/auth.py b/libs/requests/auth.py new file mode 100644 index 00000000..4af3d6d3 --- /dev/null +++ b/libs/requests/auth.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- + +""" +requests.auth +~~~~~~~~~~~~~ + +This module contains the authentication handlers for Requests. +""" + +import time +import hashlib + +from base64 import b64encode +from urlparse import urlparse + +from .utils import randombytes, parse_dict_header + + + +def _basic_auth_str(username, password): + """Returns a Basic Auth string.""" + return 'Basic %s' % b64encode('%s:%s' % (username, password)) + + +class AuthBase(object): + """Base class that all auth implementations derive from""" + + def __call__(self, r): + raise NotImplementedError('Auth hooks must be callable.') + + +class HTTPBasicAuth(AuthBase): + """Attaches HTTP Basic Authentication to the given Request object.""" + def __init__(self, username, password): + self.username = str(username) + self.password = str(password) + + def __call__(self, r): + r.headers['Authorization'] = _basic_auth_str(self.username, self.password) + return r + + +class HTTPProxyAuth(HTTPBasicAuth): + """Attaches HTTP Proxy Authenetication to a given Request object.""" + def __call__(self, r): + r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password) + return r + + +class HTTPDigestAuth(AuthBase): + """Attaches HTTP Digest Authentication to the given Request object.""" + def __init__(self, username, password): + self.username = username + self.password = password + + def handle_401(self, r): + """Takes the given response and tries digest-auth, if needed.""" + + s_auth = r.headers.get('www-authenticate', '') + + if 'digest' in s_auth.lower(): + + last_nonce = '' + nonce_count = 0 + + chal = parse_dict_header(s_auth.replace('Digest ', '')) + + realm = chal['realm'] + nonce = chal['nonce'] + qop = chal.get('qop') + algorithm = chal.get('algorithm', 'MD5') + opaque = chal.get('opaque', None) + + algorithm = algorithm.upper() + # lambdas assume digest modules are imported at the top level + if algorithm == 'MD5': + H = lambda x: hashlib.md5(x).hexdigest() + elif algorithm == 'SHA': + H = lambda x: hashlib.sha1(x).hexdigest() + # XXX MD5-sess + KD = lambda s, d: H("%s:%s" % (s, d)) + + if H is None: + return None + + # XXX not implemented yet + entdig = None + p_parsed = urlparse(r.request.url) + path = p_parsed.path + p_parsed.query + + A1 = '%s:%s:%s' % (self.username, realm, self.password) + A2 = '%s:%s' % (r.request.method, path) + + if qop == 'auth': + if nonce == last_nonce: + nonce_count += 1 + else: + nonce_count = 1 + last_nonce = nonce + + ncvalue = '%08x' % nonce_count + cnonce = (hashlib.sha1("%s:%s:%s:%s" % ( + nonce_count, nonce, time.ctime(), randombytes(8))) + .hexdigest()[:16] + ) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) + respdig = KD(H(A1), noncebit) + elif qop is None: + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + else: + # XXX handle auth-int. + return None + + # XXX should the partial digests be encoded too? + base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ + 'response="%s"' % (self.username, realm, nonce, path, respdig) + if opaque: + base += ', opaque="%s"' % opaque + if entdig: + base += ', digest="%s"' % entdig + base += ', algorithm="%s"' % algorithm + if qop: + base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce) + + r.request.headers['Authorization'] = 'Digest %s' % (base) + r.request.send(anyway=True) + _r = r.request.response + _r.history.append(r) + + return _r + + return r + + def __call__(self, r): + r.hooks['response'] = self.handle_401 + return r diff --git a/libs/requests/defaults.py b/libs/requests/defaults.py new file mode 100644 index 00000000..e8db67ab --- /dev/null +++ b/libs/requests/defaults.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +""" +requests.defaults +~~~~~~~~~~~~~~~~~ + +This module provides the Requests configuration defaults. + +Configurations: + +:base_headers: Default HTTP headers. +:verbose: Stream to write request logging to. +:timeout: Seconds until request timeout. +:max_redirects: Maximum number of redirects allowed within a request. +:decode_unicode: Decode unicode responses automatically? +:keep_alive: Reuse HTTP Connections? +:max_retries: The number of times a request should be retried in the event of a connection failure. +:safe_mode: If true, Requests will catch all errors. +:pool_maxsize: The maximium size of an HTTP connection pool. +:pool_connections: The number of active HTTP connection pools to use. + +""" + +from . import __version__ + +defaults = dict() + + +defaults['base_headers'] = { + 'User-Agent': 'python-requests/%s' % __version__, + 'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')), + 'Accept': '*/*' +} + +defaults['verbose'] = None +defaults['max_redirects'] = 30 +defaults['decode_unicode'] = True +defaults['pool_connections'] = 10 +defaults['pool_maxsize'] = 10 +defaults['max_retries'] = 0 +defaults['safe_mode'] = False +defaults['keep_alive'] = True diff --git a/libs/requests/exceptions.py b/libs/requests/exceptions.py new file mode 100644 index 00000000..d0f8f39a --- /dev/null +++ b/libs/requests/exceptions.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +""" +requests.exceptions +~~~~~~~~~~~~~~~~~~~ + +This module contains the set of Requests' exceptions. + +""" + +class RequestException(Exception): + """There was an ambiguous exception that occurred while handling your + request.""" + +class HTTPError(RequestException): + """An HTTP error occurred.""" + +class ConnectionError(RequestException): + """A Connection error occurred.""" + +class Timeout(RequestException): + """The request timed out.""" + +class URLRequired(RequestException): + """A valid URL is required to make a request.""" + +class TooManyRedirects(RequestException): + """Too many redirects.""" diff --git a/libs/requests/hooks.py b/libs/requests/hooks.py new file mode 100644 index 00000000..f9cf4808 --- /dev/null +++ b/libs/requests/hooks.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +""" +requests.hooks +~~~~~~~~~~~~~~ + +This module provides the capabilities for the Requests hooks system. + +Available hooks: + +``args``: + A dictionary of the arguments being sent to Request(). + +``pre_request``: + The Request object, directly before being sent. + +``post_request``: + The Request object, directly after being sent. + +``response``: + The response generated from a Request. + +""" + +import warnings + + +def dispatch_hook(key, hooks, hook_data): + """Dispatches a hook dictionary on a given piece of data.""" + + hooks = hooks or dict() + + if key in hooks: + try: + return hooks.get(key).__call__(hook_data) or hook_data + + except Exception, why: + warnings.warn(str(why)) + + return hook_data diff --git a/libs/requests/models.py b/libs/requests/models.py new file mode 100644 index 00000000..e84c8f1a --- /dev/null +++ b/libs/requests/models.py @@ -0,0 +1,689 @@ +# -*- coding: utf-8 -*- + +""" +requests.models +~~~~~~~~~~~~~~~ + +This module contains the primary objects that power Requests. +""" + +import urllib +import zlib + +from urlparse import urlparse, urlunparse, urljoin, urlsplit +from datetime import datetime + +from .hooks import dispatch_hook +from .structures import CaseInsensitiveDict +from .status_codes import codes +from .packages import oreos +from .auth import HTTPBasicAuth, HTTPProxyAuth +from .packages.urllib3.exceptions import MaxRetryError +from .packages.urllib3.exceptions import SSLError as _SSLError +from .packages.urllib3.exceptions import HTTPError as _HTTPError +from .packages.urllib3 import connectionpool, poolmanager +from .packages.urllib3.filepost import encode_multipart_formdata +from .exceptions import ( + Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) +from .utils import ( + get_encoding_from_headers, stream_decode_response_unicode, + decode_gzip, stream_decode_gzip, guess_filename, requote_path) + + +REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) + + + +class Request(object): + """The :class:`Request ` object. It carries out all functionality of + Requests. Recommended interface is with the Requests functions. + """ + + def __init__(self, + url=None, + headers=dict(), + files=None, + method=None, + data=dict(), + params=dict(), + auth=None, + cookies=None, + timeout=None, + redirect=False, + allow_redirects=False, + proxies=None, + hooks=None, + config=None, + _poolmanager=None): + + #: Float describes the timeout of the request. + # (Use socket.setdefaulttimeout() as fallback) + self.timeout = timeout + + #: Request URL. + self.url = url + + #: Dictionary of HTTP Headers to attach to the :class:`Request `. + self.headers = dict(headers or []) + + #: Dictionary of files to multipart upload (``{filename: content}``). + self.files = files + + #: HTTP Method to use. + self.method = method + + #: Dictionary or byte of request body data to attach to the + #: :class:`Request `. + self.data = None + + #: Dictionary or byte of querystring data to attach to the + #: :class:`Request `. + self.params = None + self.params = dict(params or []) + + #: True if :class:`Request ` is part of a redirect chain (disables history + #: and HTTPError storage). + self.redirect = redirect + + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) + self.allow_redirects = allow_redirects + + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) + self.proxies = dict(proxies or []) + + self.data, self._enc_data = self._encode_params(data) + self.params, self._enc_params = self._encode_params(params) + + #: :class:`Response ` instance, containing + #: content and metadata of HTTP Response, once :attr:`sent `. + self.response = Response() + + #: Authentication tuple or object to attach to :class:`Request `. + self.auth = auth + + #: CookieJar to attach to :class:`Request `. + self.cookies = dict(cookies or []) + + #: Dictionary of configurations for this request. + self.config = dict(config or []) + + #: True if Request has been sent. + self.sent = False + + #: Event-handling hooks. + self.hooks = hooks + + #: Session. + self.session = None + + if headers: + headers = CaseInsensitiveDict(self.headers) + else: + headers = CaseInsensitiveDict() + + for (k, v) in self.config.get('base_headers', {}).items(): + if k not in headers: + headers[k] = v + + self.headers = headers + self._poolmanager = _poolmanager + + # Pre-request hook. + r = dispatch_hook('pre_request', hooks, self) + self.__dict__.update(r.__dict__) + + + def __repr__(self): + return '' % (self.method) + + + def _build_response(self, resp, is_error=False): + """Build internal :class:`Response ` object + from given response. + """ + + def build(resp): + + response = Response() + + # Pass settings over. + response.config = self.config + + if resp: + + # Fallback to None if there's no status_code, for whatever reason. + response.status_code = getattr(resp, 'status', None) + + # Make headers case-insensitive. + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + + # Set encoding. + response.encoding = get_encoding_from_headers(response.headers) + + # Start off with our local cookies. + cookies = self.cookies or dict() + + # Add new cookies from the server. + if 'set-cookie' in response.headers: + cookie_header = response.headers['set-cookie'] + cookies = oreos.dict_from_string(cookie_header) + + # Save cookies in Response. + response.cookies = cookies + + # Save original response for later. + response.raw = resp + + if is_error: + response.error = resp + + response.url = self.full_url + + return response + + history = [] + + r = build(resp) + cookies = self.cookies + self.cookies.update(r.cookies) + + if r.status_code in REDIRECT_STATI and not self.redirect: + + while ( + ('location' in r.headers) and + ((r.status_code is codes.see_other) or (self.allow_redirects)) + ): + + if not len(history) < self.config.get('max_redirects'): + raise TooManyRedirects() + + history.append(r) + + url = r.headers['location'] + + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + + # Facilitate non-RFC2616-compliant 'location' headers + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if not urlparse(url).netloc: + url = urljoin(r.url, url) + + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + if r.status_code is codes.see_other: + method = 'GET' + else: + method = self.method + + # Remove the cookie headers that were sent. + headers = self.headers + try: + del headers['Cookie'] + except KeyError: + pass + + request = Request( + url=url, + headers=headers, + files=self.files, + method=method, + params=self.session.params, + auth=self.auth, + cookies=cookies, + redirect=True, + config=self.config, + timeout=self.timeout, + _poolmanager=self._poolmanager, + proxies = self.proxies, + ) + + request.send() + cookies.update(request.response.cookies) + r = request.response + self.cookies.update(r.cookies) + + r.history = history + + self.response = r + self.response.request = self + self.response.cookies.update(self.cookies) + + + @staticmethod + def _encode_params(data): + """Encode parameters in a piece of data. + + If the data supplied is a dictionary, encodes each parameter in it, and + returns a list of tuples containing the encoded parameters, and a urlencoded + version of that. + + Otherwise, assumes the data is already encoded appropriately, and + returns it twice. + """ + + if hasattr(data, '__iter__'): + data = dict(data) + + if hasattr(data, 'items'): + result = [] + for k, vs in data.items(): + for v in isinstance(vs, list) and vs or [vs]: + result.append((k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v)) + return result, urllib.urlencode(result, doseq=True) + else: + return data, data + + @property + def full_url(self): + """Build the actual URL to use.""" + + if not self.url: + raise URLRequired() + + # Support for unicode domain names and paths. + scheme, netloc, path, params, query, fragment = urlparse(self.url) + + if not scheme: + raise ValueError() + + netloc = netloc.encode('idna') + + if isinstance(path, unicode): + path = path.encode('utf-8') + + path = requote_path(path) + + url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) + + if self._enc_params: + if urlparse(url).query: + return '%s&%s' % (url, self._enc_params) + else: + return '%s?%s' % (url, self._enc_params) + else: + return url + + @property + def path_url(self): + """Build the path URL to use.""" + + url = [] + + p = urlsplit(self.full_url) + + # Proxies use full URLs. + if p.scheme in self.proxies: + return self.full_url + + path = p.path + if not path: + path = '/' + url.append(path) + + query = p.query + if query: + url.append('?') + url.append(query) + + return ''.join(url) + + + + def send(self, anyway=False, prefetch=False): + """Sends the request. Returns True of successful, false if not. + If there was an HTTPError during transmission, + self.response.status_code will contain the HTTPError code. + + Once a request is successfully sent, `sent` will equal True. + + :param anyway: If True, request will be sent, even if it has + already been sent. + """ + + # Build the URL + url = self.full_url + + # Logging + if self.config.get('verbose'): + self.config.get('verbose').write('%s %s %s\n' % ( + datetime.now().isoformat(), self.method, url + )) + + # Nottin' on you. + body = None + content_type = None + + # Multi-part file uploads. + if self.files: + if not isinstance(self.data, basestring): + + try: + fields = self.data.copy() + except AttributeError: + fields = dict(self.data) + + for (k, v) in self.files.items(): + # support for explicit filename + if isinstance(v, (tuple, list)): + fn, fp = v + else: + fn = guess_filename(v) or k + fp = v + fields.update({k: (fn, fp.read())}) + + (body, content_type) = encode_multipart_formdata(fields) + else: + pass + # TODO: Conflict? + else: + if self.data: + + body = self._enc_data + if isinstance(self.data, basestring): + content_type = None + else: + content_type = 'application/x-www-form-urlencoded' + + # Add content-type if it wasn't explicitly provided. + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type + + if self.auth: + if isinstance(self.auth, tuple) and len(self.auth) == 2: + # special-case basic HTTP auth + self.auth = HTTPBasicAuth(*self.auth) + + # Allow auth to make its changes. + r = self.auth(self) + + # Update self to reflect the auth changes. + self.__dict__.update(r.__dict__) + + _p = urlparse(url) + proxy = self.proxies.get(_p.scheme) + + if proxy: + conn = poolmanager.proxy_from_url(proxy) + _proxy = urlparse(proxy) + if '@' in _proxy.netloc: + auth, url = _proxy.netloc.split('@', 1) + self.proxy_auth = HTTPProxyAuth(*auth.split(':', 1)) + r = self.proxy_auth(self) + self.__dict__.update(r.__dict__) + else: + # Check to see if keep_alive is allowed. + if self.config.get('keep_alive'): + conn = self._poolmanager.connection_from_url(url) + else: + conn = connectionpool.connection_from_url(url) + + if not self.sent or anyway: + + if self.cookies: + + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: + + # Simple cookie with our dict. + c = oreos.monkeys.SimpleCookie() + for (k, v) in self.cookies.items(): + c[k] = v + + # Turn it into a header. + cookie_header = c.output(header='', sep='; ').strip() + + # Attach Cookie header to request. + self.headers['Cookie'] = cookie_header + + try: + # Send the request. + r = conn.urlopen( + method=self.method, + url=self.path_url, + body=body, + headers=self.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.config.get('max_retries', 0), + timeout=self.timeout, + ) + self.sent = True + + + except MaxRetryError, e: + if not self.config.get('safe_mode', False): + raise ConnectionError(e) + else: + r = None + + except (_SSLError, _HTTPError), e: + if not self.config.get('safe_mode', False): + raise Timeout('Request timed out.') + + self._build_response(r) + + # Response manipulation hook. + self.response = dispatch_hook('response', self.hooks, self.response) + + # Post-request hook. + r = dispatch_hook('post_request', self.hooks, self) + self.__dict__.update(r.__dict__) + + # If prefetch is True, mark content as consumed. + if prefetch: + # Save the response. + self.response.content + + return self.sent + + +class Response(object): + """The core :class:`Response ` object. All + :class:`Request ` objects contain a + :class:`response ` attribute, which is an instance + of this class. + """ + + def __init__(self): + + self._content = None + self._content_consumed = False + + #: Integer Code of responded HTTP Status. + self.status_code = None + + #: Case-insensitive Dictionary of Response Headers. + #: For example, ``headers['content-encoding']`` will return the + #: value of a ``'Content-Encoding'`` response header. + self.headers = CaseInsensitiveDict() + + #: File-like object representation of response (for advanced usage). + self.raw = None + + #: Final URL location of Response. + self.url = None + + #: Resulting :class:`HTTPError` of request, if one occurred. + self.error = None + + #: Encoding to decode with when accessing r.content. + self.encoding = None + + #: A list of :class:`Response ` objects from + #: the history of the Request. Any redirect responses will end + #: up here. + self.history = [] + + #: The :class:`Request ` that created the Response. + self.request = None + + #: A dictionary of Cookies the server sent back. + self.cookies = {} + + #: Dictionary of configurations for this request. + self.config = {} + + + def __repr__(self): + return '' % (self.status_code) + + def __nonzero__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return self.ok + + @property + def ok(self): + try: + self.raise_for_status() + except HTTPError: + return False + return True + + + def iter_content(self, chunk_size=10 * 1024, decode_unicode=None): + """Iterates over the response data. This avoids reading the content + at once into memory for large responses. The chunk size is the number + of bytes it should read into memory. This is not necessarily the + length of each item returned as decoding can take place. + """ + if self._content_consumed: + raise RuntimeError( + 'The content for this response was already consumed' + ) + + def generate(): + while 1: + chunk = self.raw.read(chunk_size) + if not chunk: + break + yield chunk + self._content_consumed = True + + gen = generate() + + if 'gzip' in self.headers.get('content-encoding', ''): + gen = stream_decode_gzip(gen) + + if decode_unicode is None: + decode_unicode = self.config.get('decode_unicode') + + if decode_unicode: + gen = stream_decode_response_unicode(gen, self) + + return gen + + + def iter_lines(self, newlines=None, decode_unicode=None): + """Iterates over the response data, one line at a time. This + avoids reading the content at once into memory for large + responses. + + :param newlines: a collection of bytes to seperate lines with. + """ + + if newlines is None: + newlines = ('\r', '\n', '\r\n') + + if self._content_consumed: + raise RuntimeError( + 'The content for this response was already consumed' + ) + + def generate(): + chunk = [] + + while 1: + c = self.raw.read(1) + if not c: + break + + if c in newlines: + yield ''.join(chunk) + chunk = [] + else: + chunk.append(c) + + self._content_consumed = True + + gen = generate() + + if 'gzip' in self.headers.get('content-encoding', ''): + gen = stream_decode_gzip(gen) + + if decode_unicode is None: + decode_unicode = self.config.get('decode_unicode') + + if decode_unicode: + gen = stream_decode_response_unicode(gen, self) + + return gen + + + @property + def content(self): + """Content of the response, in bytes or unicode + (if available). + """ + + if self._content is None: + # Read the contents. + try: + if self._content_consumed: + raise RuntimeError( + 'The content for this response was already consumed') + + self._content = self.raw.read() + except AttributeError: + self._content = None + + content = self._content + + # Decode GZip'd content. + if 'gzip' in self.headers.get('content-encoding', ''): + try: + content = decode_gzip(self._content) + except zlib.error: + pass + + # Decode unicode content. + if self.config.get('decode_unicode'): + + # Try charset from content-type + + if self.encoding: + try: + content = unicode(content, self.encoding) + except UnicodeError: + pass + + # Fall back: + try: + content = unicode(content, self.encoding, errors='replace') + except TypeError: + pass + + self._content_consumed = True + return content + + + def raise_for_status(self): + """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" + + if self.error: + raise self.error + + if (self.status_code >= 300) and (self.status_code < 400): + raise HTTPError('%s Redirection' % self.status_code) + + elif (self.status_code >= 400) and (self.status_code < 500): + raise HTTPError('%s Client Error' % self.status_code) + + elif (self.status_code >= 500) and (self.status_code < 600): + raise HTTPError('%s Server Error' % self.status_code) + + diff --git a/libs/requests/packages/__init__.py b/libs/requests/packages/__init__.py new file mode 100644 index 00000000..d62c4b71 --- /dev/null +++ b/libs/requests/packages/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from . import urllib3 diff --git a/libs/requests/packages/oreos/__init__.py b/libs/requests/packages/oreos/__init__.py new file mode 100644 index 00000000..d01340f2 --- /dev/null +++ b/libs/requests/packages/oreos/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from .core import dict_from_string \ No newline at end of file diff --git a/libs/requests/packages/oreos/core.py b/libs/requests/packages/oreos/core.py new file mode 100644 index 00000000..359d7447 --- /dev/null +++ b/libs/requests/packages/oreos/core.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- + +""" +oreos.core +~~~~~~~~~~ + +The creamy white center. +""" + +from .monkeys import SimpleCookie + + +def dict_from_string(s): + """Returns a MultiDict with Cookies.""" + + cookies = dict() + + c = SimpleCookie() + c.load(s) + + for k,v in c.items(): + cookies.update({k: v.value}) + + return cookies \ No newline at end of file diff --git a/libs/requests/packages/oreos/monkeys.py b/libs/requests/packages/oreos/monkeys.py new file mode 100644 index 00000000..6be3074c --- /dev/null +++ b/libs/requests/packages/oreos/monkeys.py @@ -0,0 +1,770 @@ +# -*- coding: utf-8 -*- + +""" +oreos.monkeys +~~~~~~~~~~~~~ + +Monkeypatches. +""" +#!/usr/bin/env python +# + +#### +# Copyright 2000 by Timothy O'Malley +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software +# and its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Timothy O'Malley not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR +# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. +# +#### +# +# Id: Cookie.py,v 2.29 2000/08/23 05:28:49 timo Exp +# by Timothy O'Malley +# +# Cookie.py is a Python module for the handling of HTTP +# cookies as a Python dictionary. See RFC 2109 for more +# information on cookies. +# +# The original idea to treat Cookies as a dictionary came from +# Dave Mitchell (davem@magnet.com) in 1995, when he released the +# first version of nscookie.py. +# +#### + +r""" +Here's a sample session to show how to use this module. +At the moment, this is the only documentation. + +The Basics +---------- + +Importing is easy.. + + >>> import Cookie + +Most of the time you start by creating a cookie. Cookies come in +three flavors, each with slightly different encoding semantics, but +more on that later. + + >>> C = Cookie.SimpleCookie() + >>> C = Cookie.SerialCookie() + >>> C = Cookie.SmartCookie() + +[Note: Long-time users of Cookie.py will remember using +Cookie.Cookie() to create an Cookie object. Although deprecated, it +is still supported by the code. See the Backward Compatibility notes +for more information.] + +Once you've created your Cookie, you can add values just as if it were +a dictionary. + + >>> C = Cookie.SmartCookie() + >>> C["fig"] = "newton" + >>> C["sugar"] = "wafer" + >>> C.output() + 'Set-Cookie: fig=newton\r\nSet-Cookie: sugar=wafer' + +Notice that the printable representation of a Cookie is the +appropriate format for a Set-Cookie: header. This is the +default behavior. You can change the header and printed +attributes by using the .output() function + + >>> C = Cookie.SmartCookie() + >>> C["rocky"] = "road" + >>> C["rocky"]["path"] = "/cookie" + >>> print C.output(header="Cookie:") + Cookie: rocky=road; Path=/cookie + >>> print C.output(attrs=[], header="Cookie:") + Cookie: rocky=road + +The load() method of a Cookie extracts cookies from a string. In a +CGI script, you would use this method to extract the cookies from the +HTTP_COOKIE environment variable. + + >>> C = Cookie.SmartCookie() + >>> C.load("chips=ahoy; vienna=finger") + >>> C.output() + 'Set-Cookie: chips=ahoy\r\nSet-Cookie: vienna=finger' + +The load() method is darn-tootin smart about identifying cookies +within a string. Escaped quotation marks, nested semicolons, and other +such trickeries do not confuse it. + + >>> C = Cookie.SmartCookie() + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> print C + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + +Each element of the Cookie also supports all of the RFC 2109 +Cookie attributes. Here's an example which sets the Path +attribute. + + >>> C = Cookie.SmartCookie() + >>> C["oreo"] = "doublestuff" + >>> C["oreo"]["path"] = "/" + >>> print C + Set-Cookie: oreo=doublestuff; Path=/ + +Each dictionary element has a 'value' attribute, which gives you +back the value associated with the key. + + >>> C = Cookie.SmartCookie() + >>> C["twix"] = "none for you" + >>> C["twix"].value + 'none for you' + + +A Bit More Advanced +------------------- + +As mentioned before, there are three different flavors of Cookie +objects, each with different encoding/decoding semantics. This +section briefly discusses the differences. + +SimpleCookie + +The SimpleCookie expects that all values should be standard strings. +Just to be sure, SimpleCookie invokes the str() builtin to convert +the value to a string, when the values are set dictionary-style. + + >>> C = Cookie.SimpleCookie() + >>> C["number"] = 7 + >>> C["string"] = "seven" + >>> C["number"].value + '7' + >>> C["string"].value + 'seven' + >>> C.output() + 'Set-Cookie: number=7\r\nSet-Cookie: string=seven' + + +SerialCookie + +The SerialCookie expects that all values should be serialized using +cPickle (or pickle, if cPickle isn't available). As a result of +serializing, SerialCookie can save almost any Python object to a +value, and recover the exact same object when the cookie has been +returned. (SerialCookie can yield some strange-looking cookie +values, however.) + + >>> C = Cookie.SerialCookie() + >>> C["number"] = 7 + >>> C["string"] = "seven" + >>> C["number"].value + 7 + >>> C["string"].value + 'seven' + >>> C.output() + 'Set-Cookie: number="I7\\012."\r\nSet-Cookie: string="S\'seven\'\\012p1\\012."' + +Be warned, however, if SerialCookie cannot de-serialize a value (because +it isn't a valid pickle'd object), IT WILL RAISE AN EXCEPTION. + + +SmartCookie + +The SmartCookie combines aspects of each of the other two flavors. +When setting a value in a dictionary-fashion, the SmartCookie will +serialize (ala cPickle) the value *if and only if* it isn't a +Python string. String objects are *not* serialized. Similarly, +when the load() method parses out values, it attempts to de-serialize +the value. If it fails, then it fallsback to treating the value +as a string. + + >>> C = Cookie.SmartCookie() + >>> C["number"] = 7 + >>> C["string"] = "seven" + >>> C["number"].value + 7 + >>> C["string"].value + 'seven' + >>> C.output() + 'Set-Cookie: number="I7\\012."\r\nSet-Cookie: string=seven' + + +Backwards Compatibility +----------------------- + +In order to keep compatibilty with earlier versions of Cookie.py, +it is still possible to use Cookie.Cookie() to create a Cookie. In +fact, this simply returns a SmartCookie. + + >>> C = Cookie.Cookie() + >>> print C.__class__.__name__ + SmartCookie + + +Finis. +""" #" +# ^ +# |----helps out font-lock + +# +# Import our required modules +# +import string + +try: + from cPickle import dumps, loads +except ImportError: + from pickle import dumps, loads + +import re, warnings + +__all__ = ["CookieError","BaseCookie","SimpleCookie","SerialCookie", + "SmartCookie","Cookie"] + +_nulljoin = ''.join +_semispacejoin = '; '.join +_spacejoin = ' '.join + +# +# Define an exception visible to External modules +# +class CookieError(Exception): + pass + + +# These quoting routines conform to the RFC2109 specification, which in +# turn references the character definitions from RFC2068. They provide +# a two-way quoting algorithm. Any non-text character is translated +# into a 4 character sequence: a forward-slash followed by the +# three-digit octal equivalent of the character. Any '\' or '"' is +# quoted with a preceeding '\' slash. +# +# These are taken from RFC2068 and RFC2109. +# _LegalChars is the list of chars which don't require "'s +# _Translator hash-table for fast quoting +# +_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~[]_" +_Translator = { + '\000' : '\\000', '\001' : '\\001', '\002' : '\\002', + '\003' : '\\003', '\004' : '\\004', '\005' : '\\005', + '\006' : '\\006', '\007' : '\\007', '\010' : '\\010', + '\011' : '\\011', '\012' : '\\012', '\013' : '\\013', + '\014' : '\\014', '\015' : '\\015', '\016' : '\\016', + '\017' : '\\017', '\020' : '\\020', '\021' : '\\021', + '\022' : '\\022', '\023' : '\\023', '\024' : '\\024', + '\025' : '\\025', '\026' : '\\026', '\027' : '\\027', + '\030' : '\\030', '\031' : '\\031', '\032' : '\\032', + '\033' : '\\033', '\034' : '\\034', '\035' : '\\035', + '\036' : '\\036', '\037' : '\\037', + + # Because of the way browsers really handle cookies (as opposed + # to what the RFC says) we also encode , and ; + + ',' : '\\054', ';' : '\\073', + + '"' : '\\"', '\\' : '\\\\', + + '\177' : '\\177', '\200' : '\\200', '\201' : '\\201', + '\202' : '\\202', '\203' : '\\203', '\204' : '\\204', + '\205' : '\\205', '\206' : '\\206', '\207' : '\\207', + '\210' : '\\210', '\211' : '\\211', '\212' : '\\212', + '\213' : '\\213', '\214' : '\\214', '\215' : '\\215', + '\216' : '\\216', '\217' : '\\217', '\220' : '\\220', + '\221' : '\\221', '\222' : '\\222', '\223' : '\\223', + '\224' : '\\224', '\225' : '\\225', '\226' : '\\226', + '\227' : '\\227', '\230' : '\\230', '\231' : '\\231', + '\232' : '\\232', '\233' : '\\233', '\234' : '\\234', + '\235' : '\\235', '\236' : '\\236', '\237' : '\\237', + '\240' : '\\240', '\241' : '\\241', '\242' : '\\242', + '\243' : '\\243', '\244' : '\\244', '\245' : '\\245', + '\246' : '\\246', '\247' : '\\247', '\250' : '\\250', + '\251' : '\\251', '\252' : '\\252', '\253' : '\\253', + '\254' : '\\254', '\255' : '\\255', '\256' : '\\256', + '\257' : '\\257', '\260' : '\\260', '\261' : '\\261', + '\262' : '\\262', '\263' : '\\263', '\264' : '\\264', + '\265' : '\\265', '\266' : '\\266', '\267' : '\\267', + '\270' : '\\270', '\271' : '\\271', '\272' : '\\272', + '\273' : '\\273', '\274' : '\\274', '\275' : '\\275', + '\276' : '\\276', '\277' : '\\277', '\300' : '\\300', + '\301' : '\\301', '\302' : '\\302', '\303' : '\\303', + '\304' : '\\304', '\305' : '\\305', '\306' : '\\306', + '\307' : '\\307', '\310' : '\\310', '\311' : '\\311', + '\312' : '\\312', '\313' : '\\313', '\314' : '\\314', + '\315' : '\\315', '\316' : '\\316', '\317' : '\\317', + '\320' : '\\320', '\321' : '\\321', '\322' : '\\322', + '\323' : '\\323', '\324' : '\\324', '\325' : '\\325', + '\326' : '\\326', '\327' : '\\327', '\330' : '\\330', + '\331' : '\\331', '\332' : '\\332', '\333' : '\\333', + '\334' : '\\334', '\335' : '\\335', '\336' : '\\336', + '\337' : '\\337', '\340' : '\\340', '\341' : '\\341', + '\342' : '\\342', '\343' : '\\343', '\344' : '\\344', + '\345' : '\\345', '\346' : '\\346', '\347' : '\\347', + '\350' : '\\350', '\351' : '\\351', '\352' : '\\352', + '\353' : '\\353', '\354' : '\\354', '\355' : '\\355', + '\356' : '\\356', '\357' : '\\357', '\360' : '\\360', + '\361' : '\\361', '\362' : '\\362', '\363' : '\\363', + '\364' : '\\364', '\365' : '\\365', '\366' : '\\366', + '\367' : '\\367', '\370' : '\\370', '\371' : '\\371', + '\372' : '\\372', '\373' : '\\373', '\374' : '\\374', + '\375' : '\\375', '\376' : '\\376', '\377' : '\\377' + } + +_idmap = ''.join(chr(x) for x in xrange(256)) + +def _quote(str, LegalChars=_LegalChars, + idmap=_idmap, translate=string.translate): + # + # If the string does not need to be double-quoted, + # then just return the string. Otherwise, surround + # the string in doublequotes and precede quote (with a \) + # special characters. + # + if "" == translate(str, idmap, LegalChars): + return str + else: + return '"' + _nulljoin( map(_Translator.get, str, str) ) + '"' +# end _quote + + +_OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") +_QuotePatt = re.compile(r"[\\].") + +def _unquote(str): + # If there aren't any doublequotes, + # then there can't be any special characters. See RFC 2109. + if len(str) < 2: + return str + if str[0] != '"' or str[-1] != '"': + return str + + # We have to assume that we must decode this string. + # Down to work. + + # Remove the "s + str = str[1:-1] + + # Check for special sequences. Examples: + # \012 --> \n + # \" --> " + # + i = 0 + n = len(str) + res = [] + while 0 <= i < n: + Omatch = _OctalPatt.search(str, i) + Qmatch = _QuotePatt.search(str, i) + if not Omatch and not Qmatch: # Neither matched + res.append(str[i:]) + break + # else: + j = k = -1 + if Omatch: j = Omatch.start(0) + if Qmatch: k = Qmatch.start(0) + if Qmatch and ( not Omatch or k < j ): # QuotePatt matched + res.append(str[i:k]) + res.append(str[k+1]) + i = k+2 + else: # OctalPatt matched + res.append(str[i:j]) + res.append( chr( int(str[j+1:j+4], 8) ) ) + i = j+4 + return _nulljoin(res) +# end _unquote + +# The _getdate() routine is used to set the expiration time in +# the cookie's HTTP header. By default, _getdate() returns the +# current time in the appropriate "expires" format for a +# Set-Cookie header. The one optional argument is an offset from +# now, in seconds. For example, an offset of -3600 means "one hour ago". +# The offset may be a floating point number. +# + +_weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] + +_monthname = [None, + 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + +def _getdate(future=0, weekdayname=_weekdayname, monthname=_monthname): + from time import gmtime, time + now = time() + year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future) + return "%s, %02d-%3s-%4d %02d:%02d:%02d GMT" % \ + (weekdayname[wd], day, monthname[month], year, hh, mm, ss) + + +# +# A class to hold ONE key,value pair. +# In a cookie, each such pair may have several attributes. +# so this class is used to keep the attributes associated +# with the appropriate key,value pair. +# This class also includes a coded_value attribute, which +# is used to hold the network representation of the +# value. This is most useful when Python objects are +# pickled for network transit. +# + +class Morsel(dict): + # RFC 2109 lists these attributes as reserved: + # path comment domain + # max-age secure version + # + # For historical reasons, these attributes are also reserved: + # expires + # + # This is an extension from Microsoft: + # httponly + # + # This dictionary provides a mapping from the lowercase + # variant on the left to the appropriate traditional + # formatting on the right. + _reserved = { "expires" : "expires", + "path" : "Path", + "comment" : "Comment", + "domain" : "Domain", + "max-age" : "Max-Age", + "secure" : "secure", + "httponly" : "httponly", + "version" : "Version", + } + + def __init__(self): + # Set defaults + self.key = self.value = self.coded_value = None + + # Set default attributes + for K in self._reserved: + dict.__setitem__(self, K, "") + # end __init__ + + def __setitem__(self, K, V): + K = K.lower() + if not K in self._reserved: + raise CookieError("Invalid Attribute %s" % K) + dict.__setitem__(self, K, V) + # end __setitem__ + + def isReservedKey(self, K): + return K.lower() in self._reserved + # end isReservedKey + + def set(self, key, val, coded_val, + LegalChars=_LegalChars, + idmap=_idmap, translate=string.translate): + # First we verify that the key isn't a reserved word + # Second we make sure it only contains legal characters + if key.lower() in self._reserved: + raise CookieError("Attempt to set a reserved key: %s" % key) + if "" != translate(key, idmap, LegalChars): + raise CookieError("Illegal key value: %s" % key) + + # It's a good key, so save it. + self.key = key + self.value = val + self.coded_value = coded_val + # end set + + def output(self, attrs=None, header = "Set-Cookie:"): + return "%s %s" % ( header, self.OutputString(attrs) ) + + __str__ = output + + def __repr__(self): + return '<%s: %s=%s>' % (self.__class__.__name__, + self.key, repr(self.value) ) + + def js_output(self, attrs=None): + # Print javascript + return """ + + """ % ( self.OutputString(attrs).replace('"',r'\"'), ) + # end js_output() + + def OutputString(self, attrs=None): + # Build up our result + # + result = [] + RA = result.append + + # First, the key=value pair + RA("%s=%s" % (self.key, self.coded_value)) + + # Now add any defined attributes + if attrs is None: + attrs = self._reserved + items = self.items() + items.sort() + for K,V in items: + if V == "": continue + if K not in attrs: continue + if K == "expires" and type(V) == type(1): + RA("%s=%s" % (self._reserved[K], _getdate(V))) + elif K == "max-age" and type(V) == type(1): + RA("%s=%d" % (self._reserved[K], V)) + elif K == "secure": + RA(str(self._reserved[K])) + elif K == "httponly": + RA(str(self._reserved[K])) + else: + RA("%s=%s" % (self._reserved[K], V)) + + # Return the result + return _semispacejoin(result) + # end OutputString +# end Morsel class + + + +# +# Pattern for finding cookie +# +# This used to be strict parsing based on the RFC2109 and RFC2068 +# specifications. I have since discovered that MSIE 3.0x doesn't +# follow the character rules outlined in those specs. As a +# result, the parsing rules here are less strict. +# + +_LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=\[\]\_]" + +_CookiePattern = re.compile( + r"(?x)" # This is a Verbose pattern + r"(?P" # Start of group 'key' + ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy + r")" # End of group 'key' + r"\s*=\s*" # Equal Sign + r"(?P" # Start of group 'val' + r'"(?:[^\\"]|\\.)*"' # Any doublequoted string + r"|" # or + r"\w{3},\s[\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr + r"|" # or + ""+ _LegalCharsPatt +"*" # Any word or empty string + r")" # End of group 'val' + r"\s*;?" # Probably ending in a semi-colon + ) + + +# At long last, here is the cookie class. +# Using this class is almost just like using a dictionary. +# See this module's docstring for example usage. +# +class BaseCookie(dict): + # A container class for a set of Morsels + # + + def value_decode(self, val): + """real_value, coded_value = value_decode(STRING) + Called prior to setting a cookie's value from the network + representation. The VALUE is the value read from HTTP + header. + Override this function to modify the behavior of cookies. + """ + return val, val + # end value_encode + + def value_encode(self, val): + """real_value, coded_value = value_encode(VALUE) + Called prior to setting a cookie's value from the dictionary + representation. The VALUE is the value being assigned. + Override this function to modify the behavior of cookies. + """ + strval = str(val) + return strval, strval + # end value_encode + + def __init__(self, input=None): + if input: self.load(input) + # end __init__ + + def __set(self, key, real_value, coded_value): + """Private method for setting a cookie's value""" + M = self.get(key, Morsel()) + M.set(key, real_value, coded_value) + dict.__setitem__(self, key, M) + # end __set + + def __setitem__(self, key, value): + """Dictionary style assignment.""" + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) + # end __setitem__ + + def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): + """Return a string suitable for HTTP.""" + result = [] + items = self.items() + items.sort() + for K,V in items: + result.append( V.output(attrs, header) ) + return sep.join(result) + # end output + + __str__ = output + + def __repr__(self): + L = [] + items = self.items() + items.sort() + for K,V in items: + L.append( '%s=%s' % (K,repr(V.value) ) ) + return '<%s: %s>' % (self.__class__.__name__, _spacejoin(L)) + + def js_output(self, attrs=None): + """Return a string suitable for JavaScript.""" + result = [] + items = self.items() + items.sort() + for K,V in items: + result.append( V.js_output(attrs) ) + return _nulljoin(result) + # end js_output + + def load(self, rawdata): + """Load cookies from a string (presumably HTTP_COOKIE) or + from a dictionary. Loading cookies from a dictionary 'd' + is equivalent to calling: + map(Cookie.__setitem__, d.keys(), d.values()) + """ + if type(rawdata) == type(""): + self.__ParseString(rawdata) + else: + # self.update() wouldn't call our custom __setitem__ + for k, v in rawdata.items(): + self[k] = v + return + # end load() + + def __ParseString(self, str, patt=_CookiePattern): + i = 0 # Our starting point + n = len(str) # Length of string + M = None # current morsel + + while 0 <= i < n: + # Start looking for a cookie + match = patt.search(str, i) + if not match: break # No more cookies + + K,V = match.group("key"), match.group("val") + i = match.end(0) + + # Parse the key, value in case it's metainfo + if K[0] == "$": + # We ignore attributes which pertain to the cookie + # mechanism as a whole. See RFC 2109. + # (Does anyone care?) + if M: + M[ K[1:] ] = V + elif K.lower() in Morsel._reserved: + if M: + M[ K ] = _unquote(V) + else: + rval, cval = self.value_decode(V) + self.__set(K, rval, cval) + M = self[K] + # end __ParseString +# end BaseCookie class + +class SimpleCookie(BaseCookie): + """SimpleCookie + SimpleCookie supports strings as cookie values. When setting + the value using the dictionary assignment notation, SimpleCookie + calls the builtin str() to convert the value to a string. Values + received from HTTP are kept as strings. + """ + def value_decode(self, val): + return _unquote( val ), val + def value_encode(self, val): + strval = str(val) + return strval, _quote( strval ) +# end SimpleCookie + +class SerialCookie(BaseCookie): + """SerialCookie + SerialCookie supports arbitrary objects as cookie values. All + values are serialized (using cPickle) before being sent to the + client. All incoming values are assumed to be valid Pickle + representations. IF AN INCOMING VALUE IS NOT IN A VALID PICKLE + FORMAT, THEN AN EXCEPTION WILL BE RAISED. + + Note: Large cookie values add overhead because they must be + retransmitted on every HTTP transaction. + + Note: HTTP has a 2k limit on the size of a cookie. This class + does not check for this limit, so be careful!!! + """ + def __init__(self, input=None): + warnings.warn("SerialCookie class is insecure; do not use it", + DeprecationWarning) + BaseCookie.__init__(self, input) + # end __init__ + def value_decode(self, val): + # This could raise an exception! + return loads( _unquote(val) ), val + def value_encode(self, val): + return val, _quote( dumps(val) ) +# end SerialCookie + +class SmartCookie(BaseCookie): + """SmartCookie + SmartCookie supports arbitrary objects as cookie values. If the + object is a string, then it is quoted. If the object is not a + string, however, then SmartCookie will use cPickle to serialize + the object into a string representation. + + Note: Large cookie values add overhead because they must be + retransmitted on every HTTP transaction. + + Note: HTTP has a 2k limit on the size of a cookie. This class + does not check for this limit, so be careful!!! + """ + def __init__(self, input=None): + warnings.warn("Cookie/SmartCookie class is insecure; do not use it", + DeprecationWarning) + BaseCookie.__init__(self, input) + # end __init__ + def value_decode(self, val): + strval = _unquote(val) + try: + return loads(strval), val + except: + return strval, val + def value_encode(self, val): + if type(val) == type(""): + return val, _quote(val) + else: + return val, _quote( dumps(val) ) +# end SmartCookie + + +########################################################### +# Backwards Compatibility: Don't break any existing code! + +# We provide Cookie() as an alias for SmartCookie() +Cookie = SmartCookie + +# +########################################################### + +def _test(): + import doctest, Cookie + return doctest.testmod(Cookie) + +if __name__ == "__main__": + _test() + + +#Local Variables: +#tab-width: 4 +#end: diff --git a/libs/requests/packages/oreos/structures.py b/libs/requests/packages/oreos/structures.py new file mode 100644 index 00000000..063d5f96 --- /dev/null +++ b/libs/requests/packages/oreos/structures.py @@ -0,0 +1,399 @@ +# -*- coding: utf-8 -*- + +""" +oreos.sructures +~~~~~~~~~~~~~~~ + +The plastic blue packaging. + +This is mostly directly stolen from mitsuhiko/werkzeug. +""" + +__all__ = ('MultiDict',) + +class _Missing(object): + + def __repr__(self): + return 'no value' + + def __reduce__(self): + return '_missing' + +_missing = _Missing() + + + +def iter_multi_items(mapping): + """Iterates over the items of a mapping yielding keys and values + without dropping any from more complex structures. + """ + if isinstance(mapping, MultiDict): + for item in mapping.iteritems(multi=True): + yield item + elif isinstance(mapping, dict): + for key, value in mapping.iteritems(): + if isinstance(value, (tuple, list)): + for value in value: + yield key, value + else: + yield key, value + else: + for item in mapping: + yield item + + + +class TypeConversionDict(dict): + """Works like a regular dict but the :meth:`get` method can perform + type conversions. :class:`MultiDict` and :class:`CombinedMultiDict` + are subclasses of this class and provide the same feature. + + .. versionadded:: 0.5 + """ + + def get(self, key, default=None, type=None): + """Return the default value if the requested data doesn't exist. + If `type` is provided and is a callable it should convert the value, + return it or raise a :exc:`ValueError` if that is not possible. In + this case the function will return the default as if the value was not + found: + + >>> d = TypeConversionDict(foo='42', bar='blub') + >>> d.get('foo', type=int) + 42 + >>> d.get('bar', -1, type=int) + -1 + + :param key: The key to be looked up. + :param default: The default value to be returned if the key can't + be looked up. If not further specified `None` is + returned. + :param type: A callable that is used to cast the value in the + :class:`MultiDict`. If a :exc:`ValueError` is raised + by this callable the default value is returned. + """ + try: + rv = self[key] + if type is not None: + rv = type(rv) + except (KeyError, ValueError): + rv = default + return rv + + +class MultiDict(TypeConversionDict): + """A :class:`MultiDict` is a dictionary subclass customized to deal with + multiple values for the same key which is for example used by the parsing + functions in the wrappers. This is necessary because some HTML form + elements pass multiple values for the same key. + + :class:`MultiDict` implements all standard dictionary methods. + Internally, it saves all values for a key as a list, but the standard dict + access methods will only return the first value for a key. If you want to + gain access to the other values, too, you have to use the `list` methods as + explained below. + + Basic Usage: + + >>> d = MultiDict([('a', 'b'), ('a', 'c')]) + >>> d + MultiDict([('a', 'b'), ('a', 'c')]) + >>> d['a'] + 'b' + >>> d.getlist('a') + ['b', 'c'] + >>> 'a' in d + True + + It behaves like a normal dict thus all dict functions will only return the + first value when multiple values for one key are found. + + From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a + subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will + render a page for a ``400 BAD REQUEST`` if caught in a catch-all for HTTP + exceptions. + + A :class:`MultiDict` can be constructed from an iterable of + ``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2 + onwards some keyword parameters. + + :param mapping: the initial value for the :class:`MultiDict`. Either a + regular dict, an iterable of ``(key, value)`` tuples + or `None`. + """ + + def __init__(self, mapping=None): + if isinstance(mapping, MultiDict): + dict.__init__(self, ((k, l[:]) for k, l in mapping.iterlists())) + elif isinstance(mapping, dict): + tmp = {} + for key, value in mapping.iteritems(): + if isinstance(value, (tuple, list)): + value = list(value) + else: + value = [value] + tmp[key] = value + dict.__init__(self, tmp) + else: + tmp = {} + for key, value in mapping or (): + tmp.setdefault(key, []).append(value) + dict.__init__(self, tmp) + + def __getstate__(self): + return dict(self.lists()) + + def __setstate__(self, value): + dict.clear(self) + dict.update(self, value) + + def __iter__(self): + return self.iterkeys() + + def __getitem__(self, key): + """Return the first data value for this key; + raises KeyError if not found. + + :param key: The key to be looked up. + :raise KeyError: if the key does not exist. + """ + if key in self: + return dict.__getitem__(self, key)[0] + raise KeyError(key) + + def __setitem__(self, key, value): + """Like :meth:`add` but removes an existing key first. + + :param key: the key for the value. + :param value: the value to set. + """ + dict.__setitem__(self, key, [value]) + + def add(self, key, value): + """Adds a new value for the key. + + .. versionadded:: 0.6 + + :param key: the key for the value. + :param value: the value to add. + """ + dict.setdefault(self, key, []).append(value) + + def getlist(self, key, type=None): + """Return the list of items for a given key. If that key is not in the + `MultiDict`, the return value will be an empty list. Just as `get` + `getlist` accepts a `type` parameter. All items will be converted + with the callable defined there. + + :param key: The key to be looked up. + :param type: A callable that is used to cast the value in the + :class:`MultiDict`. If a :exc:`ValueError` is raised + by this callable the value will be removed from the list. + :return: a :class:`list` of all the values for the key. + """ + try: + rv = dict.__getitem__(self, key) + except KeyError: + return [] + if type is None: + return list(rv) + result = [] + for item in rv: + try: + result.append(type(item)) + except ValueError: + pass + return result + + def setlist(self, key, new_list): + """Remove the old values for a key and add new ones. Note that the list + you pass the values in will be shallow-copied before it is inserted in + the dictionary. + + >>> d = MultiDict() + >>> d.setlist('foo', ['1', '2']) + >>> d['foo'] + '1' + >>> d.getlist('foo') + ['1', '2'] + + :param key: The key for which the values are set. + :param new_list: An iterable with the new values for the key. Old values + are removed first. + """ + dict.__setitem__(self, key, list(new_list)) + + def setdefault(self, key, default=None): + """Returns the value for the key if it is in the dict, otherwise it + returns `default` and sets that value for `key`. + + :param key: The key to be looked up. + :param default: The default value to be returned if the key is not + in the dict. If not further specified it's `None`. + """ + if key not in self: + self[key] = default + else: + default = self[key] + return default + + def setlistdefault(self, key, default_list=None): + """Like `setdefault` but sets multiple values. The list returned + is not a copy, but the list that is actually used internally. This + means that you can put new values into the dict by appending items + to the list: + + >>> d = MultiDict({"foo": 1}) + >>> d.setlistdefault("foo").extend([2, 3]) + >>> d.getlist("foo") + [1, 2, 3] + + :param key: The key to be looked up. + :param default: An iterable of default values. It is either copied + (in case it was a list) or converted into a list + before returned. + :return: a :class:`list` + """ + if key not in self: + default_list = list(default_list or ()) + dict.__setitem__(self, key, default_list) + else: + default_list = dict.__getitem__(self, key) + return default_list + + def items(self, multi=False): + """Return a list of ``(key, value)`` pairs. + + :param multi: If set to `True` the list returned will have a + pair for each value of each key. Otherwise it + will only contain pairs for the first value of + each key. + + :return: a :class:`list` + """ + return list(self.iteritems(multi)) + + def lists(self): + """Return a list of ``(key, values)`` pairs, where values is the list of + all values associated with the key. + + :return: a :class:`list` + """ + return list(self.iterlists()) + + def values(self): + """Returns a list of the first value on every key's value list. + + :return: a :class:`list`. + """ + return [self[key] for key in self.iterkeys()] + + def listvalues(self): + """Return a list of all values associated with a key. Zipping + :meth:`keys` and this is the same as calling :meth:`lists`: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> zip(d.keys(), d.listvalues()) == d.lists() + True + + :return: a :class:`list` + """ + return list(self.iterlistvalues()) + + def iteritems(self, multi=False): + """Like :meth:`items` but returns an iterator.""" + for key, values in dict.iteritems(self): + if multi: + for value in values: + yield key, value + else: + yield key, values[0] + + def iterlists(self): + """Like :meth:`items` but returns an iterator.""" + for key, values in dict.iteritems(self): + yield key, list(values) + + def itervalues(self): + """Like :meth:`values` but returns an iterator.""" + for values in dict.itervalues(self): + yield values[0] + + def iterlistvalues(self): + """Like :meth:`listvalues` but returns an iterator.""" + return dict.itervalues(self) + + def copy(self): + """Return a shallow copy of this object.""" + return self.__class__(self) + + def to_dict(self, flat=True): + """Return the contents as regular dict. If `flat` is `True` the + returned dict will only have the first item present, if `flat` is + `False` all values will be returned as lists. + + :param flat: If set to `False` the dict returned will have lists + with all the values in it. Otherwise it will only + contain the first value for each key. + :return: a :class:`dict` + """ + if flat: + return dict(self.iteritems()) + return dict(self.lists()) + + def update(self, other_dict): + """update() extends rather than replaces existing key lists.""" + for key, value in iter_multi_items(other_dict): + MultiDict.add(self, key, value) + + def pop(self, key, default=_missing): + """Pop the first item for a list on the dict. Afterwards the + key is removed from the dict, so additional values are discarded: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> d.pop("foo") + 1 + >>> "foo" in d + False + + :param key: the key to pop. + :param default: if provided the value to return if the key was + not in the dictionary. + """ + try: + return dict.pop(self, key)[0] + except KeyError, e: + if default is not _missing: + return default + raise KeyError(str(e)) + + def popitem(self): + """Pop an item from the dict.""" + try: + item = dict.popitem(self) + return (item[0], item[1][0]) + except KeyError, e: + raise KeyError(str(e)) + + def poplist(self, key): + """Pop the list for a key from the dict. If the key is not in the dict + an empty list is returned. + + .. versionchanged:: 0.5 + If the key does no longer exist a list is returned instead of + raising an error. + """ + return dict.pop(self, key, []) + + def popitemlist(self): + """Pop a ``(key, list)`` tuple from the dict.""" + try: + return dict.popitem(self) + except KeyError, e: + raise KeyError(str(e)) + + def __copy__(self): + return self.copy() + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.items(multi=True)) diff --git a/libs/requests/packages/urllib3/__init__.py b/libs/requests/packages/urllib3/__init__.py new file mode 100644 index 00000000..20b1fb4e --- /dev/null +++ b/libs/requests/packages/urllib3/__init__.py @@ -0,0 +1,48 @@ +# urllib3/__init__.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" +urllib3 - Thread-safe connection pooling and re-using. +""" + +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '1.0.2' + + +from .connectionpool import ( + HTTPConnectionPool, + HTTPSConnectionPool, + connection_from_url, + get_host, + make_headers) + + +from .exceptions import ( + HTTPError, + MaxRetryError, + SSLError, + TimeoutError) + +from .poolmanager import PoolManager, ProxyManager, proxy_from_url +from .response import HTTPResponse +from .filepost import encode_multipart_formdata + + +# Set default logging handler to avoid "No handler found" warnings. +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + +# ... Clean up. +del logging +del NullHandler diff --git a/libs/requests/packages/urllib3/_collections.py b/libs/requests/packages/urllib3/_collections.py new file mode 100644 index 00000000..00b2cd58 --- /dev/null +++ b/libs/requests/packages/urllib3/_collections.py @@ -0,0 +1,131 @@ +# urllib3/_collections.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from collections import deque + +from threading import RLock + +__all__ = ['RecentlyUsedContainer'] + + +class AccessEntry(object): + __slots__ = ('key', 'is_valid') + + def __init__(self, key, is_valid=True): + self.key = key + self.is_valid = is_valid + + +class RecentlyUsedContainer(dict): + """ + Provides a dict-like that maintains up to ``maxsize`` keys while throwing + away the least-recently-used keys beyond ``maxsize``. + """ + + # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we + # will attempt to cleanup the invalidated entries in the access_log + # datastructure during the next 'get' operation. + CLEANUP_FACTOR = 10 + + def __init__(self, maxsize=10): + self._maxsize = maxsize + + self._container = {} + + # We use a deque to to store our keys ordered by the last access. + self.access_log = deque() + self.access_log_lock = RLock() + + # We look up the access log entry by the key to invalidate it so we can + # insert a new authorative entry at the head without having to dig and + # find the old entry for removal immediately. + self.access_lookup = {} + + # Trigger a heap cleanup when we get past this size + self.access_log_limit = maxsize * self.CLEANUP_FACTOR + + def _invalidate_entry(self, key): + "If exists: Invalidate old entry and return it." + old_entry = self.access_lookup.get(key) + if old_entry: + old_entry.is_valid = False + + return old_entry + + def _push_entry(self, key): + "Push entry onto our access log, invalidate the old entry if exists." + self._invalidate_entry(key) + + new_entry = AccessEntry(key) + self.access_lookup[key] = new_entry + + self.access_log_lock.acquire() + self.access_log.appendleft(new_entry) + self.access_log_lock.release() + + def _prune_entries(self, num): + "Pop entries from our access log until we popped ``num`` valid ones." + while num > 0: + self.access_log_lock.acquire() + p = self.access_log.pop() + self.access_log_lock.release() + + if not p.is_valid: + continue # Invalidated entry, skip + + dict.pop(self, p.key, None) + self.access_lookup.pop(p.key, None) + num -= 1 + + def _prune_invalidated_entries(self): + "Rebuild our access_log without the invalidated entries." + self.access_log_lock.acquire() + self.access_log = deque(e for e in self.access_log if e.is_valid) + self.access_log_lock.release() + + def _get_ordered_access_keys(self): + "Return ordered access keys for inspection. Used for testing." + self.access_log_lock.acquire() + r = [e.key for e in self.access_log if e.is_valid] + self.access_log_lock.release() + + return r + + def __getitem__(self, key): + item = dict.get(self, key) + + if not item: + raise KeyError(key) + + # Insert new entry with new high priority, also implicitly invalidates + # the old entry. + self._push_entry(key) + + if len(self.access_log) > self.access_log_limit: + # Heap is getting too big, try to clean up any tailing invalidated + # entries. + self._prune_invalidated_entries() + + return item + + def __setitem__(self, key, item): + # Add item to our container and access log + dict.__setitem__(self, key, item) + self._push_entry(key) + + # Discard invalid and excess entries + self._prune_entries(len(self) - self._maxsize) + + def __delitem__(self, key): + self._invalidate_entry(key) + self.access_lookup.pop(key, None) + dict.__delitem__(self, key) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default diff --git a/libs/requests/packages/urllib3/connectionpool.py b/libs/requests/packages/urllib3/connectionpool.py new file mode 100644 index 00000000..be9b7feb --- /dev/null +++ b/libs/requests/packages/urllib3/connectionpool.py @@ -0,0 +1,546 @@ +# urllib3/connectionpool.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import logging +import socket + + +from httplib import HTTPConnection, HTTPSConnection, HTTPException +from Queue import Queue, Empty, Full +from select import select +from socket import error as SocketError, timeout as SocketTimeout + + +try: + import ssl + BaseSSLError = ssl.SSLError +except ImportError: + ssl = None + BaseSSLError = None + + +from .request import RequestMethods +from .response import HTTPResponse +from .exceptions import ( + SSLError, + MaxRetryError, + TimeoutError, + HostChangedError, + EmptyPoolError, +) + + +log = logging.getLogger(__name__) + +_Default = object() + + +## Connection objects (extension of httplib) + +class VerifiedHTTPSConnection(HTTPSConnection): + """ + Based on httplib.HTTPSConnection but wraps the socket with + SSL certification. + """ + cert_reqs = None + ca_certs = None + + def set_cert(self, key_file=None, cert_file=None, + cert_reqs='CERT_NONE', ca_certs=None): + ssl_req_scheme = { + 'CERT_NONE': ssl.CERT_NONE, + 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, + 'CERT_REQUIRED': ssl.CERT_REQUIRED + } + + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE + self.ca_certs = ca_certs + + def connect(self): + # Add certificate verification + sock = socket.create_connection((self.host, self.port), self.timeout) + + # Wrap socket using verification with the root certs in + # trusted_root_certs + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs) + + +## Pool objects + +class ConnectionPool(object): + """ + Base class for all connection pools, such as + :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + """ + pass + + +class HTTPConnectionPool(ConnectionPool, RequestMethods): + """ + Thread-safe connection pool for one host. + + :param host: + Host used for this HTTP Connection (e.g. "localhost"), passed into + :class:`httplib.HTTPConnection`. + + :param port: + Port used for this HTTP Connection (None is equivalent to 80), passed + into :class:`httplib.HTTPConnection`. + + :param strict: + Causes BadStatusLine to be raised if the status line can't be parsed + as a valid HTTP/1.0 or 1.1 status line, passed into + :class:`httplib.HTTPConnection`. + + :param timeout: + Socket timeout for each individual connection, can be a float. None + disables timeout. + + :param maxsize: + Number of connections to save that can be reused. More than 1 is useful + in multithreaded situations. If ``block`` is set to false, more + connections will be created but they will not be saved once they've + been used. + + :param block: + If set to True, no more than ``maxsize`` connections will be used at + a time. When no free connections are available, the call will block + until a connection has been released. This is a useful side effect for + particular multithreaded situations where one does not want to use more + than maxsize connections per host to prevent flooding. + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + """ + + scheme = 'http' + + def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, + block=False, headers=None): + self.host = host + self.port = port + self.strict = strict + self.timeout = timeout + self.pool = Queue(maxsize) + self.block = block + self.headers = headers or {} + + # Fill the queue up so that doing get() on it will block properly + for _ in xrange(maxsize): + self.pool.put(None) + + # These are mostly for testing and debugging purposes. + self.num_connections = 0 + self.num_requests = 0 + + def _new_conn(self): + """ + Return a fresh :class:`httplib.HTTPConnection`. + """ + self.num_connections += 1 + log.info("Starting new HTTP connection (%d): %s" % + (self.num_connections, self.host)) + return HTTPConnection(host=self.host, port=self.port) + + def _get_conn(self, timeout=None): + """ + Get a connection. Will return a pooled connection if one is available. + + If no connections are available and :prop:`.block` is ``False``, then a + fresh connection is returned. + + :param timeout: + Seconds to wait before giving up and raising + :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and + :prop:`.block` is ``True``. + """ + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + + # If this is a persistent connection, check if it got disconnected + if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]: + # Either data is buffered (bad), or the connection is dropped. + log.info("Resetting dropped connection: %s" % self.host) + conn.close() + + except Empty: + if self.block: + raise EmptyPoolError("Pool reached maximum size and no more " + "connections are allowed.") + pass # Oh well, we'll create a new connection then + + return conn or self._new_conn() + + def _put_conn(self, conn): + """ + Put a connection back into the pool. + + :param conn: + Connection object for the current host and port as returned by + :meth:`._new_conn` or :meth:`._get_conn`. + + If the pool is already full, the connection is discarded because we + exceeded maxsize. If connections are discarded frequently, then maxsize + should be increased. + """ + try: + self.pool.put(conn, block=False) + except Full: + # This should never happen if self.block == True + log.warning("HttpConnectionPool is full, discarding connection: %s" + % self.host) + + def _make_request(self, conn, method, url, timeout=_Default, + **httplib_request_kw): + """ + Perform a request on a given httplib connection object taken from our + pool. + """ + self.num_requests += 1 + + if timeout is _Default: + timeout = self.timeout + + conn.timeout = timeout # This only does anything in Py26+ + + conn.request(method, url, **httplib_request_kw) + conn.sock.settimeout(timeout) + httplib_response = conn.getresponse() + + log.debug("\"%s %s %s\" %s %s" % + (method, url, + conn._http_vsn_str, # pylint: disable-msg=W0212 + httplib_response.status, httplib_response.length)) + + return httplib_response + + + def is_same_host(self, url): + """ + Check if the given ``url`` is a member of the same host as this + conncetion pool. + """ + # TODO: Add optional support for socket.gethostbyname checking. + return (url.startswith('/') or + get_host(url) == (self.scheme, self.host, self.port)) + + def urlopen(self, method, url, body=None, headers=None, retries=3, + redirect=True, assert_same_host=True, timeout=_Default, + pool_timeout=None, release_conn=None, **response_kw): + """ + Get a connection from the pool and perform an HTTP request. This is the + lowest level call for making a request, so you'll need to specify all + the raw details. + + .. note:: + + More commonly, it's appropriate to use a convenience method provided + by :class:`.RequestMethods`, such as :meth:`.request`. + + .. note:: + + `release_conn` will only behave as expected if + `preload_content=False` because we want to make + `preload_content=False` the default behaviour someday soon without + breaking backwards compatibility. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param body: + Data to send in the request body (useful for creating + POST requests, see HTTPConnectionPool.post_url for + more convenience). + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param retries: + Number of retries to allow before raising a MaxRetryError exception. + + :param redirect: + Automatically handle redirects (status codes 301, 302, 303, 307), + each redirect counts as a retry. + + :param assert_same_host: + If ``True``, will make sure that the host of the pool requests is + consistent else will raise HostChangedError. When False, you can + use the pool on an HTTP proxy and request foreign hosts. + + :param timeout: + If specified, overrides the default timeout for this one request. + + :param pool_timeout: + If set and the pool is set to block=True, then this method will + block for ``pool_timeout`` seconds and raise EmptyPoolError if no + connection is available within the time period. + + :param release_conn: + If False, then the urlopen call will not release the connection + back into the pool once a response is received (but will release if + you read the entire contents of the response such as when + `preload_content=True`). This is useful if you're not preloading + the response's content immediately. You will need to call + ``r.release_conn()`` on the response ``r`` to return the connection + back into the pool. If None, it takes the value of + ``response_kw.get('preload_content', True)``. + + :param \**response_kw: + Additional parameters are passed to + :meth:`urllib3.response.HTTPResponse.from_httplib` + """ + if headers is None: + headers = self.headers + + if retries < 0: + raise MaxRetryError("Max retries exceeded for url: %s" % url) + + if timeout is _Default: + timeout = self.timeout + + if release_conn is None: + release_conn = response_kw.get('preload_content', True) + + # Check host + if assert_same_host and not self.is_same_host(url): + host = "%s://%s" % (self.scheme, self.host) + if self.port: + host = "%s:%d" % (host, self.port) + + raise HostChangedError("Connection pool with host '%s' tried to " + "open a foreign host: %s" % (host, url)) + + conn = None + + try: + # Request a connection from the queue + # (Could raise SocketError: Bad file descriptor) + conn = self._get_conn(timeout=pool_timeout) + + # Make the request on the httplib connection object + httplib_response = self._make_request(conn, method, url, + timeout=timeout, + body=body, headers=headers) + + # If we're going to release the connection in ``finally:``, then + # the request doesn't need to know about the connection. Otherwise + # it will also try to release it and we'll have a double-release + # mess. + response_conn = not release_conn and conn + + # Import httplib's response into our own wrapper object + response = HTTPResponse.from_httplib(httplib_response, + pool=self, + connection=response_conn, + **response_kw) + + # else: + # The connection will be put back into the pool when + # ``response.release_conn()`` is called (implicitly by + # ``response.read()``) + + except (Empty), e: + # Timed out by queue + raise TimeoutError("Request timed out. (pool_timeout=%s)" % + pool_timeout) + + except (SocketTimeout), e: + # Timed out by socket + raise TimeoutError("Request timed out. (timeout=%s)" % + timeout) + + except (BaseSSLError), e: + # SSL certificate error + raise SSLError(e) + + except (HTTPException, SocketError), e: + # Connection broken, discard. It will be replaced next _get_conn(). + conn = None + + finally: + if conn and release_conn: + # Put the connection back to be reused + self._put_conn(conn) + + if not conn: + log.warn("Retrying (%d attempts remain) after connection " + "broken by '%r': %s" % (retries, e, url)) + return self.urlopen(method, url, body, headers, retries - 1, + redirect, assert_same_host) # Try again + + # Handle redirection + if (redirect and + response.status in [301, 302, 303, 307] and + 'location' in response.headers): # Redirect, retry + log.info("Redirecting %s -> %s" % + (url, response.headers.get('location'))) + return self.urlopen(method, response.headers.get('location'), body, + headers, retries - 1, redirect, + assert_same_host) + + return response + + +class HTTPSConnectionPool(HTTPConnectionPool): + """ + Same as :class:`.HTTPConnectionPool`, but HTTPS. + + When Python is compiled with the :mod:`ssl` module, then + :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, + instead of :class:httplib.HTTPSConnection`. + + The ``key_file``, ``cert_file``, ``cert_reqs``, and ``ca_certs`` parameters + are only used if :mod:`ssl` is available and are fed into + :meth:`ssl.wrap_socket` to upgrade the connection socket into an SSL socket. + """ + + scheme = 'https' + + def __init__(self, host, port=None, + strict=False, timeout=None, maxsize=1, + block=False, headers=None, + key_file=None, cert_file=None, + cert_reqs='CERT_NONE', ca_certs=None): + + super(HTTPSConnectionPool, self).__init__(host, port, + strict, timeout, maxsize, + block, headers) + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = cert_reqs + self.ca_certs = ca_certs + + def _new_conn(self): + """ + Return a fresh :class:`httplib.HTTPSConnection`. + """ + self.num_connections += 1 + log.info("Starting new HTTPS connection (%d): %s" + % (self.num_connections, self.host)) + + if not ssl: + return HTTPSConnection(host=self.host, port=self.port) + + connection = VerifiedHTTPSConnection(host=self.host, port=self.port) + connection.set_cert(key_file=self.key_file, cert_file=self.cert_file, + cert_reqs=self.cert_reqs, ca_certs=self.ca_certs) + return connection + + +## Helpers + +def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, + basic_auth=None): + """ + Shortcuts for generating request headers. + + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. + + :param accept_encoding: + Can be a boolean, list, or string. + ``True`` translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + :param user_agent: + String representing the user-agent you want, such as + "python-urllib3/0.6" + + :param basic_auth: + Colon-separated username:password string for 'authorization: basic ...' + auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = 'gzip,deflate' + headers['accept-encoding'] = accept_encoding + + if user_agent: + headers['user-agent'] = user_agent + + if keep_alive: + headers['connection'] = 'keep-alive' + + if basic_auth: + headers['authorization'] = 'Basic ' + \ + basic_auth.encode('base64').strip() + + return headers + + +def get_host(url): + """ + Given a url, return its scheme, host and port (None if it's not there). + + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) + """ + # This code is actually similar to urlparse.urlsplit, but much + # simplified for our needs. + port = None + scheme = 'http' + if '://' in url: + scheme, url = url.split('://', 1) + if '/' in url: + url, _path = url.split('/', 1) + if '@' in url: + _auth, url = url.split('@', 1) + if ':' in url: + url, port = url.split(':', 1) + port = int(port) + return scheme, url, port + + +def connection_from_url(url, **kw): + """ + Given a url, return an :class:`.ConnectionPool` instance of its host. + + This is a shortcut for not having to parse out the scheme, host, and port + of the url before creating an :class:`.ConnectionPool` instance. + + :param url: + Absolute URL string that must include the scheme. Port is optional. + + :param \**kw: + Passes additional parameters to the constructor of the appropriate + :class:`.ConnectionPool`. Useful for specifying things like + timeout, maxsize, headers, etc. + + Example: :: + + >>> conn = connection_from_url('http://google.com/') + >>> r = conn.request('GET', '/') + """ + scheme, host, port = get_host(url) + if scheme == 'https': + return HTTPSConnectionPool(host, port=port, **kw) + else: + return HTTPConnectionPool(host, port=port, **kw) diff --git a/libs/requests/packages/urllib3/exceptions.py b/libs/requests/packages/urllib3/exceptions.py new file mode 100644 index 00000000..69f459bd --- /dev/null +++ b/libs/requests/packages/urllib3/exceptions.py @@ -0,0 +1,35 @@ +# urllib3/exceptions.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +## Exceptions + +class HTTPError(Exception): + "Base exception used by this module." + pass + + +class SSLError(Exception): + "Raised when SSL certificate fails in an HTTPS connection." + pass + + +class MaxRetryError(HTTPError): + "Raised when the maximum number of retries is exceeded." + pass + + +class TimeoutError(HTTPError): + "Raised when a socket timeout occurs." + pass + + +class HostChangedError(HTTPError): + "Raised when an existing pool gets a request for a foreign host." + pass + +class EmptyPoolError(HTTPError): + "Raised when a pool runs out of connections and no more are allowed." + pass diff --git a/libs/requests/packages/urllib3/filepost.py b/libs/requests/packages/urllib3/filepost.py new file mode 100644 index 00000000..2ffea8bb --- /dev/null +++ b/libs/requests/packages/urllib3/filepost.py @@ -0,0 +1,71 @@ +# urllib3/filepost.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import codecs +import mimetools +import mimetypes + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO # pylint: disable-msg=W0404 + + +writer = codecs.lookup('utf-8')[3] + + +def get_content_type(filename): + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + + +def encode_multipart_formdata(fields, boundary=None): + """ + Encode a dictionary of ``fields`` using the multipart/form-data mime format. + + :param fields: + Dictionary of fields. The key is treated as the field name, and the + value as the body of the form-data. If the value is a tuple of two + elements, then the first element is treated as the filename of the + form-data section. + + :param boundary: + If not specified, then a random boundary will be generated using + :func:`mimetools.choose_boundary`. + """ + body = StringIO() + if boundary is None: + boundary = mimetools.choose_boundary() + + for fieldname, value in fields.iteritems(): + body.write('--%s\r\n' % (boundary)) + + if isinstance(value, tuple): + filename, data = value + writer(body).write('Content-Disposition: form-data; name="%s"; ' + 'filename="%s"\r\n' % (fieldname, filename)) + body.write('Content-Type: %s\r\n\r\n' % + (get_content_type(filename))) + else: + data = value + writer(body).write('Content-Disposition: form-data; name="%s"\r\n' + % (fieldname)) + body.write('Content-Type: text/plain\r\n\r\n') + + if isinstance(data, int): + data = str(data) # Backwards compatibility + + if isinstance(data, unicode): + writer(body).write(data) + else: + body.write(data) + + body.write('\r\n') + + body.write('--%s--\r\n' % (boundary)) + + content_type = 'multipart/form-data; boundary=%s' % boundary + + return body.getvalue(), content_type diff --git a/libs/requests/packages/urllib3/poolmanager.py b/libs/requests/packages/urllib3/poolmanager.py new file mode 100644 index 00000000..c08e327f --- /dev/null +++ b/libs/requests/packages/urllib3/poolmanager.py @@ -0,0 +1,128 @@ +# urllib3/poolmanager.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from ._collections import RecentlyUsedContainer +from .connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool, + get_host, connection_from_url, +) + + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] + + +from .request import RequestMethods +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool + + +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + +port_by_scheme = { + 'http': 80, + 'https': 443, +} + + +class PoolManager(RequestMethods): + """ + Allows for arbitrary requests while transparently keeping track of + necessary connection pools for you. + + :param num_pools: + Number of connection pools to cache before discarding the least recently + used pool. + + :param \**connection_pool_kw: + Additional parameters are used to create fresh + :class:`urllib3.connectionpool.ConnectionPool` instances. + + Example: :: + + >>> manager = PoolManager() + >>> r = manager.urlopen("http://google.com/") + >>> r = manager.urlopen("http://google.com/mail") + >>> r = manager.urlopen("http://yahoo.com/") + >>> len(r.pools) + 2 + + """ + + # TODO: Make sure there are no memory leaks here. + + def __init__(self, num_pools=10, **connection_pool_kw): + self.connection_pool_kw = connection_pool_kw + self.pools = RecentlyUsedContainer(num_pools) + + def connection_from_host(self, host, port=80, scheme='http'): + """ + Get a :class:`ConnectionPool` based on the host, port, and scheme. + + Note that an appropriate ``port`` value is required here to normalize + connection pools in our container most effectively. + """ + pool_key = (scheme, host, port) + + # If the scheme, host, or port doesn't match existing open connections, + # open a new ConnectionPool. + pool = self.pools.get(pool_key) + if pool: + return pool + + # Make a fresh ConnectionPool of the desired type + pool_cls = pool_classes_by_scheme[scheme] + pool = pool_cls(host, port, **self.connection_pool_kw) + + self.pools[pool_key] = pool + + return pool + + def connection_from_url(self, url): + """ + Similar to :func:`urllib3.connectionpool.connection_from_url` but + doesn't pass any additional parameters to the + :class:`urllib3.connectionpool.ConnectionPool` constructor. + + Additional parameters are taken from the :class:`.PoolManager` + constructor. + """ + scheme, host, port = get_host(url) + + port = port or port_by_scheme.get(scheme, 80) + + return self.connection_from_host(host, port=port, scheme=scheme) + + def urlopen(self, method, url, **kw): + """ + Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`. + + ``url`` must be absolute, such that an appropriate + :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. + """ + conn = self.connection_from_url(url) + return conn.urlopen(method, url, assert_same_host=False, **kw) + + +class ProxyManager(RequestMethods): + """ + Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method + will make requests to any url through the defined proxy. + """ + + def __init__(self, proxy_pool): + self.proxy_pool = proxy_pool + + def urlopen(self, method, url, **kw): + "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." + kw['assert_same_host'] = False + return self.proxy_pool.urlopen(method, url, **kw) + + +def proxy_from_url(url, **pool_kw): + proxy_pool = connection_from_url(url, **pool_kw) + return ProxyManager(proxy_pool) diff --git a/libs/requests/packages/urllib3/request.py b/libs/requests/packages/urllib3/request.py new file mode 100644 index 00000000..a7e0b5de --- /dev/null +++ b/libs/requests/packages/urllib3/request.py @@ -0,0 +1,145 @@ +# urllib3/request.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from urllib import urlencode + +from .filepost import encode_multipart_formdata + + +__all__ = ['RequestMethods'] + + +class RequestMethods(object): + """ + Convenience mixin for classes who implement a :meth:`urlopen` method, such + as :class:`~urllib3.connectionpool.HTTPConnectionPool` and + :class:`~urllib3.poolmanager.PoolManager`. + + Provides behavior for making common types of HTTP request methods and + decides which type of request field encoding to use. + + Specifically, + + :meth:`.request_encode_url` is for sending requests whose fields are encoded + in the URL (such as GET, HEAD, DELETE). + + :meth:`.request_encode_body` is for sending requests whose fields are + encoded in the *body* of the request using multipart or www-orm-urlencoded + (such as for POST, PUT, PATCH). + + :meth:`.request` is for making any kind of request, it will look up the + appropriate encoding format and use one of the above two methods to make + the request. + """ + + _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + + _encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE']) + + def urlopen(self, method, url, body=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **kw): + raise NotImplemented("Classes extending RequestMethods must implement " + "their own ``urlopen`` method.") + + def request(self, method, url, fields=None, headers=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the appropriate encoding of + ``fields`` based on the ``method`` used. + + This is a convenience method that requires the least amount of manual + effort. It can be used in most situations, while still having the option + to drop down to more specific methods when necessary, such as + :meth:`request_encode_url`, :meth:`request_encode_body`, + or even the lowest level :meth:`urlopen`. + """ + method = method.upper() + + if method in self._encode_url_methods: + return self.request_encode_url(method, url, fields=fields, + headers=headers, + **urlopen_kw) + else: + return self.request_encode_body(method, url, fields=fields, + headers=headers, + **urlopen_kw) + + def request_encode_url(self, method, url, fields=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the url. This is useful for request methods like GET, HEAD, DELETE, etc. + """ + if fields: + url += '?' + urlencode(fields) + return self.urlopen(method, url, **urlopen_kw) + + def request_encode_body(self, method, url, fields=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the body. This is useful for request methods like POST, PUT, PATCH, etc. + + When ``encode_multipart=True`` (default), then + :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the + payload with the appropriate content type. Otherwise + :meth:`urllib.urlencode` is used with the + 'application/x-www-form-urlencoded' content type. + + Multipart encoding must be used when posting files, and it's reasonably + safe to use it in other times too. However, it may break request signing, + such as with OAuth. + + Supports an optional ``fields`` parameter of key/value strings AND + key/filetuple. A filetuple is a (filename, data) tuple. For example: :: + + fields = { + 'foo': 'bar', + 'fakefile': ('foofile.txt', 'contents of foofile'), + 'realfile': ('barfile.txt', open('realfile').read()), + 'nonamefile': ('contents of nonamefile field'), + } + + When uploading a file, providing a filename (the first parameter of the + tuple) is optional but recommended to best mimick behavior of browsers. + + Note that if ``headers`` are supplied, the 'Content-Type' header will be + overwritten because it depends on the dynamic random boundary string + which is used to compose the body of the request. The random boundary + string can be explicitly set with the ``multipart_boundary`` parameter. + """ + if encode_multipart: + body, content_type = encode_multipart_formdata(fields or {}, + boundary=multipart_boundary) + else: + body, content_type = (urlencode(fields or {}), + 'application/x-www-form-urlencoded') + + headers = headers or {} + headers.update({'Content-Type': content_type}) + + return self.urlopen(method, url, body=body, headers=headers, + **urlopen_kw) + + # Deprecated: + + def get_url(self, url, fields=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_url('GET', url, fields=fields, + **urlopen_kw) + + def post_url(self, url, fields=None, headers=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_body('POST', url, fields=fields, + headers=headers, + **urlopen_kw) diff --git a/libs/requests/packages/urllib3/response.py b/libs/requests/packages/urllib3/response.py new file mode 100644 index 00000000..4cd15c11 --- /dev/null +++ b/libs/requests/packages/urllib3/response.py @@ -0,0 +1,181 @@ +# urllib3/response.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import gzip +import logging +import zlib + + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO # pylint: disable-msg=W0404 + + +from .exceptions import HTTPError + + +log = logging.getLogger(__name__) + + +def decode_gzip(data): + gzipper = gzip.GzipFile(fileobj=StringIO(data)) + return gzipper.read() + + +def decode_deflate(data): + try: + return zlib.decompress(data) + except zlib.error: + return zlib.decompress(data, -zlib.MAX_WBITS) + + +class HTTPResponse(object): + """ + HTTP Response container. + + Backwards-compatible to httplib's HTTPResponse but the response ``body`` is + loaded and decoded on-demand when the ``data`` property is accessed. + + Extra parameters for behaviour not present in httplib.HTTPResponse: + + :param preload_content: + If True, the response's body will be preloaded during construction. + + :param decode_content: + If True, attempts to decode specific content-encoding's based on headers + (like 'gzip' and 'deflate') will be skipped and raw data will be used + instead. + + :param original_response: + When this HTTPResponse wrapper is generated from an httplib.HTTPResponse + object, it's convenient to include the original for debug purposes. It's + otherwise unused. + """ + + CONTENT_DECODERS = { + 'gzip': decode_gzip, + 'deflate': decode_deflate, + } + + def __init__(self, body='', headers=None, status=0, version=0, reason=None, + strict=0, preload_content=True, decode_content=True, + original_response=None, pool=None, connection=None): + self.headers = headers or {} + self.status = status + self.version = version + self.reason = reason + self.strict = strict + + self._decode_content = decode_content + self._body = None + self._fp = None + self._original_response = original_response + + self._pool = pool + self._connection = connection + + if hasattr(body, 'read'): + self._fp = body + + if preload_content: + self._body = self.read(decode_content=decode_content) + + def release_conn(self): + if not self._pool or not self._connection: + return + + self._pool._put_conn(self._connection) + self._connection = None + + @property + def data(self): + # For backwords-compat with earlier urllib3 0.4 and earlier. + if self._body: + return self._body + + if self._fp: + return self.read(decode_content=self._decode_content, + cache_content=True) + + def read(self, amt=None, decode_content=True, cache_content=False): + """ + Similar to :meth:`httplib.HTTPResponse.read`, but with two additional + parameters: ``decode_content`` and ``cache_content``. + + :param amt: + How much of the content to read. If specified, decoding and caching + is skipped because we can't decode partial content nor does it make + sense to cache partial content as the full response. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. (Overridden if ``amt`` is set.) + + :param cache_content: + If True, will save the returned data such that the same result is + returned despite of the state of the underlying file object. This + is useful if you want the ``.data`` property to continue working + after having ``.read()`` the file object. (Overridden if ``amt`` is + set.) + """ + content_encoding = self.headers.get('content-encoding') + decoder = self.CONTENT_DECODERS.get(content_encoding) + + data = self._fp and self._fp.read(amt) + + try: + + if amt: + return data + + if not decode_content or not decoder: + if cache_content: + self._body = data + + return data + + try: + data = decoder(data) + except IOError: + raise HTTPError("Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding) + + if cache_content: + self._body = data + + return data + + finally: + + if self._original_response and self._original_response.isclosed(): + self.release_conn() + + @staticmethod + def from_httplib(r, **response_kw): + """ + Given an :class:`httplib.HTTPResponse` instance ``r``, return a + corresponding :class:`urllib3.response.HTTPResponse` object. + + Remaining parameters are passed to the HTTPResponse constructor, along + with ``original_response=r``. + """ + + return HTTPResponse(body=r, + headers=dict(r.getheaders()), + status=r.status, + version=r.version, + reason=r.reason, + strict=r.strict, + original_response=r, + **response_kw) + + # Backwards-compatibility methods for httplib.HTTPResponse + def getheaders(self): + return self.headers + + def getheader(self, name, default=None): + return self.headers.get(name, default) diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py new file mode 100644 index 00000000..20d22613 --- /dev/null +++ b/libs/requests/sessions.py @@ -0,0 +1,279 @@ +# -*- coding: utf-8 -*- + +""" +requests.session +~~~~~~~~~~~~~~~~ + +This module provides a Session object to manage and persist settings across +requests (cookies, auth, proxies). + +""" + +from .defaults import defaults +from .models import Request +from .hooks import dispatch_hook +from .utils import header_expand +from .packages.urllib3.poolmanager import PoolManager + + +def merge_kwargs(local_kwarg, default_kwarg): + """Merges kwarg dictionaries. + + If a local key in the dictionary is set to None, it will be removed. + """ + + if default_kwarg is None: + return local_kwarg + + if isinstance(local_kwarg, basestring): + return local_kwarg + + if local_kwarg is None: + return default_kwarg + + # Bypass if not a dictionary (e.g. timeout) + if not hasattr(default_kwarg, 'items'): + return local_kwarg + + # Update new values. + kwargs = default_kwarg.copy() + kwargs.update(local_kwarg) + + # Remove keys that are set to None. + for (k,v) in local_kwarg.items(): + if v is None: + del kwargs[k] + + return kwargs + + +class Session(object): + """A Requests session.""" + + __attrs__ = [ + 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', + 'params', 'config'] + + + def __init__(self, + headers=None, + cookies=None, + auth=None, + timeout=None, + proxies=None, + hooks=None, + params=None, + config=None): + + self.headers = headers or {} + self.cookies = cookies or {} + self.auth = auth + self.timeout = timeout + self.proxies = proxies or {} + self.hooks = hooks or {} + self.params = params or {} + self.config = config or {} + + for (k, v) in defaults.items(): + self.config.setdefault(k, v) + + self.poolmanager = PoolManager( + num_pools=self.config.get('pool_connections'), + maxsize=self.config.get('pool_maxsize') + ) + + # Set up a CookieJar to be used by default + self.cookies = {} + + # Add passed cookies in. + if cookies is not None: + self.cookies.update(cookies) + + def __repr__(self): + return '' % (id(self)) + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def request(self, method, url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=False, + proxies=None, + hooks=None, + return_response=True, + config=None, + prefetch=False): + + """Constructs and sends a :class:`Request `. + Returns :class:`Response ` object. + + :param method: method for the new :class:`Request` object. + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param return_response: (optional) If False, an un-sent Request object will returned. + :param config: (optional) A configuration dictionary. + :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. + """ + + method = str(method).upper() + + # Default empty dicts for dict params. + cookies = {} if cookies is None else cookies + data = {} if data is None else data + files = {} if files is None else files + headers = {} if headers is None else headers + params = {} if params is None else params + hooks = {} if hooks is None else hooks + # use session's hooks as defaults + for key, cb in self.hooks.iteritems(): + hooks.setdefault(key, cb) + + # Expand header values. + if headers: + for k, v in headers.items() or {}: + headers[k] = header_expand(v) + + args = dict( + method=method, + url=url, + data=data, + params=params, + headers=headers, + cookies=cookies, + files=files, + auth=auth, + hooks=hooks, + timeout=timeout, + allow_redirects=allow_redirects, + proxies=proxies, + config=config, + _poolmanager=self.poolmanager + ) + + # Merge local kwargs with session kwargs. + for attr in self.__attrs__: + session_val = getattr(self, attr, None) + local_val = args.get(attr) + + args[attr] = merge_kwargs(local_val, session_val) + + # Arguments manipulation hook. + args = dispatch_hook('args', args['hooks'], args) + + # Create the (empty) response. + r = Request(**args) + + # Give the response some context. + r.session = self + + # Don't send if asked nicely. + if not return_response: + return r + + # Send the HTTP Request. + r.send(prefetch=prefetch) + + # Send any cookies back up the to the session. + self.cookies.update(r.response.cookies) + + # Return the response. + return r.response + + + def get(self, url, **kwargs): + """Sends a GET request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return self.request('get', url, **kwargs) + + + def options(self, url, **kwargs): + """Sends a OPTIONS request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return self.request('options', url, **kwargs) + + + def head(self, url, **kwargs): + """Sends a HEAD request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return self.request('head', url, **kwargs) + + + def post(self, url, data=None, **kwargs): + """Sends a POST request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return self.request('post', url, data=data, **kwargs) + + + def put(self, url, data=None, **kwargs): + """Sends a PUT request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return self.request('put', url, data=data, **kwargs) + + + def patch(self, url, data=None, **kwargs): + """Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return self.request('patch', url, data=data, **kwargs) + + + def delete(self, url, **kwargs): + """Sends a DELETE request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + return self.request('delete', url, **kwargs) + + +def session(**kwargs): + """Returns a :class:`Session` for context-management.""" + + return Session(**kwargs) diff --git a/libs/requests/status_codes.py b/libs/requests/status_codes.py new file mode 100644 index 00000000..fab8e957 --- /dev/null +++ b/libs/requests/status_codes.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +from .structures import LookupDict + +_codes = { + + # Informational. + 100: ('continue',), + 101: ('switching_protocols',), + 102: ('processing',), + 103: ('checkpoint',), + 122: ('uri_too_long', 'request_uri_too_long'), + 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/'), + 201: ('created',), + 202: ('accepted',), + 203: ('non_authoritative_info', 'non_authoritative_information'), + 204: ('no_content',), + 205: ('reset_content', 'reset'), + 206: ('partial_content', 'partial'), + 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), + 208: ('im_used',), + + # Redirection. + 300: ('multiple_choices',), + 301: ('moved_permanently', 'moved', '\\o-'), + 302: ('found',), + 303: ('see_other', 'other'), + 304: ('not_modified',), + 305: ('use_proxy',), + 306: ('switch_proxy',), + 307: ('temporary_redirect', 'temporary_moved', 'temporary'), + 308: ('resume_incomplete', 'resume'), + + # Client Error. + 400: ('bad_request', 'bad'), + 401: ('unauthorized',), + 402: ('payment_required', 'payment'), + 403: ('forbidden',), + 404: ('not_found', '-o-'), + 405: ('method_not_allowed', 'not_allowed'), + 406: ('not_acceptable',), + 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), + 408: ('request_timeout', 'timeout'), + 409: ('conflict',), + 410: ('gone',), + 411: ('length_required',), + 412: ('precondition_failed', 'precondition'), + 413: ('request_entity_too_large',), + 414: ('request_uri_too_large',), + 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), + 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), + 417: ('expectation_failed',), + 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), + 422: ('unprocessable_entity', 'unprocessable'), + 423: ('locked',), + 424: ('failed_dependency', 'dependency'), + 425: ('unordered_collection', 'unordered'), + 426: ('upgrade_required', 'upgrade'), + 428: ('precondition_required', 'precondition'), + 429: ('too_many_requests', 'too_many'), + 431: ('header_fields_too_large', 'fields_too_large'), + 444: ('no_response', 'none'), + 449: ('retry_with', 'retry'), + 450: ('blocked_by_windows_parental_controls', 'parental_controls'), + 499: ('client_closed_request',), + + # Server Error. + 500: ('internal_server_error', 'server_error', '/o\\'), + 501: ('not_implemented',), + 502: ('bad_gateway',), + 503: ('service_unavailable', 'unavailable'), + 504: ('gateway_timeout',), + 505: ('http_version_not_supported', 'http_version'), + 506: ('variant_also_negotiates',), + 507: ('insufficient_storage',), + 509: ('bandwidth_limit_exceeded', 'bandwidth'), + 510: ('not_extended',), +} + +codes = LookupDict(name='status_codes') + +for (code, titles) in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith('\\'): + setattr(codes, title.upper(), code) \ No newline at end of file diff --git a/libs/requests/structures.py b/libs/requests/structures.py new file mode 100644 index 00000000..35a903fd --- /dev/null +++ b/libs/requests/structures.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +""" +requests.structures +~~~~~~~~~~~~~~~~~~~ + +Data structures that power Requests. + +""" + + +class CaseInsensitiveDict(dict): + """Case-insensitive Dictionary + + For example, ``headers['content-encoding']`` will return the + value of a ``'Content-Encoding'`` response header.""" + + @property + def lower_keys(self): + if not hasattr(self, '_lower_keys') or not self._lower_keys: + self._lower_keys = dict((k.lower(), k) for k in self.iterkeys()) + return self._lower_keys + + def _clear_lower_keys(self): + if hasattr(self, '_lower_keys'): + self._lower_keys.clear() + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + self._clear_lower_keys() + + def __delitem__(self, key): + dict.__delitem__(self, key) + self._lower_keys.clear() + + def __contains__(self, key): + return key.lower() in self.lower_keys + + def __getitem__(self, key): + # We allow fall-through here, so values default to None + if key in self: + return dict.__getitem__(self, self.lower_keys[key.lower()]) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + +class LookupDict(dict): + """Dictionary lookup object.""" + + def __init__(self, name=None): + self.name = name + super(LookupDict, self).__init__() + + def __repr__(self): + return '' % (self.name) + + def __getitem__(self, key): + # We allow fall-through here, so values default to None + + return self.__dict__.get(key, None) + + def get(self, key, default=None): + return self.__dict__.get(key, default) \ No newline at end of file diff --git a/libs/requests/utils.py b/libs/requests/utils.py new file mode 100644 index 00000000..6d7b8686 --- /dev/null +++ b/libs/requests/utils.py @@ -0,0 +1,381 @@ +# -*- coding: utf-8 -*- + +""" +requests.utils +~~~~~~~~~~~~~~ + +This module provides utility functions that are used within Requests +that are also useful for external consumption. + +""" + +import cgi +import codecs +import cookielib +import os +import random +import re +import zlib +import urllib + +from urllib2 import parse_http_list as _parse_list_header + + +def guess_filename(obj): + """Tries to guess the filename of the given object.""" + name = getattr(obj, 'name', None) + if name and name[0] != '<' and name[-1] != '>': + return name + +# From mitsuhiko/werkzeug (used with permission). +def parse_list_header(value): + """Parse lists as described by RFC 2068 Section 2. + + In particular, parse comma-separated lists where the elements of + the list may include quoted-strings. A quoted-string could + contain a comma. A non-quoted string could have quotes in the + middle. Quotes are removed automatically after parsing. + + It basically works like :func:`parse_set_header` just that items + may appear multiple times and case sensitivity is preserved. + + The return value is a standard :class:`list`: + + >>> parse_list_header('token, "quoted value"') + ['token', 'quoted value'] + + To create a header from the :class:`list` again, use the + :func:`dump_header` function. + + :param value: a string with a list header. + :return: :class:`list` + """ + result = [] + for item in _parse_list_header(value): + if item[:1] == item[-1:] == '"': + item = unquote_header_value(item[1:-1]) + result.append(item) + return result + + +# From mitsuhiko/werkzeug (used with permission). +def parse_dict_header(value): + """Parse lists of key, value pairs as described by RFC 2068 Section 2 and + convert them into a python dict: + + >>> d = parse_dict_header('foo="is a fish", bar="as well"') + >>> type(d) is dict + True + >>> sorted(d.items()) + [('bar', 'as well'), ('foo', 'is a fish')] + + If there is no value for a key it will be `None`: + + >>> parse_dict_header('key_without_value') + {'key_without_value': None} + + To create a header from the :class:`dict` again, use the + :func:`dump_header` function. + + :param value: a string with a dict header. + :return: :class:`dict` + """ + result = {} + for item in _parse_list_header(value): + if '=' not in item: + result[item] = None + continue + name, value = item.split('=', 1) + if value[:1] == value[-1:] == '"': + value = unquote_header_value(value[1:-1]) + result[name] = value + return result + + +# From mitsuhiko/werkzeug (used with permission). +def unquote_header_value(value, is_filename=False): + r"""Unquotes a header value. (Reversal of :func:`quote_header_value`). + This does not use the real unquoting but what browsers are actually + using for quoting. + + :param value: the header value to unquote. + """ + if value and value[0] == value[-1] == '"': + # this is not the real unquoting, but fixing this so that the + # RFC is met will result in bugs with internet explorer and + # probably some other browsers as well. IE for example is + # uploading files with "C:\foo\bar.txt" as filename + value = value[1:-1] + + # if this is a filename and the starting characters look like + # a UNC path, then just return the value without quotes. Using the + # replace sequence below on a UNC path has the effect of turning + # the leading double slash into a single slash and then + # _fix_ie_filename() doesn't work correctly. See #458. + if not is_filename or value[:2] != '\\\\': + return value.replace('\\\\', '\\').replace('\\"', '"') + return value + + +def header_expand(headers): + """Returns an HTTP Header value string from a dictionary. + + Example expansion:: + + {'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}} + # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c + + (('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {})) + # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c + """ + + collector = [] + + if isinstance(headers, dict): + headers = headers.items() + + elif isinstance(headers, basestring): + return headers + + for i, (value, params) in enumerate(headers): + + _params = [] + + for (p_k, p_v) in params.items(): + + _params.append('%s=%s' % (p_k, p_v)) + + collector.append(value) + collector.append('; ') + + if len(params): + + collector.append('; '.join(_params)) + + if not len(headers) == i+1: + collector.append(', ') + + + # Remove trailing separators. + if collector[-1] in (', ', '; '): + del collector[-1] + + return ''.join(collector) + + + +def randombytes(n): + """Return n random bytes.""" + # Use /dev/urandom if it is available. Fall back to random module + # if not. It might be worthwhile to extend this function to use + # other platform-specific mechanisms for getting random bytes. + if os.path.exists("/dev/urandom"): + f = open("/dev/urandom") + s = f.read(n) + f.close() + return s + else: + L = [chr(random.randrange(0, 256)) for i in range(n)] + return "".join(L) + + +def dict_from_cookiejar(cj): + """Returns a key/value dictionary from a CookieJar. + + :param cj: CookieJar object to extract cookies from. + """ + + cookie_dict = {} + + for _, cookies in cj._cookies.items(): + for _, cookies in cookies.items(): + for cookie in cookies.values(): + # print cookie + cookie_dict[cookie.name] = cookie.value + + return cookie_dict + + +def cookiejar_from_dict(cookie_dict): + """Returns a CookieJar from a key/value dictionary. + + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ + + # return cookiejar if one was passed in + if isinstance(cookie_dict, cookielib.CookieJar): + return cookie_dict + + # create cookiejar + cj = cookielib.CookieJar() + + cj = add_dict_to_cookiejar(cj, cookie_dict) + + return cj + + +def add_dict_to_cookiejar(cj, cookie_dict): + """Returns a CookieJar from a key/value dictionary. + + :param cj: CookieJar to insert cookies into. + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ + + for k, v in cookie_dict.items(): + + cookie = cookielib.Cookie( + version=0, + name=k, + value=v, + port=None, + port_specified=False, + domain='', + domain_specified=False, + domain_initial_dot=False, + path='/', + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False + ) + + # add cookie to cookiejar + cj.set_cookie(cookie) + + return cj + + +def get_encodings_from_content(content): + """Returns encodings from given content string. + + :param content: bytestring to extract encodings from. + """ + + charset_re = re.compile(r']', flags=re.I) + + return charset_re.findall(content) + + +def get_encoding_from_headers(headers): + """Returns encodings from given HTTP Header Dict. + + :param headers: dictionary to extract encoding from. + """ + + content_type = headers.get('content-type') + + if not content_type: + return None + + content_type, params = cgi.parse_header(content_type) + + if 'charset' in params: + return params['charset'].strip("'\"") + + +def unicode_from_html(content): + """Attempts to decode an HTML string into unicode. + If unsuccessful, the original content is returned. + """ + + encodings = get_encodings_from_content(content) + + for encoding in encodings: + + try: + return unicode(content, encoding) + except (UnicodeError, TypeError): + pass + + return content + + +def stream_decode_response_unicode(iterator, r): + """Stream decodes a iterator.""" + + if r.encoding is None: + for item in iterator: + yield item + return + + decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') + for chunk in iterator: + rv = decoder.decode(chunk) + if rv: + yield rv + rv = decoder.decode('', final=True) + if rv: + yield rv + + +def get_unicode_from_response(r): + """Returns the requested content back in unicode. + + :param r: Response object to get unicode content from. + + Tried: + + 1. charset from content-type + + 2. every encodings from ```` + + 3. fall back and replace all unicode characters + + """ + + tried_encodings = [] + + # Try charset from content-type + encoding = get_encoding_from_headers(r.headers) + + if encoding: + try: + return unicode(r.content, encoding) + except UnicodeError: + tried_encodings.append(encoding) + + # Fall back: + try: + return unicode(r.content, encoding, errors='replace') + except TypeError: + return r.content + + +def decode_gzip(content): + """Return gzip-decoded string. + + :param content: bytestring to gzip-decode. + """ + + return zlib.decompress(content, 16 + zlib.MAX_WBITS) + + +def stream_decode_gzip(iterator): + """Stream decodes a gzip-encoded iterator""" + try: + dec = zlib.decompressobj(16 + zlib.MAX_WBITS) + for chunk in iterator: + rv = dec.decompress(chunk) + if rv: + yield rv + buf = dec.decompress('') + rv = buf + dec.flush() + if rv: + yield rv + except zlib.error: + pass + + +def requote_path(path): + """Re-quote the given URL path component. + + This function passes the given path through an unquote/quote cycle to + ensure that it is fully and consistently quoted. + """ + parts = path.split("/") + parts = (urllib.quote(urllib.unquote(part), safe="") for part in parts) + return "/".join(parts) diff --git a/libs/suds/__init__.py b/libs/suds/__init__.py new file mode 100644 index 00000000..166a2065 --- /dev/null +++ b/libs/suds/__init__.py @@ -0,0 +1,154 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Suds is a lightweight SOAP python client that provides a +service proxy for Web Services. +""" + +import os +import sys + +# +# Project properties +# + +__version__ = '0.4' +__build__="GA R699-20100913" + +# +# Exceptions +# + +class MethodNotFound(Exception): + def __init__(self, name): + Exception.__init__(self, "Method not found: '%s'" % name) + +class PortNotFound(Exception): + def __init__(self, name): + Exception.__init__(self, "Port not found: '%s'" % name) + +class ServiceNotFound(Exception): + def __init__(self, name): + Exception.__init__(self, "Service not found: '%s'" % name) + +class TypeNotFound(Exception): + def __init__(self, name): + Exception.__init__(self, "Type not found: '%s'" % tostr(name)) + +class BuildError(Exception): + msg = \ + """ + An error occured while building a instance of (%s). As a result + the object you requested could not be constructed. It is recommended + that you construct the type manually using a Suds object. + Please open a ticket with a description of this error. + Reason: %s + """ + def __init__(self, name, exception): + Exception.__init__(self, BuildError.msg % (name, exception)) + +class SoapHeadersNotPermitted(Exception): + msg = \ + """ + Method (%s) was invoked with SOAP headers. The WSDL does not + define SOAP headers for this method. Retry without the soapheaders + keyword argument. + """ + def __init__(self, name): + Exception.__init__(self, self.msg % name) + +class WebFault(Exception): + def __init__(self, fault, document): + if hasattr(fault, 'faultstring'): + Exception.__init__(self, "Server raised fault: '%s'" % fault.faultstring) + self.fault = fault + self.document = document + +# +# Logging +# + +class Repr: + def __init__(self, x): + self.x = x + def __str__(self): + return repr(self.x) + +# +# Utility +# + +def tostr(object, encoding=None): + """ get a unicode safe string representation of an object """ + if isinstance(object, basestring): + if encoding is None: + return object + else: + return object.encode(encoding) + if isinstance(object, tuple): + s = ['('] + for item in object: + if isinstance(item, basestring): + s.append(item) + else: + s.append(tostr(item)) + s.append(', ') + s.append(')') + return ''.join(s) + if isinstance(object, list): + s = ['['] + for item in object: + if isinstance(item, basestring): + s.append(item) + else: + s.append(tostr(item)) + s.append(', ') + s.append(']') + return ''.join(s) + if isinstance(object, dict): + s = ['{'] + for item in object.items(): + if isinstance(item[0], basestring): + s.append(item[0]) + else: + s.append(tostr(item[0])) + s.append(' = ') + if isinstance(item[1], basestring): + s.append(item[1]) + else: + s.append(tostr(item[1])) + s.append(', ') + s.append('}') + return ''.join(s) + try: + return unicode(object) + except: + return str(object) + +class null: + """ + The I{null} object. + Used to pass NULL for optional XML nodes. + """ + pass + +def objid(obj): + return obj.__class__.__name__\ + +':'+hex(id(obj)) + + +import client diff --git a/libs/suds/bindings/__init__.py b/libs/suds/bindings/__init__.py new file mode 100644 index 00000000..5471ebad --- /dev/null +++ b/libs/suds/bindings/__init__.py @@ -0,0 +1,20 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides modules containing classes to support Web Services (SOAP) +bindings. +""" \ No newline at end of file diff --git a/libs/suds/bindings/binding.py b/libs/suds/bindings/binding.py new file mode 100644 index 00000000..4a7a996a --- /dev/null +++ b/libs/suds/bindings/binding.py @@ -0,0 +1,538 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides classes for (WS) SOAP bindings. +""" + +from logging import getLogger +from suds import * +from suds.sax import Namespace +from suds.sax.parser import Parser +from suds.sax.document import Document +from suds.sax.element import Element +from suds.sudsobject import Factory, Object +from suds.mx import Content +from suds.mx.literal import Literal as MxLiteral +from suds.umx.basic import Basic as UmxBasic +from suds.umx.typed import Typed as UmxTyped +from suds.bindings.multiref import MultiRef +from suds.xsd.query import TypeQuery, ElementQuery +from suds.xsd.sxbasic import Element as SchemaElement +from suds.options import Options +from suds.plugin import PluginContainer +from copy import deepcopy + +log = getLogger(__name__) + +envns = ('SOAP-ENV', 'http://schemas.xmlsoap.org/soap/envelope/') + + +class Binding: + """ + The soap binding class used to process outgoing and imcoming + soap messages per the WSDL port binding. + @cvar replyfilter: The reply filter function. + @type replyfilter: (lambda s,r: r) + @ivar wsdl: The wsdl. + @type wsdl: L{suds.wsdl.Definitions} + @ivar schema: The collective schema contained within the wsdl. + @type schema: L{xsd.schema.Schema} + @ivar options: A dictionary options. + @type options: L{Options} + """ + + replyfilter = (lambda s,r: r) + + def __init__(self, wsdl): + """ + @param wsdl: A wsdl. + @type wsdl: L{wsdl.Definitions} + """ + self.wsdl = wsdl + self.multiref = MultiRef() + + def schema(self): + return self.wsdl.schema + + def options(self): + return self.wsdl.options + + def unmarshaller(self, typed=True): + """ + Get the appropriate XML decoder. + @return: Either the (basic|typed) unmarshaller. + @rtype: L{UmxTyped} + """ + if typed: + return UmxTyped(self.schema()) + else: + return UmxBasic() + + def marshaller(self): + """ + Get the appropriate XML encoder. + @return: An L{MxLiteral} marshaller. + @rtype: L{MxLiteral} + """ + return MxLiteral(self.schema(), self.options().xstq) + + def param_defs(self, method): + """ + Get parameter definitions. + Each I{pdef} is a tuple (I{name}, L{xsd.sxbase.SchemaObject}) + @param method: A servic emethod. + @type method: I{service.Method} + @return: A collection of parameter definitions + @rtype: [I{pdef},..] + """ + raise Exception, 'not implemented' + + def get_message(self, method, args, kwargs): + """ + Get the soap message for the specified method, args and soapheaders. + This is the entry point for creating the outbound soap message. + @param method: The method being invoked. + @type method: I{service.Method} + @param args: A list of args for the method invoked. + @type args: list + @param kwargs: Named (keyword) args for the method invoked. + @type kwargs: dict + @return: The soap envelope. + @rtype: L{Document} + """ + + content = self.headercontent(method) + header = self.header(content) + content = self.bodycontent(method, args, kwargs) + body = self.body(content) + env = self.envelope(header, body) + if self.options().prefixes: + body.normalizePrefixes() + env.promotePrefixes() + else: + env.refitPrefixes() + return Document(env) + + def get_reply(self, method, reply): + """ + Process the I{reply} for the specified I{method} by sax parsing the I{reply} + and then unmarshalling into python object(s). + @param method: The name of the invoked method. + @type method: str + @param reply: The reply XML received after invoking the specified method. + @type reply: str + @return: The unmarshalled reply. The returned value is an L{Object} for a + I{list} depending on whether the service returns a single object or a + collection. + @rtype: tuple ( L{Element}, L{Object} ) + """ + reply = self.replyfilter(reply) + sax = Parser() + replyroot = sax.parse(string=reply) + plugins = PluginContainer(self.options().plugins) + plugins.message.parsed(reply=replyroot) + soapenv = replyroot.getChild('Envelope') + soapenv.promotePrefixes() + soapbody = soapenv.getChild('Body') + self.detect_fault(soapbody) + soapbody = self.multiref.process(soapbody) + nodes = self.replycontent(method, soapbody) + rtypes = self.returned_types(method) + if len(rtypes) > 1: + result = self.replycomposite(rtypes, nodes) + return (replyroot, result) + if len(rtypes) == 1: + if rtypes[0].unbounded(): + result = self.replylist(rtypes[0], nodes) + return (replyroot, result) + if len(nodes): + unmarshaller = self.unmarshaller() + resolved = rtypes[0].resolve(nobuiltin=True) + result = unmarshaller.process(nodes[0], resolved) + return (replyroot, result) + return (replyroot, None) + + def detect_fault(self, body): + """ + Detect I{hidden} soapenv:Fault element in the soap body. + @param body: The soap envelope body. + @type body: L{Element} + @raise WebFault: When found. + """ + fault = body.getChild('Fault', envns) + if fault is None: + return + unmarshaller = self.unmarshaller(False) + p = unmarshaller.process(fault) + if self.options().faults: + raise WebFault(p, fault) + return self + + + def replylist(self, rt, nodes): + """ + Construct a I{list} reply. This mehod is called when it has been detected + that the reply is a list. + @param rt: The return I{type}. + @type rt: L{suds.xsd.sxbase.SchemaObject} + @param nodes: A collection of XML nodes. + @type nodes: [L{Element},...] + @return: A list of I{unmarshalled} objects. + @rtype: [L{Object},...] + """ + result = [] + resolved = rt.resolve(nobuiltin=True) + unmarshaller = self.unmarshaller() + for node in nodes: + sobject = unmarshaller.process(node, resolved) + result.append(sobject) + return result + + def replycomposite(self, rtypes, nodes): + """ + Construct a I{composite} reply. This method is called when it has been + detected that the reply has multiple root nodes. + @param rtypes: A list of known return I{types}. + @type rtypes: [L{suds.xsd.sxbase.SchemaObject},...] + @param nodes: A collection of XML nodes. + @type nodes: [L{Element},...] + @return: The I{unmarshalled} composite object. + @rtype: L{Object},... + """ + dictionary = {} + for rt in rtypes: + dictionary[rt.name] = rt + unmarshaller = self.unmarshaller() + composite = Factory.object('reply') + for node in nodes: + tag = node.name + rt = dictionary.get(tag, None) + if rt is None: + if node.get('id') is None: + raise Exception('<%s/> not mapped to message part' % tag) + else: + continue + resolved = rt.resolve(nobuiltin=True) + sobject = unmarshaller.process(node, resolved) + value = getattr(composite, tag, None) + if value is None: + if rt.unbounded(): + value = [] + setattr(composite, tag, value) + value.append(sobject) + else: + setattr(composite, tag, sobject) + else: + if not isinstance(value, list): + value = [value,] + setattr(composite, tag, value) + value.append(sobject) + return composite + + def get_fault(self, reply): + """ + Extract the fault from the specified soap reply. If I{faults} is True, an + exception is raised. Otherwise, the I{unmarshalled} fault L{Object} is + returned. This method is called when the server raises a I{web fault}. + @param reply: A soap reply message. + @type reply: str + @return: A fault object. + @rtype: tuple ( L{Element}, L{Object} ) + """ + reply = self.replyfilter(reply) + sax = Parser() + faultroot = sax.parse(string=reply) + soapenv = faultroot.getChild('Envelope') + soapbody = soapenv.getChild('Body') + fault = soapbody.getChild('Fault') + unmarshaller = self.unmarshaller(False) + p = unmarshaller.process(fault) + if self.options().faults: + raise WebFault(p, faultroot) + return (faultroot, p.detail) + + def mkparam(self, method, pdef, object): + """ + Builds a parameter for the specified I{method} using the parameter + definition (pdef) and the specified value (object). + @param method: A method name. + @type method: str + @param pdef: A parameter definition. + @type pdef: tuple: (I{name}, L{xsd.sxbase.SchemaObject}) + @param object: The parameter value. + @type object: any + @return: The parameter fragment. + @rtype: L{Element} + """ + marshaller = self.marshaller() + content = \ + Content(tag=pdef[0], + value=object, + type=pdef[1], + real=pdef[1].resolve()) + return marshaller.process(content) + + def mkheader(self, method, hdef, object): + """ + Builds a soapheader for the specified I{method} using the header + definition (hdef) and the specified value (object). + @param method: A method name. + @type method: str + @param hdef: A header definition. + @type hdef: tuple: (I{name}, L{xsd.sxbase.SchemaObject}) + @param object: The header value. + @type object: any + @return: The parameter fragment. + @rtype: L{Element} + """ + marshaller = self.marshaller() + if isinstance(object, (list, tuple)): + tags = [] + for item in object: + tags.append(self.mkheader(method, hdef, item)) + return tags + content = Content(tag=hdef[0], value=object, type=hdef[1]) + return marshaller.process(content) + + def envelope(self, header, body): + """ + Build the B{} for an soap outbound message. + @param header: The soap message B{header}. + @type header: L{Element} + @param body: The soap message B{body}. + @type body: L{Element} + @return: The soap envelope containing the body and header. + @rtype: L{Element} + """ + env = Element('Envelope', ns=envns) + env.addPrefix(Namespace.xsins[0], Namespace.xsins[1]) + env.append(header) + env.append(body) + return env + + def header(self, content): + """ + Build the B{} for an soap outbound message. + @param content: The header content. + @type content: L{Element} + @return: the soap body fragment. + @rtype: L{Element} + """ + header = Element('Header', ns=envns) + header.append(content) + return header + + def bodycontent(self, method, args, kwargs): + """ + Get the content for the soap I{body} node. + @param method: A service method. + @type method: I{service.Method} + @param args: method parameter values + @type args: list + @param kwargs: Named (keyword) args for the method invoked. + @type kwargs: dict + @return: The xml content for the + @rtype: [L{Element},..] + """ + raise Exception, 'not implemented' + + def headercontent(self, method): + """ + Get the content for the soap I{Header} node. + @param method: A service method. + @type method: I{service.Method} + @return: The xml content for the + @rtype: [L{Element},..] + """ + n = 0 + content = [] + wsse = self.options().wsse + if wsse is not None: + content.append(wsse.xml()) + headers = self.options().soapheaders + if not isinstance(headers, (tuple,list,dict)): + headers = (headers,) + if len(headers) == 0: + return content + pts = self.headpart_types(method) + if isinstance(headers, (tuple,list)): + for header in headers: + if isinstance(header, Element): + content.append(deepcopy(header)) + continue + if len(pts) == n: break + h = self.mkheader(method, pts[n], header) + ns = pts[n][1].namespace('ns0') + h.setPrefix(ns[0], ns[1]) + content.append(h) + n += 1 + else: + for pt in pts: + header = headers.get(pt[0]) + if header is None: + continue + h = self.mkheader(method, pt, header) + ns = pt[1].namespace('ns0') + h.setPrefix(ns[0], ns[1]) + content.append(h) + return content + + def replycontent(self, method, body): + """ + Get the reply body content. + @param method: A service method. + @type method: I{service.Method} + @param body: The soap body + @type body: L{Element} + @return: the body content + @rtype: [L{Element},...] + """ + raise Exception, 'not implemented' + + def body(self, content): + """ + Build the B{} for an soap outbound message. + @param content: The body content. + @type content: L{Element} + @return: the soap body fragment. + @rtype: L{Element} + """ + body = Element('Body', ns=envns) + body.append(content) + return body + + def bodypart_types(self, method, input=True): + """ + Get a list of I{parameter definitions} (pdef) defined for the specified method. + Each I{pdef} is a tuple (I{name}, L{xsd.sxbase.SchemaObject}) + @param method: A service method. + @type method: I{service.Method} + @param input: Defines input/output message. + @type input: boolean + @return: A list of parameter definitions + @rtype: [I{pdef},] + """ + result = [] + if input: + parts = method.soap.input.body.parts + else: + parts = method.soap.output.body.parts + for p in parts: + if p.element is not None: + query = ElementQuery(p.element) + else: + query = TypeQuery(p.type) + pt = query.execute(self.schema()) + if pt is None: + raise TypeNotFound(query.ref) + if p.type is not None: + pt = PartElement(p.name, pt) + if input: + if pt.name is None: + result.append((p.name, pt)) + else: + result.append((pt.name, pt)) + else: + result.append(pt) + return result + + def headpart_types(self, method, input=True): + """ + Get a list of I{parameter definitions} (pdef) defined for the specified method. + Each I{pdef} is a tuple (I{name}, L{xsd.sxbase.SchemaObject}) + @param method: A service method. + @type method: I{service.Method} + @param input: Defines input/output message. + @type input: boolean + @return: A list of parameter definitions + @rtype: [I{pdef},] + """ + result = [] + if input: + headers = method.soap.input.headers + else: + headers = method.soap.output.headers + for header in headers: + part = header.part + if part.element is not None: + query = ElementQuery(part.element) + else: + query = TypeQuery(part.type) + pt = query.execute(self.schema()) + if pt is None: + raise TypeNotFound(query.ref) + if part.type is not None: + pt = PartElement(part.name, pt) + if input: + if pt.name is None: + result.append((part.name, pt)) + else: + result.append((pt.name, pt)) + else: + result.append(pt) + return result + + def returned_types(self, method): + """ + Get the L{xsd.sxbase.SchemaObject} returned by the I{method}. + @param method: A service method. + @type method: I{service.Method} + @return: The name of the type return by the method. + @rtype: [I{rtype},..] + """ + result = [] + for rt in self.bodypart_types(method, input=False): + result.append(rt) + return result + + +class PartElement(SchemaElement): + """ + A part used to represent a message part when the part + references a schema type and thus assumes to be an element. + @ivar resolved: The part type. + @type resolved: L{suds.xsd.sxbase.SchemaObject} + """ + + def __init__(self, name, resolved): + """ + @param name: The part name. + @type name: str + @param resolved: The part type. + @type resolved: L{suds.xsd.sxbase.SchemaObject} + """ + root = Element('element', ns=Namespace.xsdns) + SchemaElement.__init__(self, resolved.schema, root) + self.__resolved = resolved + self.name = name + self.form_qualified = False + + def implany(self): + return self + + def optional(self): + return True + + def namespace(self, prefix=None): + return Namespace.default + + def resolve(self, nobuiltin=False): + if nobuiltin and self.__resolved.builtin(): + return self + else: + return self.__resolved + \ No newline at end of file diff --git a/libs/suds/bindings/document.py b/libs/suds/bindings/document.py new file mode 100644 index 00000000..cace0d5c --- /dev/null +++ b/libs/suds/bindings/document.py @@ -0,0 +1,160 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides classes for the (WS) SOAP I{document/literal}. +""" + +from logging import getLogger +from suds import * +from suds.bindings.binding import Binding +from suds.sax.element import Element + +log = getLogger(__name__) + + +class Document(Binding): + """ + The document/literal style. Literal is the only (@use) supported + since document/encoded is pretty much dead. + Although the soap specification supports multiple documents within the soap + , it is very uncommon. As such, suds presents an I{RPC} view of + service methods defined with a single document parameter. This is done so + that the user can pass individual parameters instead of one, single document. + To support the complete specification, service methods defined with multiple documents + (multiple message parts), must present a I{document} view for that method. + """ + + def bodycontent(self, method, args, kwargs): + # + # The I{wrapped} vs I{bare} style is detected in 2 ways. + # If there is 2+ parts in the message then it is I{bare}. + # If there is only (1) part and that part resolves to a builtin then + # it is I{bare}. Otherwise, it is I{wrapped}. + # + if not len(method.soap.input.body.parts): + return () + wrapped = method.soap.input.body.wrapped + if wrapped: + pts = self.bodypart_types(method) + root = self.document(pts[0]) + else: + root = [] + n = 0 + for pd in self.param_defs(method): + if n < len(args): + value = args[n] + else: + value = kwargs.get(pd[0]) + n += 1 + p = self.mkparam(method, pd, value) + if p is None: + continue + if not wrapped: + ns = pd[1].namespace('ns0') + p.setPrefix(ns[0], ns[1]) + root.append(p) + return root + + def replycontent(self, method, body): + wrapped = method.soap.output.body.wrapped + if wrapped: + return body[0].children + else: + return body.children + + def document(self, wrapper): + """ + Get the document root. For I{document/literal}, this is the + name of the wrapper element qualifed by the schema tns. + @param wrapper: The method name. + @type wrapper: L{xsd.sxbase.SchemaObject} + @return: A root element. + @rtype: L{Element} + """ + tag = wrapper[1].name + ns = wrapper[1].namespace('ns0') + d = Element(tag, ns=ns) + return d + + def mkparam(self, method, pdef, object): + # + # Expand list parameters into individual parameters + # each with the type information. This is because in document + # arrays are simply unbounded elements. + # + if isinstance(object, (list, tuple)): + tags = [] + for item in object: + tags.append(self.mkparam(method, pdef, item)) + return tags + else: + return Binding.mkparam(self, method, pdef, object) + + def param_defs(self, method): + # + # Get parameter definitions for document literal. + # The I{wrapped} vs I{bare} style is detected in 2 ways. + # If there is 2+ parts in the message then it is I{bare}. + # If there is only (1) part and that part resolves to a builtin then + # it is I{bare}. Otherwise, it is I{wrapped}. + # + pts = self.bodypart_types(method) + wrapped = method.soap.input.body.wrapped + if not wrapped: + return pts + result = [] + # wrapped + for p in pts: + resolved = p[1].resolve() + for child, ancestry in resolved: + if child.isattr(): + continue + if self.bychoice(ancestry): + log.debug( + '%s\ncontained by , excluded as param for %s()', + child, + method.name) + continue + result.append((child.name, child)) + return result + + def returned_types(self, method): + result = [] + wrapped = method.soap.output.body.wrapped + rts = self.bodypart_types(method, input=False) + if wrapped: + for pt in rts: + resolved = pt.resolve(nobuiltin=True) + for child, ancestry in resolved: + result.append(child) + break + else: + result += rts + return result + + def bychoice(self, ancestry): + """ + The ancestry contains a + @param ancestry: A list of ancestors. + @type ancestry: list + @return: True if contains + @rtype: boolean + """ + for x in ancestry: + if x.choice(): + return True + return False \ No newline at end of file diff --git a/libs/suds/bindings/multiref.py b/libs/suds/bindings/multiref.py new file mode 100644 index 00000000..e539592b --- /dev/null +++ b/libs/suds/bindings/multiref.py @@ -0,0 +1,126 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides classes for handling soap multirefs. +""" + +from logging import getLogger +from suds import * +from suds.sax.element import Element + +log = getLogger(__name__) + +soapenc = (None, 'http://schemas.xmlsoap.org/soap/encoding/') + +class MultiRef: + """ + Resolves and replaces multirefs. + @ivar nodes: A list of non-multiref nodes. + @type nodes: list + @ivar catalog: A dictionary of multiref nodes by id. + @type catalog: dict + """ + + def __init__(self): + self.nodes = [] + self.catalog = {} + + def process(self, body): + """ + Process the specified soap envelope body and replace I{multiref} node + references with the contents of the referenced node. + @param body: A soap envelope body node. + @type body: L{Element} + @return: The processed I{body} + @rtype: L{Element} + """ + self.nodes = [] + self.catalog = {} + self.build_catalog(body) + self.update(body) + body.children = self.nodes + return body + + def update(self, node): + """ + Update the specified I{node} by replacing the I{multiref} references with + the contents of the referenced nodes and remove the I{href} attribute. + @param node: A node to update. + @type node: L{Element} + @return: The updated node + @rtype: L{Element} + """ + self.replace_references(node) + for c in node.children: + self.update(c) + return node + + def replace_references(self, node): + """ + Replacing the I{multiref} references with the contents of the + referenced nodes and remove the I{href} attribute. Warning: since + the I{ref} is not cloned, + @param node: A node to update. + @type node: L{Element} + """ + href = node.getAttribute('href') + if href is None: + return + id = href.getValue() + ref = self.catalog.get(id) + if ref is None: + log.error('soap multiref: %s, not-resolved', id) + return + node.append(ref.children) + node.setText(ref.getText()) + for a in ref.attributes: + if a.name != 'id': + node.append(a) + node.remove(href) + + def build_catalog(self, body): + """ + Create the I{catalog} of multiref nodes by id and the list of + non-multiref nodes. + @param body: A soap envelope body node. + @type body: L{Element} + """ + for child in body.children: + if self.soaproot(child): + self.nodes.append(child) + id = child.get('id') + if id is None: continue + key = '#%s' % id + self.catalog[key] = child + + def soaproot(self, node): + """ + Get whether the specified I{node} is a soap encoded root. + This is determined by examining @soapenc:root='1'. + The node is considered to be a root when the attribute + is not specified. + @param node: A node to evaluate. + @type node: L{Element} + @return: True if a soap encoded root. + @rtype: bool + """ + root = node.getAttribute('root', ns=soapenc) + if root is None: + return True + else: + return ( root.value == '1' ) + \ No newline at end of file diff --git a/libs/suds/bindings/rpc.py b/libs/suds/bindings/rpc.py new file mode 100644 index 00000000..f780aa48 --- /dev/null +++ b/libs/suds/bindings/rpc.py @@ -0,0 +1,98 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides classes for the (WS) SOAP I{rpc/literal} and I{rpc/encoded} bindings. +""" + +from logging import getLogger +from suds import * +from suds.mx.encoded import Encoded as MxEncoded +from suds.umx.encoded import Encoded as UmxEncoded +from suds.bindings.binding import Binding, envns +from suds.sax.element import Element + +log = getLogger(__name__) + + +encns = ('SOAP-ENC', 'http://schemas.xmlsoap.org/soap/encoding/') + +class RPC(Binding): + """ + RPC/Literal binding style. + """ + + def param_defs(self, method): + return self.bodypart_types(method) + + def envelope(self, header, body): + env = Binding.envelope(self, header, body) + env.addPrefix(encns[0], encns[1]) + env.set('%s:encodingStyle' % envns[0], + 'http://schemas.xmlsoap.org/soap/encoding/') + return env + + def bodycontent(self, method, args, kwargs): + n = 0 + root = self.method(method) + for pd in self.param_defs(method): + if n < len(args): + value = args[n] + else: + value = kwargs.get(pd[0]) + p = self.mkparam(method, pd, value) + if p is not None: + root.append(p) + n += 1 + return root + + def replycontent(self, method, body): + return body[0].children + + def method(self, method): + """ + Get the document root. For I{rpc/(literal|encoded)}, this is the + name of the method qualifed by the schema tns. + @param method: A service method. + @type method: I{service.Method} + @return: A root element. + @rtype: L{Element} + """ + ns = method.soap.input.body.namespace + if ns[0] is None: + ns = ('ns0', ns[1]) + method = Element(method.name, ns=ns) + return method + + +class Encoded(RPC): + """ + RPC/Encoded (section 5) binding style. + """ + + def marshaller(self): + return MxEncoded(self.schema()) + + def unmarshaller(self, typed=True): + """ + Get the appropriate XML decoder. + @return: Either the (basic|typed) unmarshaller. + @rtype: L{UmxTyped} + """ + if typed: + return UmxEncoded(self.schema()) + else: + return RPC.unmarshaller(self, typed) diff --git a/libs/suds/builder.py b/libs/suds/builder.py new file mode 100644 index 00000000..c2aad985 --- /dev/null +++ b/libs/suds/builder.py @@ -0,0 +1,121 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{builder} module provides an wsdl/xsd defined types factory +""" + +from logging import getLogger +from suds import * +from suds.sudsobject import Factory + +log = getLogger(__name__) + + +class Builder: + """ Builder used to construct an object for types defined in the schema """ + + def __init__(self, resolver): + """ + @param resolver: A schema object name resolver. + @type resolver: L{resolver.Resolver} + """ + self.resolver = resolver + + def build(self, name): + """ build a an object for the specified typename as defined in the schema """ + if isinstance(name, basestring): + type = self.resolver.find(name) + if type is None: + raise TypeNotFound(name) + else: + type = name + cls = type.name + if type.mixed(): + data = Factory.property(cls) + else: + data = Factory.object(cls) + resolved = type.resolve() + md = data.__metadata__ + md.sxtype = resolved + md.ordering = self.ordering(resolved) + history = [] + self.add_attributes(data, resolved) + for child, ancestry in type.children(): + if self.skip_child(child, ancestry): + continue + self.process(data, child, history[:]) + return data + + def process(self, data, type, history): + """ process the specified type then process its children """ + if type in history: + return + if type.enum(): + return + history.append(type) + resolved = type.resolve() + value = None + if type.unbounded(): + value = [] + else: + if len(resolved) > 0: + if resolved.mixed(): + value = Factory.property(resolved.name) + md = value.__metadata__ + md.sxtype = resolved + else: + value = Factory.object(resolved.name) + md = value.__metadata__ + md.sxtype = resolved + md.ordering = self.ordering(resolved) + setattr(data, type.name, value) + if value is not None: + data = value + if not isinstance(data, list): + self.add_attributes(data, resolved) + for child, ancestry in resolved.children(): + if self.skip_child(child, ancestry): + continue + self.process(data, child, history[:]) + + def add_attributes(self, data, type): + """ add required attributes """ + for attr, ancestry in type.attributes(): + name = '_%s' % attr.name + value = attr.get_default() + setattr(data, name, value) + + def skip_child(self, child, ancestry): + """ get whether or not to skip the specified child """ + if child.any(): return True + for x in ancestry: + if x.choice(): + return True + return False + + def ordering(self, type): + """ get the ordering """ + result = [] + for child, ancestry in type.resolve(): + name = child.name + if child.name is None: + continue + if child.isattr(): + name = '_%s' % child.name + result.append(name) + return result + \ No newline at end of file diff --git a/libs/suds/cache.py b/libs/suds/cache.py new file mode 100644 index 00000000..801c23cf --- /dev/null +++ b/libs/suds/cache.py @@ -0,0 +1,337 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains basic caching classes. +""" + +import os +import suds +from tempfile import gettempdir as tmp +from suds.transport import * +from suds.sax.parser import Parser +from suds.sax.element import Element +from datetime import datetime as dt +from datetime import timedelta +from cStringIO import StringIO +from logging import getLogger +try: + import cPickle as pickle +except: + import pickle + +log = getLogger(__name__) + + +class Cache: + """ + An object object cache. + """ + + def get(self, id): + """ + Get a object from the cache by ID. + @param id: The object ID. + @type id: str + @return: The object, else None + @rtype: any + """ + raise Exception('not-implemented') + + def getf(self, id): + """ + Get a object from the cache by ID. + @param id: The object ID. + @type id: str + @return: The object, else None + @rtype: any + """ + raise Exception('not-implemented') + + def put(self, id, object): + """ + Put a object into the cache. + @param id: The object ID. + @type id: str + @param object: The object to add. + @type object: any + """ + raise Exception('not-implemented') + + def putf(self, id, fp): + """ + Write a fp into the cache. + @param id: The object ID. + @type id: str + @param fp: File pointer. + @type fp: file-like object. + """ + raise Exception('not-implemented') + + def purge(self, id): + """ + Purge a object from the cache by id. + @param id: A object ID. + @type id: str + """ + raise Exception('not-implemented') + + def clear(self): + """ + Clear all objects from the cache. + """ + raise Exception('not-implemented') + + +class NoCache(Cache): + """ + The passthru object cache. + """ + + def get(self, id): + return None + + def getf(self, id): + return None + + def put(self, id, object): + pass + + def putf(self, id, fp): + pass + + +class FileCache(Cache): + """ + A file-based URL cache. + @cvar fnprefix: The file name prefix. + @type fnsuffix: str + @ivar duration: The cached file duration which defines how + long the file will be cached. + @type duration: (unit, value) + @ivar location: The directory for the cached files. + @type location: str + """ + fnprefix = 'suds' + units = ('months', 'weeks', 'days', 'hours', 'minutes', 'seconds') + + def __init__(self, location=None, **duration): + """ + @param location: The directory for the cached files. + @type location: str + @param duration: The cached file duration which defines how + long the file will be cached. A duration=0 means forever. + The duration may be: (months|weeks|days|hours|minutes|seconds). + @type duration: {unit:value} + """ + if location is None: + location = os.path.join(tmp(), 'suds') + self.location = location + self.duration = (None, 0) + self.setduration(**duration) + self.checkversion() + + def fnsuffix(self): + """ + Get the file name suffix + @return: The suffix + @rtype: str + """ + return 'gcf' + + def setduration(self, **duration): + """ + Set the caching duration which defines how long the + file will be cached. + @param duration: The cached file duration which defines how + long the file will be cached. A duration=0 means forever. + The duration may be: (months|weeks|days|hours|minutes|seconds). + @type duration: {unit:value} + """ + if len(duration) == 1: + arg = duration.items()[0] + if not arg[0] in self.units: + raise Exception('must be: %s' % str(self.units)) + self.duration = arg + return self + + def setlocation(self, location): + """ + Set the location (directory) for the cached files. + @param location: The directory for the cached files. + @type location: str + """ + self.location = location + + def mktmp(self): + """ + Make the I{location} directory if it doesn't already exits. + """ + try: + if not os.path.isdir(self.location): + os.makedirs(self.location) + except: + log.debug(self.location, exc_info=1) + return self + + def put(self, id, bfr): + try: + fn = self.__fn(id) + f = self.open(fn, 'w') + f.write(bfr) + f.close() + return bfr + except: + log.debug(id, exc_info=1) + return bfr + + def putf(self, id, fp): + try: + fn = self.__fn(id) + f = self.open(fn, 'w') + f.write(fp.read()) + fp.close() + f.close() + return open(fn) + except: + log.debug(id, exc_info=1) + return fp + + def get(self, id): + try: + f = self.getf(id) + bfr = f.read() + f.close() + return bfr + except: + pass + + def getf(self, id): + try: + fn = self.__fn(id) + self.validate(fn) + return self.open(fn) + except: + pass + + def validate(self, fn): + """ + Validate that the file has not expired based on the I{duration}. + @param fn: The file name. + @type fn: str + """ + if self.duration[1] < 1: + return + created = dt.fromtimestamp(os.path.getctime(fn)) + d = { self.duration[0]:self.duration[1] } + expired = created+timedelta(**d) + if expired < dt.now(): + log.debug('%s expired, deleted', fn) + os.remove(fn) + + def clear(self): + for fn in os.listdir(self.location): + if os.path.isdir(fn): + continue + if fn.startswith(self.fnprefix): + log.debug('deleted: %s', fn) + os.remove(os.path.join(self.location, fn)) + + def purge(self, id): + fn = self.__fn(id) + try: + os.remove(fn) + except: + pass + + def open(self, fn, *args): + """ + Open the cache file making sure the directory is created. + """ + self.mktmp() + return open(fn, *args) + + def checkversion(self): + path = os.path.join(self.location, 'version') + try: + + f = self.open(path) + version = f.read() + f.close() + if version != suds.__version__: + raise Exception() + except: + self.clear() + f = self.open(path, 'w') + f.write(suds.__version__) + f.close() + + def __fn(self, id): + name = id + suffix = self.fnsuffix() + fn = '%s-%s.%s' % (self.fnprefix, name, suffix) + return os.path.join(self.location, fn) + + +class DocumentCache(FileCache): + """ + Provides xml document caching. + """ + + def fnsuffix(self): + return 'xml' + + def get(self, id): + try: + fp = FileCache.getf(self, id) + if fp is None: + return None + p = Parser() + return p.parse(fp) + except: + FileCache.purge(self, id) + + def put(self, id, object): + if isinstance(object, Element): + FileCache.put(self, id, str(object)) + return object + + +class ObjectCache(FileCache): + """ + Provides pickled object caching. + @cvar protocol: The pickling protocol. + @type protocol: int + """ + protocol = 2 + + def fnsuffix(self): + return 'px' + + def get(self, id): + try: + fp = FileCache.getf(self, id) + if fp is None: + return None + else: + return pickle.load(fp) + except: + FileCache.purge(self, id) + + def put(self, id, object): + bfr = pickle.dumps(object, self.protocol) + FileCache.put(self, id, bfr) + return object diff --git a/libs/suds/client.py b/libs/suds/client.py new file mode 100644 index 00000000..5a740975 --- /dev/null +++ b/libs/suds/client.py @@ -0,0 +1,785 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{2nd generation} service proxy provides access to web services. +See I{README.txt} +""" + +import suds +import suds.metrics as metrics +from cookielib import CookieJar +from suds import * +from suds.reader import DefinitionsReader +from suds.transport import TransportError, Request +from suds.transport.https import HttpAuthenticated +from suds.servicedefinition import ServiceDefinition +from suds import sudsobject +from sudsobject import Factory as InstFactory +from sudsobject import Object +from suds.resolver import PathResolver +from suds.builder import Builder +from suds.wsdl import Definitions +from suds.cache import ObjectCache +from suds.sax.document import Document +from suds.sax.parser import Parser +from suds.options import Options +from suds.properties import Unskin +from urlparse import urlparse +from copy import deepcopy +from suds.plugin import PluginContainer +from logging import getLogger + +log = getLogger(__name__) + + +class Client(object): + """ + A lightweight web services client. + I{(2nd generation)} API. + @ivar wsdl: The WSDL object. + @type wsdl:L{Definitions} + @ivar service: The service proxy used to invoke operations. + @type service: L{Service} + @ivar factory: The factory used to create objects. + @type factory: L{Factory} + @ivar sd: The service definition + @type sd: L{ServiceDefinition} + @ivar messages: The last sent/received messages. + @type messages: str[2] + """ + @classmethod + def items(cls, sobject): + """ + Extract the I{items} from a suds object much like the + items() method works on I{dict}. + @param sobject: A suds object + @type sobject: L{Object} + @return: A list of items contained in I{sobject}. + @rtype: [(key, value),...] + """ + return sudsobject.items(sobject) + + @classmethod + def dict(cls, sobject): + """ + Convert a sudsobject into a dictionary. + @param sobject: A suds object + @type sobject: L{Object} + @return: A python dictionary containing the + items contained in I{sobject}. + @rtype: dict + """ + return sudsobject.asdict(sobject) + + @classmethod + def metadata(cls, sobject): + """ + Extract the metadata from a suds object. + @param sobject: A suds object + @type sobject: L{Object} + @return: The object's metadata + @rtype: L{sudsobject.Metadata} + """ + return sobject.__metadata__ + + def __init__(self, url, **kwargs): + """ + @param url: The URL for the WSDL. + @type url: str + @param kwargs: keyword arguments. + @see: L{Options} + """ + options = Options() + options.transport = HttpAuthenticated() + self.options = options + options.cache = ObjectCache(days=1) + self.set_options(**kwargs) + reader = DefinitionsReader(options, Definitions) + self.wsdl = reader.open(url) + plugins = PluginContainer(options.plugins) + plugins.init.initialized(wsdl=self.wsdl) + self.factory = Factory(self.wsdl) + self.service = ServiceSelector(self, self.wsdl.services) + self.sd = [] + for s in self.wsdl.services: + sd = ServiceDefinition(self.wsdl, s) + self.sd.append(sd) + self.messages = dict(tx=None, rx=None) + + def set_options(self, **kwargs): + """ + Set options. + @param kwargs: keyword arguments. + @see: L{Options} + """ + p = Unskin(self.options) + p.update(kwargs) + + def add_prefix(self, prefix, uri): + """ + Add I{static} mapping of an XML namespace prefix to a namespace. + This is useful for cases when a wsdl and referenced schemas make heavy + use of namespaces and those namespaces are subject to changed. + @param prefix: An XML namespace prefix. + @type prefix: str + @param uri: An XML namespace URI. + @type uri: str + @raise Exception: when prefix is already mapped. + """ + root = self.wsdl.root + mapped = root.resolvePrefix(prefix, None) + if mapped is None: + root.addPrefix(prefix, uri) + return + if mapped[1] != uri: + raise Exception('"%s" already mapped as "%s"' % (prefix, mapped)) + + def last_sent(self): + """ + Get last sent I{soap} message. + @return: The last sent I{soap} message. + @rtype: L{Document} + """ + return self.messages.get('tx') + + def last_received(self): + """ + Get last received I{soap} message. + @return: The last received I{soap} message. + @rtype: L{Document} + """ + return self.messages.get('rx') + + def clone(self): + """ + Get a shallow clone of this object. + The clone only shares the WSDL. All other attributes are + unique to the cloned object including options. + @return: A shallow clone. + @rtype: L{Client} + """ + class Uninitialized(Client): + def __init__(self): + pass + clone = Uninitialized() + clone.options = Options() + cp = Unskin(clone.options) + mp = Unskin(self.options) + cp.update(deepcopy(mp)) + clone.wsdl = self.wsdl + clone.factory = self.factory + clone.service = ServiceSelector(clone, self.wsdl.services) + clone.sd = self.sd + clone.messages = dict(tx=None, rx=None) + return clone + + def __str__(self): + return unicode(self) + + def __unicode__(self): + s = ['\n'] + build = suds.__build__.split() + s.append('Suds ( https://fedorahosted.org/suds/ )') + s.append(' version: %s' % suds.__version__) + s.append(' %s build: %s' % (build[0], build[1])) + for sd in self.sd: + s.append('\n\n%s' % unicode(sd)) + return ''.join(s) + + +class Factory: + """ + A factory for instantiating types defined in the wsdl + @ivar resolver: A schema type resolver. + @type resolver: L{PathResolver} + @ivar builder: A schema object builder. + @type builder: L{Builder} + """ + + def __init__(self, wsdl): + """ + @param wsdl: A schema object. + @type wsdl: L{wsdl.Definitions} + """ + self.wsdl = wsdl + self.resolver = PathResolver(wsdl) + self.builder = Builder(self.resolver) + + def create(self, name): + """ + create a WSDL type by name + @param name: The name of a type defined in the WSDL. + @type name: str + @return: The requested object. + @rtype: L{Object} + """ + timer = metrics.Timer() + timer.start() + type = self.resolver.find(name) + if type is None: + raise TypeNotFound(name) + if type.enum(): + result = InstFactory.object(name) + for e, a in type.children(): + setattr(result, e.name, e.name) + else: + try: + result = self.builder.build(type) + except Exception, e: + log.error("create '%s' failed", name, exc_info=True) + raise BuildError(name, e) + timer.stop() + metrics.log.debug('%s created: %s', name, timer) + return result + + def separator(self, ps): + """ + Set the path separator. + @param ps: The new path separator. + @type ps: char + """ + self.resolver = PathResolver(self.wsdl, ps) + + +class ServiceSelector: + """ + The B{service} selector is used to select a web service. + In most cases, the wsdl only defines (1) service in which access + by subscript is passed through to a L{PortSelector}. This is also the + behavior when a I{default} service has been specified. In cases + where multiple services have been defined and no default has been + specified, the service is found by name (or index) and a L{PortSelector} + for the service is returned. In all cases, attribute access is + forwarded to the L{PortSelector} for either the I{first} service or the + I{default} service (when specified). + @ivar __client: A suds client. + @type __client: L{Client} + @ivar __services: A list of I{wsdl} services. + @type __services: list + """ + def __init__(self, client, services): + """ + @param client: A suds client. + @type client: L{Client} + @param services: A list of I{wsdl} services. + @type services: list + """ + self.__client = client + self.__services = services + + def __getattr__(self, name): + """ + Request to access an attribute is forwarded to the + L{PortSelector} for either the I{first} service or the + I{default} service (when specified). + @param name: The name of a method. + @type name: str + @return: A L{PortSelector}. + @rtype: L{PortSelector}. + """ + default = self.__ds() + if default is None: + port = self.__find(0) + else: + port = default + return getattr(port, name) + + def __getitem__(self, name): + """ + Provides selection of the I{service} by name (string) or + index (integer). In cases where only (1) service is defined + or a I{default} has been specified, the request is forwarded + to the L{PortSelector}. + @param name: The name (or index) of a service. + @type name: (int|str) + @return: A L{PortSelector} for the specified service. + @rtype: L{PortSelector}. + """ + if len(self.__services) == 1: + port = self.__find(0) + return port[name] + default = self.__ds() + if default is not None: + port = default + return port[name] + return self.__find(name) + + def __find(self, name): + """ + Find a I{service} by name (string) or index (integer). + @param name: The name (or index) of a service. + @type name: (int|str) + @return: A L{PortSelector} for the found service. + @rtype: L{PortSelector}. + """ + service = None + if not len(self.__services): + raise Exception, 'No services defined' + if isinstance(name, int): + try: + service = self.__services[name] + name = service.name + except IndexError: + raise ServiceNotFound, 'at [%d]' % name + else: + for s in self.__services: + if name == s.name: + service = s + break + if service is None: + raise ServiceNotFound, name + return PortSelector(self.__client, service.ports, name) + + def __ds(self): + """ + Get the I{default} service if defined in the I{options}. + @return: A L{PortSelector} for the I{default} service. + @rtype: L{PortSelector}. + """ + ds = self.__client.options.service + if ds is None: + return None + else: + return self.__find(ds) + + +class PortSelector: + """ + The B{port} selector is used to select a I{web service} B{port}. + In cases where multiple ports have been defined and no default has been + specified, the port is found by name (or index) and a L{MethodSelector} + for the port is returned. In all cases, attribute access is + forwarded to the L{MethodSelector} for either the I{first} port or the + I{default} port (when specified). + @ivar __client: A suds client. + @type __client: L{Client} + @ivar __ports: A list of I{service} ports. + @type __ports: list + @ivar __qn: The I{qualified} name of the port (used for logging). + @type __qn: str + """ + def __init__(self, client, ports, qn): + """ + @param client: A suds client. + @type client: L{Client} + @param ports: A list of I{service} ports. + @type ports: list + @param qn: The name of the service. + @type qn: str + """ + self.__client = client + self.__ports = ports + self.__qn = qn + + def __getattr__(self, name): + """ + Request to access an attribute is forwarded to the + L{MethodSelector} for either the I{first} port or the + I{default} port (when specified). + @param name: The name of a method. + @type name: str + @return: A L{MethodSelector}. + @rtype: L{MethodSelector}. + """ + default = self.__dp() + if default is None: + m = self.__find(0) + else: + m = default + return getattr(m, name) + + def __getitem__(self, name): + """ + Provides selection of the I{port} by name (string) or + index (integer). In cases where only (1) port is defined + or a I{default} has been specified, the request is forwarded + to the L{MethodSelector}. + @param name: The name (or index) of a port. + @type name: (int|str) + @return: A L{MethodSelector} for the specified port. + @rtype: L{MethodSelector}. + """ + default = self.__dp() + if default is None: + return self.__find(name) + else: + return default + + def __find(self, name): + """ + Find a I{port} by name (string) or index (integer). + @param name: The name (or index) of a port. + @type name: (int|str) + @return: A L{MethodSelector} for the found port. + @rtype: L{MethodSelector}. + """ + port = None + if not len(self.__ports): + raise Exception, 'No ports defined: %s' % self.__qn + if isinstance(name, int): + qn = '%s[%d]' % (self.__qn, name) + try: + port = self.__ports[name] + except IndexError: + raise PortNotFound, qn + else: + qn = '.'.join((self.__qn, name)) + for p in self.__ports: + if name == p.name: + port = p + break + if port is None: + raise PortNotFound, qn + qn = '.'.join((self.__qn, port.name)) + return MethodSelector(self.__client, port.methods, qn) + + def __dp(self): + """ + Get the I{default} port if defined in the I{options}. + @return: A L{MethodSelector} for the I{default} port. + @rtype: L{MethodSelector}. + """ + dp = self.__client.options.port + if dp is None: + return None + else: + return self.__find(dp) + + +class MethodSelector: + """ + The B{method} selector is used to select a B{method} by name. + @ivar __client: A suds client. + @type __client: L{Client} + @ivar __methods: A dictionary of methods. + @type __methods: dict + @ivar __qn: The I{qualified} name of the method (used for logging). + @type __qn: str + """ + def __init__(self, client, methods, qn): + """ + @param client: A suds client. + @type client: L{Client} + @param methods: A dictionary of methods. + @type methods: dict + @param qn: The I{qualified} name of the port. + @type qn: str + """ + self.__client = client + self.__methods = methods + self.__qn = qn + + def __getattr__(self, name): + """ + Get a method by name and return it in an I{execution wrapper}. + @param name: The name of a method. + @type name: str + @return: An I{execution wrapper} for the specified method name. + @rtype: L{Method} + """ + return self[name] + + def __getitem__(self, name): + """ + Get a method by name and return it in an I{execution wrapper}. + @param name: The name of a method. + @type name: str + @return: An I{execution wrapper} for the specified method name. + @rtype: L{Method} + """ + m = self.__methods.get(name) + if m is None: + qn = '.'.join((self.__qn, name)) + raise MethodNotFound, qn + return Method(self.__client, m) + + +class Method: + """ + The I{method} (namespace) object. + @ivar client: A client object. + @type client: L{Client} + @ivar method: A I{wsdl} method. + @type I{wsdl} Method. + """ + + def __init__(self, client, method): + """ + @param client: A client object. + @type client: L{Client} + @param method: A I{raw} method. + @type I{raw} Method. + """ + self.client = client + self.method = method + + def __call__(self, *args, **kwargs): + """ + Invoke the method. + """ + clientclass = self.clientclass(kwargs) + client = clientclass(self.client, self.method) + if not self.faults(): + try: + return client.invoke(args, kwargs) + except WebFault, e: + return (500, e) + else: + return client.invoke(args, kwargs) + + def faults(self): + """ get faults option """ + return self.client.options.faults + + def clientclass(self, kwargs): + """ get soap client class """ + if SimClient.simulation(kwargs): + return SimClient + else: + return SoapClient + + +class SoapClient: + """ + A lightweight soap based web client B{**not intended for external use} + @ivar service: The target method. + @type service: L{Service} + @ivar method: A target method. + @type method: L{Method} + @ivar options: A dictonary of options. + @type options: dict + @ivar cookiejar: A cookie jar. + @type cookiejar: libcookie.CookieJar + """ + + def __init__(self, client, method): + """ + @param client: A suds client. + @type client: L{Client} + @param method: A target method. + @type method: L{Method} + """ + self.client = client + self.method = method + self.options = client.options + self.cookiejar = CookieJar() + + def invoke(self, args, kwargs): + """ + Send the required soap message to invoke the specified method + @param args: A list of args for the method invoked. + @type args: list + @param kwargs: Named (keyword) args for the method invoked. + @type kwargs: dict + @return: The result of the method invocation. + @rtype: I{builtin}|I{subclass of} L{Object} + """ + timer = metrics.Timer() + timer.start() + result = None + binding = self.method.binding.input + soapenv = binding.get_message(self.method, args, kwargs) + timer.stop() + metrics.log.debug( + "message for '%s' created: %s", + self.method.name, + timer) + timer.start() + result = self.send(soapenv) + timer.stop() + metrics.log.debug( + "method '%s' invoked: %s", + self.method.name, + timer) + return result + + def send(self, soapenv): + """ + Send soap message. + @param soapenv: A soap envelope to send. + @type soapenv: L{Document} + @return: The reply to the sent message. + @rtype: I{builtin} or I{subclass of} L{Object} + """ + result = None + location = self.location() + binding = self.method.binding.input + transport = self.options.transport + retxml = self.options.retxml + prettyxml = self.options.prettyxml + log.debug('sending to (%s)\nmessage:\n%s', location, soapenv) + try: + self.last_sent(soapenv) + plugins = PluginContainer(self.options.plugins) + plugins.message.marshalled(envelope=soapenv.root()) + if prettyxml: + soapenv = soapenv.str() + else: + soapenv = soapenv.plain() + soapenv = soapenv.encode('utf-8') + plugins.message.sending(envelope=soapenv) + request = Request(location, soapenv) + request.headers = self.headers() + reply = transport.send(request) + ctx = plugins.message.received(reply=reply.message) + reply.message = ctx.reply + if retxml: + result = reply.message + else: + result = self.succeeded(binding, reply.message) + except TransportError, e: + if e.httpcode in (202,204): + result = None + else: + log.error(self.last_sent()) + result = self.failed(binding, e) + return result + + def headers(self): + """ + Get http headers or the http/https request. + @return: A dictionary of header/values. + @rtype: dict + """ + action = self.method.soap.action + stock = { 'Content-Type' : 'text/xml; charset=utf-8', 'SOAPAction': action } + result = dict(stock, **self.options.headers) + log.debug('headers = %s', result) + return result + + def succeeded(self, binding, reply): + """ + Request succeeded, process the reply + @param binding: The binding to be used to process the reply. + @type binding: L{bindings.binding.Binding} + @param reply: The raw reply text. + @type reply: str + @return: The method result. + @rtype: I{builtin}, L{Object} + @raise WebFault: On server. + """ + log.debug('http succeeded:\n%s', reply) + plugins = PluginContainer(self.options.plugins) + if len(reply) > 0: + reply, result = binding.get_reply(self.method, reply) + self.last_received(reply) + else: + result = None + ctx = plugins.message.unmarshalled(reply=result) + result = ctx.reply + if self.options.faults: + return result + else: + return (200, result) + + def failed(self, binding, error): + """ + Request failed, process reply based on reason + @param binding: The binding to be used to process the reply. + @type binding: L{suds.bindings.binding.Binding} + @param error: The http error message + @type error: L{transport.TransportError} + """ + status, reason = (error.httpcode, tostr(error)) + reply = error.fp.read() + log.debug('http failed:\n%s', reply) + if status == 500: + if len(reply) > 0: + r, p = binding.get_fault(reply) + self.last_received(r) + return (status, p) + else: + return (status, None) + if self.options.faults: + raise Exception((status, reason)) + else: + return (status, None) + + def location(self): + p = Unskin(self.options) + return p.get('location', self.method.location) + + def last_sent(self, d=None): + key = 'tx' + messages = self.client.messages + if d is None: + return messages.get(key) + else: + messages[key] = d + + def last_received(self, d=None): + key = 'rx' + messages = self.client.messages + if d is None: + return messages.get(key) + else: + messages[key] = d + + +class SimClient(SoapClient): + """ + Loopback client used for message/reply simulation. + """ + + injkey = '__inject' + + @classmethod + def simulation(cls, kwargs): + """ get whether loopback has been specified in the I{kwargs}. """ + return kwargs.has_key(SimClient.injkey) + + def invoke(self, args, kwargs): + """ + Send the required soap message to invoke the specified method + @param args: A list of args for the method invoked. + @type args: list + @param kwargs: Named (keyword) args for the method invoked. + @type kwargs: dict + @return: The result of the method invocation. + @rtype: I{builtin} or I{subclass of} L{Object} + """ + simulation = kwargs[self.injkey] + msg = simulation.get('msg') + reply = simulation.get('reply') + fault = simulation.get('fault') + if msg is None: + if reply is not None: + return self.__reply(reply, args, kwargs) + if fault is not None: + return self.__fault(fault) + raise Exception('(reply|fault) expected when msg=None') + sax = Parser() + msg = sax.parse(string=msg) + return self.send(msg) + + def __reply(self, reply, args, kwargs): + """ simulate the reply """ + binding = self.method.binding.input + msg = binding.get_message(self.method, args, kwargs) + log.debug('inject (simulated) send message:\n%s', msg) + binding = self.method.binding.output + return self.succeeded(binding, reply) + + def __fault(self, reply): + """ simulate the (fault) reply """ + binding = self.method.binding.output + if self.options.faults: + r, p = binding.get_fault(reply) + self.last_received(r) + return (500, p) + else: + return (500, None) diff --git a/libs/suds/metrics.py b/libs/suds/metrics.py new file mode 100644 index 00000000..403224ae --- /dev/null +++ b/libs/suds/metrics.py @@ -0,0 +1,62 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{metrics} module defines classes and other resources +designed for collecting and reporting performance metrics. +""" + +import time +from logging import getLogger +from suds import * +from math import modf + +log = getLogger(__name__) + +class Timer: + + def __init__(self): + self.started = 0 + self.stopped = 0 + + def start(self): + self.started = time.time() + self.stopped = 0 + return self + + def stop(self): + if self.started > 0: + self.stopped = time.time() + return self + + def duration(self): + return ( self.stopped - self.started ) + + def __str__(self): + if self.started == 0: + return 'not-running' + if self.started > 0 and self.stopped == 0: + return 'started: %d (running)' % self.started + duration = self.duration() + jmod = ( lambda m : (m[1], m[0]*1000) ) + if duration < 1: + ms = (duration*1000) + return '%d (ms)' % ms + if duration < 60: + m = modf(duration) + return '%d.%.3d (seconds)' % jmod(m) + m = modf(duration/60) + return '%d.%.3d (minutes)' % jmod(m) diff --git a/libs/suds/mx/__init__.py b/libs/suds/mx/__init__.py new file mode 100644 index 00000000..77e6ca17 --- /dev/null +++ b/libs/suds/mx/__init__.py @@ -0,0 +1,59 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides modules containing classes to support +marshalling (XML). +""" + +from suds.sudsobject import Object + + +class Content(Object): + """ + Marshaller Content. + @ivar tag: The content tag. + @type tag: str + @ivar value: The content's value. + @type value: I{any} + """ + + extensions = [] + + def __init__(self, tag=None, value=None, **kwargs): + """ + @param tag: The content tag. + @type tag: str + @param value: The content's value. + @type value: I{any} + """ + Object.__init__(self) + self.tag = tag + self.value = value + for k,v in kwargs.items(): + setattr(self, k, v) + + def __getattr__(self, name): + if name not in self.__dict__: + if name in self.extensions: + v = None + setattr(self, name, v) + else: + raise AttributeError, \ + 'Content has no attribute %s' % name + else: + v = self.__dict__[name] + return v \ No newline at end of file diff --git a/libs/suds/mx/appender.py b/libs/suds/mx/appender.py new file mode 100644 index 00000000..206abc04 --- /dev/null +++ b/libs/suds/mx/appender.py @@ -0,0 +1,316 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides appender classes for I{marshalling}. +""" + +from logging import getLogger +from suds import * +from suds.mx import * +from suds.sudsobject import footprint +from suds.sudsobject import Object, Property +from suds.sax.element import Element +from suds.sax.text import Text +from copy import deepcopy + +log = getLogger(__name__) + +class Matcher: + """ + Appender matcher. + @ivar cls: A class object. + @type cls: I{classobj} + """ + + def __init__(self, cls): + """ + @param cls: A class object. + @type cls: I{classobj} + """ + self.cls = cls + + def __eq__(self, x): + if self.cls is None: + return ( x is None ) + else: + return isinstance(x, self.cls) + + +class ContentAppender: + """ + Appender used to add content to marshalled objects. + @ivar default: The default appender. + @type default: L{Appender} + @ivar appenders: A I{table} of appenders mapped by class. + @type appenders: I{table} + """ + + def __init__(self, marshaller): + """ + @param marshaller: A marshaller. + @type marshaller: L{suds.mx.core.Core} + """ + self.default = PrimativeAppender(marshaller) + self.appenders = ( + (Matcher(None), + NoneAppender(marshaller)), + (Matcher(null), + NoneAppender(marshaller)), + (Matcher(Property), + PropertyAppender(marshaller)), + (Matcher(Object), + ObjectAppender(marshaller)), + (Matcher(Element), + ElementAppender(marshaller)), + (Matcher(Text), + TextAppender(marshaller)), + (Matcher(list), + ListAppender(marshaller)), + (Matcher(tuple), + ListAppender(marshaller)), + (Matcher(dict), + DictAppender(marshaller)), + ) + + def append(self, parent, content): + """ + Select an appender and append the content to parent. + @param parent: A parent node. + @type parent: L{Element} + @param content: The content to append. + @type content: L{Content} + """ + appender = self.default + for a in self.appenders: + if a[0] == content.value: + appender = a[1] + break + appender.append(parent, content) + + +class Appender: + """ + An appender used by the marshaller to append content. + @ivar marshaller: A marshaller. + @type marshaller: L{suds.mx.core.Core} + """ + + def __init__(self, marshaller): + """ + @param marshaller: A marshaller. + @type marshaller: L{suds.mx.core.Core} + """ + self.marshaller = marshaller + + def node(self, content): + """ + Create and return an XML node that is qualified + using the I{type}. Also, make sure all referenced namespace + prefixes are declared. + @param content: The content for which proccessing has ended. + @type content: L{Object} + @return: A new node. + @rtype: L{Element} + """ + return self.marshaller.node(content) + + def setnil(self, node, content): + """ + Set the value of the I{node} to nill. + @param node: A I{nil} node. + @type node: L{Element} + @param content: The content for which proccessing has ended. + @type content: L{Object} + """ + self.marshaller.setnil(node, content) + + def setdefault(self, node, content): + """ + Set the value of the I{node} to a default value. + @param node: A I{nil} node. + @type node: L{Element} + @param content: The content for which proccessing has ended. + @type content: L{Object} + @return: The default. + """ + return self.marshaller.setdefault(node, content) + + def optional(self, content): + """ + Get whether the specified content is optional. + @param content: The content which to check. + @type content: L{Content} + """ + return self.marshaller.optional(content) + + def suspend(self, content): + """ + Notify I{marshaller} that appending this content has suspended. + @param content: The content for which proccessing has been suspended. + @type content: L{Object} + """ + self.marshaller.suspend(content) + + def resume(self, content): + """ + Notify I{marshaller} that appending this content has resumed. + @param content: The content for which proccessing has been resumed. + @type content: L{Object} + """ + self.marshaller.resume(content) + + def append(self, parent, content): + """ + Append the specified L{content} to the I{parent}. + @param content: The content to append. + @type content: L{Object} + """ + self.marshaller.append(parent, content) + + +class PrimativeAppender(Appender): + """ + An appender for python I{primative} types. + """ + + def append(self, parent, content): + if content.tag.startswith('_'): + attr = content.tag[1:] + value = tostr(content.value) + if value: + parent.set(attr, value) + else: + child = self.node(content) + child.setText(tostr(content.value)) + parent.append(child) + + +class NoneAppender(Appender): + """ + An appender for I{None} values. + """ + + def append(self, parent, content): + child = self.node(content) + default = self.setdefault(child, content) + if default is None: + self.setnil(child, content) + parent.append(child) + + +class PropertyAppender(Appender): + """ + A L{Property} appender. + """ + + def append(self, parent, content): + p = content.value + child = self.node(content) + child.setText(p.get()) + parent.append(child) + for item in p.items(): + cont = Content(tag=item[0], value=item[1]) + Appender.append(self, child, cont) + + +class ObjectAppender(Appender): + """ + An L{Object} appender. + """ + + def append(self, parent, content): + object = content.value + if self.optional(content) and footprint(object) == 0: + return + child = self.node(content) + parent.append(child) + for item in object: + cont = Content(tag=item[0], value=item[1]) + Appender.append(self, child, cont) + + +class DictAppender(Appender): + """ + An python I{dict} appender. + """ + + def append(self, parent, content): + d = content.value + if self.optional(content) and len(d) == 0: + return + child = self.node(content) + parent.append(child) + for item in d.items(): + cont = Content(tag=item[0], value=item[1]) + Appender.append(self, child, cont) + + +class ElementWrapper(Element): + """ + Element wrapper. + """ + + def __init__(self, content): + Element.__init__(self, content.name, content.parent) + self.__content = content + + def str(self, indent=0): + return self.__content.str(indent) + + +class ElementAppender(Appender): + """ + An appender for I{Element} types. + """ + + def append(self, parent, content): + if content.tag.startswith('_'): + raise Exception('raw XML not valid as attribute value') + child = ElementWrapper(content.value) + parent.append(child) + + +class ListAppender(Appender): + """ + A list/tuple appender. + """ + + def append(self, parent, content): + collection = content.value + if len(collection): + self.suspend(content) + for item in collection: + cont = Content(tag=content.tag, value=item) + Appender.append(self, parent, cont) + self.resume(content) + + +class TextAppender(Appender): + """ + An appender for I{Text} values. + """ + + def append(self, parent, content): + if content.tag.startswith('_'): + attr = content.tag[1:] + value = tostr(content.value) + if value: + parent.set(attr, value) + else: + child = self.node(content) + child.setText(content.value) + parent.append(child) diff --git a/libs/suds/mx/basic.py b/libs/suds/mx/basic.py new file mode 100644 index 00000000..336f6843 --- /dev/null +++ b/libs/suds/mx/basic.py @@ -0,0 +1,48 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides basic I{marshaller} classes. +""" + +from logging import getLogger +from suds import * +from suds.mx import * +from suds.mx.core import Core + +log = getLogger(__name__) + + +class Basic(Core): + """ + A I{basic} (untyped) marshaller. + """ + + def process(self, value, tag=None): + """ + Process (marshal) the tag with the specified value using the + optional type information. + @param value: The value (content) of the XML node. + @type value: (L{Object}|any) + @param tag: The (optional) tag name for the value. The default is + value.__class__.__name__ + @type tag: str + @return: An xml node. + @rtype: L{Element} + """ + content = Content(tag=tag, value=value) + result = Core.process(self, content) + return result \ No newline at end of file diff --git a/libs/suds/mx/core.py b/libs/suds/mx/core.py new file mode 100644 index 00000000..3c9ef597 --- /dev/null +++ b/libs/suds/mx/core.py @@ -0,0 +1,158 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides I{marshaller} core classes. +""" + +from logging import getLogger +from suds import * +from suds.mx import * +from suds.mx.appender import ContentAppender +from suds.sax.element import Element +from suds.sax.document import Document +from suds.sudsobject import Property + + +log = getLogger(__name__) + + +class Core: + """ + An I{abstract} marshaller. This class implement the core + functionality of the marshaller. + @ivar appender: A content appender. + @type appender: L{ContentAppender} + """ + + def __init__(self): + """ + """ + self.appender = ContentAppender(self) + + def process(self, content): + """ + Process (marshal) the tag with the specified value using the + optional type information. + @param content: The content to process. + @type content: L{Object} + """ + log.debug('processing:\n%s', content) + self.reset() + if content.tag is None: + content.tag = content.value.__class__.__name__ + document = Document() + if isinstance(content.value, Property): + root = self.node(content) + self.append(document, content) + else: + self.append(document, content) + return document.root() + + def append(self, parent, content): + """ + Append the specified L{content} to the I{parent}. + @param parent: The parent node to append to. + @type parent: L{Element} + @param content: The content to append. + @type content: L{Object} + """ + log.debug('appending parent:\n%s\ncontent:\n%s', parent, content) + if self.start(content): + self.appender.append(parent, content) + self.end(parent, content) + + def reset(self): + """ + Reset the marshaller. + """ + pass + + def node(self, content): + """ + Create and return an XML node. + @param content: The content for which proccessing has been suspended. + @type content: L{Object} + @return: An element. + @rtype: L{Element} + """ + return Element(content.tag) + + def start(self, content): + """ + Appending this content has started. + @param content: The content for which proccessing has started. + @type content: L{Content} + @return: True to continue appending + @rtype: boolean + """ + return True + + def suspend(self, content): + """ + Appending this content has suspended. + @param content: The content for which proccessing has been suspended. + @type content: L{Content} + """ + pass + + def resume(self, content): + """ + Appending this content has resumed. + @param content: The content for which proccessing has been resumed. + @type content: L{Content} + """ + pass + + def end(self, parent, content): + """ + Appending this content has ended. + @param parent: The parent node ending. + @type parent: L{Element} + @param content: The content for which proccessing has ended. + @type content: L{Content} + """ + pass + + def setnil(self, node, content): + """ + Set the value of the I{node} to nill. + @param node: A I{nil} node. + @type node: L{Element} + @param content: The content to set nil. + @type content: L{Content} + """ + pass + + def setdefault(self, node, content): + """ + Set the value of the I{node} to a default value. + @param node: A I{nil} node. + @type node: L{Element} + @param content: The content to set the default value. + @type content: L{Content} + @return: The default. + """ + pass + + def optional(self, content): + """ + Get whether the specified content is optional. + @param content: The content which to check. + @type content: L{Content} + """ + return False + diff --git a/libs/suds/mx/encoded.py b/libs/suds/mx/encoded.py new file mode 100644 index 00000000..9cbc8c5f --- /dev/null +++ b/libs/suds/mx/encoded.py @@ -0,0 +1,133 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides encoded I{marshaller} classes. +""" + +from logging import getLogger +from suds import * +from suds.mx import * +from suds.mx.literal import Literal +from suds.mx.typer import Typer +from suds.sudsobject import Factory, Object +from suds.xsd.query import TypeQuery + +log = getLogger(__name__) + +# +# Add encoded extensions +# aty = The soap (section 5) encoded array type. +# +Content.extensions.append('aty') + + +class Encoded(Literal): + """ + A SOAP section (5) encoding marshaller. + This marshaller supports rpc/encoded soap styles. + """ + + def start(self, content): + # + # For soap encoded arrays, the 'aty' (array type) information + # is extracted and added to the 'content'. Then, the content.value + # is replaced with an object containing an 'item=[]' attribute + # containing values that are 'typed' suds objects. + # + start = Literal.start(self, content) + if start and isinstance(content.value, (list,tuple)): + resolved = content.type.resolve() + for c in resolved: + if hasattr(c[0], 'aty'): + content.aty = (content.tag, c[0].aty) + self.cast(content) + break + return start + + def end(self, parent, content): + # + # For soap encoded arrays, the soapenc:arrayType attribute is + # added with proper type and size information. + # Eg: soapenc:arrayType="xs:int[3]" + # + Literal.end(self, parent, content) + if content.aty is None: + return + tag, aty = content.aty + ns0 = ('at0', aty[1]) + ns1 = ('at1', 'http://schemas.xmlsoap.org/soap/encoding/') + array = content.value.item + child = parent.getChild(tag) + child.addPrefix(ns0[0], ns0[1]) + child.addPrefix(ns1[0], ns1[1]) + name = '%s:arrayType' % ns1[0] + value = '%s:%s[%d]' % (ns0[0], aty[0], len(array)) + child.set(name, value) + + def encode(self, node, content): + if content.type.any(): + Typer.auto(node, content.value) + return + if content.real.any(): + Typer.auto(node, content.value) + return + ns = None + name = content.real.name + if self.xstq: + ns = content.real.namespace() + Typer.manual(node, name, ns) + + def cast(self, content): + """ + Cast the I{untyped} list items found in content I{value}. + Each items contained in the list is checked for XSD type information. + Items (values) that are I{untyped}, are replaced with suds objects and + type I{metadata} is added. + @param content: The content holding the collection. + @type content: L{Content} + @return: self + @rtype: L{Encoded} + """ + aty = content.aty[1] + resolved = content.type.resolve() + array = Factory.object(resolved.name) + array.item = [] + query = TypeQuery(aty) + ref = query.execute(self.schema) + if ref is None: + raise TypeNotFound(qref) + for x in content.value: + if isinstance(x, (list, tuple)): + array.item.append(x) + continue + if isinstance(x, Object): + md = x.__metadata__ + md.sxtype = ref + array.item.append(x) + continue + if isinstance(x, dict): + x = Factory.object(ref.name, x) + md = x.__metadata__ + md.sxtype = ref + array.item.append(x) + continue + x = Factory.property(ref.name, x) + md = x.__metadata__ + md.sxtype = ref + array.item.append(x) + content.value = array + return self diff --git a/libs/suds/mx/literal.py b/libs/suds/mx/literal.py new file mode 100644 index 00000000..937ad8ec --- /dev/null +++ b/libs/suds/mx/literal.py @@ -0,0 +1,291 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides literal I{marshaller} classes. +""" + +from logging import getLogger +from suds import * +from suds.mx import * +from suds.mx.core import Core +from suds.mx.typer import Typer +from suds.resolver import GraphResolver, Frame +from suds.sax.element import Element +from suds.sudsobject import Factory + +log = getLogger(__name__) + + +# +# Add typed extensions +# type = The expected xsd type +# real = The 'true' XSD type +# ancestry = The 'type' ancestry +# +Content.extensions.append('type') +Content.extensions.append('real') +Content.extensions.append('ancestry') + + + +class Typed(Core): + """ + A I{typed} marshaller. + This marshaller is semi-typed as needed to support both + I{document/literal} and I{rpc/literal} soap message styles. + @ivar schema: An xsd schema. + @type schema: L{xsd.schema.Schema} + @ivar resolver: A schema type resolver. + @type resolver: L{GraphResolver} + """ + + def __init__(self, schema, xstq=True): + """ + @param schema: A schema object + @type schema: L{xsd.schema.Schema} + @param xstq: The B{x}ml B{s}chema B{t}ype B{q}ualified flag indicates + that the I{xsi:type} attribute values should be qualified by namespace. + @type xstq: bool + """ + Core.__init__(self) + self.schema = schema + self.xstq = xstq + self.resolver = GraphResolver(self.schema) + + def reset(self): + self.resolver.reset() + + def start(self, content): + # + # Start marshalling the 'content' by ensuring that both the + # 'content' _and_ the resolver are primed with the XSD type + # information. The 'content' value is both translated and + # sorted based on the XSD type. Only values that are objects + # have their attributes sorted. + # + log.debug('starting content:\n%s', content) + if content.type is None: + name = content.tag + if name.startswith('_'): + name = '@'+name[1:] + content.type = self.resolver.find(name, content.value) + if content.type is None: + raise TypeNotFound(content.tag) + else: + known = None + if isinstance(content.value, Object): + known = self.resolver.known(content.value) + if known is None: + log.debug('object has no type information', content.value) + known = content.type + frame = Frame(content.type, resolved=known) + self.resolver.push(frame) + frame = self.resolver.top() + content.real = frame.resolved + content.ancestry = frame.ancestry + self.translate(content) + self.sort(content) + if self.skip(content): + log.debug('skipping (optional) content:\n%s', content) + self.resolver.pop() + return False + else: + return True + + def suspend(self, content): + # + # Suspend to process a list content. Primarily, this + # involves popping the 'list' content off the resolver's + # stack so the list items can be marshalled. + # + self.resolver.pop() + + def resume(self, content): + # + # Resume processing a list content. To do this, we + # really need to simply push the 'list' content + # back onto the resolver stack. + # + self.resolver.push(Frame(content.type)) + + def end(self, parent, content): + # + # End processing the content. Make sure the content + # ending matches the top of the resolver stack since for + # list processing we play games with the resolver stack. + # + log.debug('ending content:\n%s', content) + current = self.resolver.top().type + if current == content.type: + self.resolver.pop() + else: + raise Exception, \ + 'content (end) mismatch: top=(%s) cont=(%s)' % \ + (current, content) + + def node(self, content): + # + # Create an XML node and namespace qualify as defined + # by the schema (elementFormDefault). + # + ns = content.type.namespace() + if content.type.form_qualified: + node = Element(content.tag, ns=ns) + node.addPrefix(ns[0], ns[1]) + else: + node = Element(content.tag) + self.encode(node, content) + log.debug('created - node:\n%s', node) + return node + + def setnil(self, node, content): + # + # Set the 'node' nil only if the XSD type + # specifies that it is permitted. + # + if content.type.nillable: + node.setnil() + + def setdefault(self, node, content): + # + # Set the node to the default value specified + # by the XSD type. + # + default = content.type.default + if default is None: + pass + else: + node.setText(default) + return default + + def optional(self, content): + if content.type.optional(): + return True + for a in content.ancestry: + if a.optional(): + return True + return False + + def encode(self, node, content): + # Add (soap) encoding information only if the resolved + # type is derived by extension. Further, the xsi:type values + # is qualified by namespace only if the content (tag) and + # referenced type are in different namespaces. + if content.type.any(): + return + if not content.real.extension(): + return + if content.type.resolve() == content.real: + return + ns = None + name = content.real.name + if self.xstq: + ns = content.real.namespace('ns1') + Typer.manual(node, name, ns) + + def skip(self, content): + """ + Get whether to skip this I{content}. + Should be skipped when the content is optional + and either the value=None or the value is an empty list. + @param content: The content to skip. + @type content: L{Object} + @return: True if content is to be skipped. + @rtype: bool + """ + if self.optional(content): + v = content.value + if v is None: + return True + if isinstance(v, (list,tuple)) and len(v) == 0: + return True + return False + + def optional(self, content): + if content.type.optional(): + return True + for a in content.ancestry: + if a.optional(): + return True + return False + + def translate(self, content): + """ + Translate using the XSD type information. + Python I{dict} is translated to a suds object. Most + importantly, primative values are translated from python + types to XML types using the XSD type. + @param content: The content to translate. + @type content: L{Object} + @return: self + @rtype: L{Typed} + """ + v = content.value + if v is None: + return + if isinstance(v, dict): + cls = content.real.name + content.value = Factory.object(cls, v) + md = content.value.__metadata__ + md.sxtype = content.type + return + v = content.real.translate(v, False) + content.value = v + return self + + def sort(self, content): + """ + Sort suds object attributes based on ordering defined + in the XSD type information. + @param content: The content to sort. + @type content: L{Object} + @return: self + @rtype: L{Typed} + """ + v = content.value + if isinstance(v, Object): + md = v.__metadata__ + md.ordering = self.ordering(content.real) + return self + + def ordering(self, type): + """ + Get the attribute ordering defined in the specified + XSD type information. + @param type: An XSD type object. + @type type: SchemaObject + @return: An ordered list of attribute names. + @rtype: list + """ + result = [] + for child, ancestry in type.resolve(): + name = child.name + if child.name is None: + continue + if child.isattr(): + name = '_%s' % child.name + result.append(name) + return result + + +class Literal(Typed): + """ + A I{literal} marshaller. + This marshaller is semi-typed as needed to support both + I{document/literal} and I{rpc/literal} soap message styles. + """ + pass \ No newline at end of file diff --git a/libs/suds/mx/typer.py b/libs/suds/mx/typer.py new file mode 100644 index 00000000..ea88df7b --- /dev/null +++ b/libs/suds/mx/typer.py @@ -0,0 +1,123 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides sx typing classes. +""" + +from logging import getLogger +from suds import * +from suds.mx import * +from suds.sax import Namespace as NS +from suds.sax.text import Text + +log = getLogger(__name__) + + +class Typer: + """ + Provides XML node typing as either automatic or manual. + @cvar types: A dict of class to xs type mapping. + @type types: dict + """ + + types = { + int : ('int', NS.xsdns), + long : ('long', NS.xsdns), + float : ('float', NS.xsdns), + str : ('string', NS.xsdns), + unicode : ('string', NS.xsdns), + Text : ('string', NS.xsdns), + bool : ('boolean', NS.xsdns), + } + + @classmethod + def auto(cls, node, value=None): + """ + Automatically set the node's xsi:type attribute based on either I{value}'s + class or the class of the node's text. When I{value} is an unmapped class, + the default type (xs:any) is set. + @param node: An XML node + @type node: L{sax.element.Element} + @param value: An object that is or would be the node's text. + @type value: I{any} + @return: The specified node. + @rtype: L{sax.element.Element} + """ + if value is None: + value = node.getText() + if isinstance(value, Object): + known = cls.known(value) + if known.name is None: + return node + tm = (known.name, known.namespace()) + else: + tm = cls.types.get(value.__class__, cls.types.get(str)) + cls.manual(node, *tm) + return node + + @classmethod + def manual(cls, node, tval, ns=None): + """ + Set the node's xsi:type attribute based on either I{value}'s + class or the class of the node's text. Then adds the referenced + prefix(s) to the node's prefix mapping. + @param node: An XML node + @type node: L{sax.element.Element} + @param tval: The name of the schema type. + @type tval: str + @param ns: The XML namespace of I{tval}. + @type ns: (prefix, uri) + @return: The specified node. + @rtype: L{sax.element.Element} + """ + xta = ':'.join((NS.xsins[0], 'type')) + node.addPrefix(NS.xsins[0], NS.xsins[1]) + if ns is None: + node.set(xta, tval) + else: + ns = cls.genprefix(node, ns) + qname = ':'.join((ns[0], tval)) + node.set(xta, qname) + node.addPrefix(ns[0], ns[1]) + return node + + @classmethod + def genprefix(cls, node, ns): + """ + Generate a prefix. + @param node: An XML node on which the prefix will be used. + @type node: L{sax.element.Element} + @param ns: A namespace needing an unique prefix. + @type ns: (prefix, uri) + @return: The I{ns} with a new prefix. + """ + for n in range(1, 1024): + p = 'ns%d' % n + u = node.resolvePrefix(p, default=None) + if u is None or u == ns[1]: + return (p, ns[1]) + raise Exception('auto prefix, exhausted') + + @classmethod + def known(cls, object): + try: + md = object.__metadata__ + known = md.sxtype + return known + except: + pass + diff --git a/libs/suds/options.py b/libs/suds/options.py new file mode 100644 index 00000000..86ea2458 --- /dev/null +++ b/libs/suds/options.py @@ -0,0 +1,123 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Suds basic options classes. +""" + +from suds.properties import * +from suds.wsse import Security +from suds.xsd.doctor import Doctor +from suds.transport import Transport +from suds.cache import Cache, NoCache + + +class TpLinker(AutoLinker): + """ + Transport (auto) linker used to manage linkage between + transport objects Properties and those Properties that contain them. + """ + + def updated(self, properties, prev, next): + if isinstance(prev, Transport): + tp = Unskin(prev.options) + properties.unlink(tp) + if isinstance(next, Transport): + tp = Unskin(next.options) + properties.link(tp) + + +class Options(Skin): + """ + Options: + - B{cache} - The XML document cache. May be set (None) for no caching. + - type: L{Cache} + - default: L{NoCache} + - B{faults} - Raise faults raised by server, + else return tuple from service method invocation as (httpcode, object). + - type: I{bool} + - default: True + - B{service} - The default service name. + - type: I{str} + - default: None + - B{port} - The default service port name, not tcp port. + - type: I{str} + - default: None + - B{location} - This overrides the service port address I{URL} defined + in the WSDL. + - type: I{str} + - default: None + - B{transport} - The message transport. + - type: L{Transport} + - default: None + - B{soapheaders} - The soap headers to be included in the soap message. + - type: I{any} + - default: None + - B{wsse} - The web services I{security} provider object. + - type: L{Security} + - default: None + - B{doctor} - A schema I{doctor} object. + - type: L{Doctor} + - default: None + - B{xstq} - The B{x}ml B{s}chema B{t}ype B{q}ualified flag indicates + that the I{xsi:type} attribute values should be qualified by namespace. + - type: I{bool} + - default: True + - B{prefixes} - Elements of the soap message should be qualified (when needed) + using XML prefixes as opposed to xmlns="" syntax. + - type: I{bool} + - default: True + - B{retxml} - Flag that causes the I{raw} soap envelope to be returned instead + of the python object graph. + - type: I{bool} + - default: False + - B{prettyxml} - Flag that causes I{pretty} xml to be rendered when generating + the outbound soap envelope. + - type: I{bool} + - default: False + - B{autoblend} - Flag that ensures that the schema(s) defined within the + WSDL import each other. + - type: I{bool} + - default: False + - B{cachingpolicy} - The caching policy. + - type: I{int} + - 0 = Cache XML documents. + - 1 = Cache WSDL (pickled) object. + - default: 0 + - B{plugins} - A plugin container. + - type: I{list} + """ + def __init__(self, **kwargs): + domain = __name__ + definitions = [ + Definition('cache', Cache, NoCache()), + Definition('faults', bool, True), + Definition('transport', Transport, None, TpLinker()), + Definition('service', (int, basestring), None), + Definition('port', (int, basestring), None), + Definition('location', basestring, None), + Definition('soapheaders', (), ()), + Definition('wsse', Security, None), + Definition('doctor', Doctor, None), + Definition('xstq', bool, True), + Definition('prefixes', bool, True), + Definition('retxml', bool, False), + Definition('prettyxml', bool, False), + Definition('autoblend', bool, False), + Definition('cachingpolicy', int, 0), + Definition('plugins', (list, tuple), []), + ] + Skin.__init__(self, domain, definitions, kwargs) diff --git a/libs/suds/plugin.py b/libs/suds/plugin.py new file mode 100644 index 00000000..061c564f --- /dev/null +++ b/libs/suds/plugin.py @@ -0,0 +1,257 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The plugin module provides classes for implementation +of suds plugins. +""" + +from suds import * +from logging import getLogger + +log = getLogger(__name__) + + +class Context(object): + """ + Plugin context. + """ + pass + + +class InitContext(Context): + """ + Init Context. + @ivar wsdl: The wsdl. + @type wsdl: L{wsdl.Definitions} + """ + pass + + +class DocumentContext(Context): + """ + The XML document load context. + @ivar url: The URL. + @type url: str + @ivar document: Either the XML text or the B{parsed} document root. + @type document: (str|L{sax.element.Element}) + """ + pass + + +class MessageContext(Context): + """ + The context for sending the soap envelope. + @ivar envelope: The soap envelope to be sent. + @type envelope: (str|L{sax.element.Element}) + @ivar reply: The reply. + @type reply: (str|L{sax.element.Element}|object) + """ + pass + + +class Plugin: + """ + Plugin base. + """ + pass + + +class InitPlugin(Plugin): + """ + The base class for suds I{init} plugins. + """ + + def initialized(self, context): + """ + Suds client initialization. + Called after wsdl the has been loaded. Provides the plugin + with the opportunity to inspect/modify the WSDL. + @param context: The init context. + @type context: L{InitContext} + """ + pass + + +class DocumentPlugin(Plugin): + """ + The base class for suds I{document} plugins. + """ + + def loaded(self, context): + """ + Suds has loaded a WSDL/XSD document. Provides the plugin + with an opportunity to inspect/modify the unparsed document. + Called after each WSDL/XSD document is loaded. + @param context: The document context. + @type context: L{DocumentContext} + """ + pass + + def parsed(self, context): + """ + Suds has parsed a WSDL/XSD document. Provides the plugin + with an opportunity to inspect/modify the parsed document. + Called after each WSDL/XSD document is parsed. + @param context: The document context. + @type context: L{DocumentContext} + """ + pass + + +class MessagePlugin(Plugin): + """ + The base class for suds I{soap message} plugins. + """ + + def marshalled(self, context): + """ + Suds will send the specified soap envelope. + Provides the plugin with the opportunity to inspect/modify + the envelope Document before it is sent. + @param context: The send context. + The I{envelope} is the envelope docuemnt. + @type context: L{MessageContext} + """ + pass + + def sending(self, context): + """ + Suds will send the specified soap envelope. + Provides the plugin with the opportunity to inspect/modify + the message text it is sent. + @param context: The send context. + The I{envelope} is the envelope text. + @type context: L{MessageContext} + """ + pass + + def received(self, context): + """ + Suds has received the specified reply. + Provides the plugin with the opportunity to inspect/modify + the received XML text before it is SAX parsed. + @param context: The reply context. + The I{reply} is the raw text. + @type context: L{MessageContext} + """ + pass + + def parsed(self, context): + """ + Suds has sax parsed the received reply. + Provides the plugin with the opportunity to inspect/modify + the sax parsed DOM tree for the reply before it is unmarshalled. + @param context: The reply context. + The I{reply} is DOM tree. + @type context: L{MessageContext} + """ + pass + + def unmarshalled(self, context): + """ + Suds has unmarshalled the received reply. + Provides the plugin with the opportunity to inspect/modify + the unmarshalled reply object before it is returned. + @param context: The reply context. + The I{reply} is unmarshalled suds object. + @type context: L{MessageContext} + """ + pass + + +class PluginContainer: + """ + Plugin container provides easy method invocation. + @ivar plugins: A list of plugin objects. + @type plugins: [L{Plugin},] + @cvar ctxclass: A dict of plugin method / context classes. + @type ctxclass: dict + """ + + domains = {\ + 'init': (InitContext, InitPlugin), + 'document': (DocumentContext, DocumentPlugin), + 'message': (MessageContext, MessagePlugin ), + } + + def __init__(self, plugins): + """ + @param plugins: A list of plugin objects. + @type plugins: [L{Plugin},] + """ + self.plugins = plugins + + def __getattr__(self, name): + domain = self.domains.get(name) + if domain: + plugins = [] + ctx, pclass = domain + for p in self.plugins: + if isinstance(p, pclass): + plugins.append(p) + return PluginDomain(ctx, plugins) + else: + raise Exception, 'plugin domain (%s), invalid' % name + + +class PluginDomain: + """ + The plugin domain. + @ivar ctx: A context. + @type ctx: L{Context} + @ivar plugins: A list of plugins (targets). + @type plugins: list + """ + + def __init__(self, ctx, plugins): + self.ctx = ctx + self.plugins = plugins + + def __getattr__(self, name): + return Method(name, self) + + +class Method: + """ + Plugin method. + @ivar name: The method name. + @type name: str + @ivar domain: The plugin domain. + @type domain: L{PluginDomain} + """ + + def __init__(self, name, domain): + """ + @param name: The method name. + @type name: str + @param domain: A plugin domain. + @type domain: L{PluginDomain} + """ + self.name = name + self.domain = domain + + def __call__(self, **kwargs): + ctx = self.domain.ctx() + ctx.__dict__.update(kwargs) + for plugin in self.domain.plugins: + try: + method = getattr(plugin, self.name, None) + if method and callable(method): + method(ctx) + except Exception, pe: + log.exception(pe) + return ctx diff --git a/libs/suds/properties.py b/libs/suds/properties.py new file mode 100644 index 00000000..50b25939 --- /dev/null +++ b/libs/suds/properties.py @@ -0,0 +1,543 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Properties classes. +""" + +from logging import getLogger + +log = getLogger(__name__) + + +class AutoLinker(object): + """ + Base class, provides interface for I{automatic} link + management between a L{Properties} object and the L{Properties} + contained within I{values}. + """ + def updated(self, properties, prev, next): + """ + Notification that a values was updated and the linkage + between the I{properties} contained with I{prev} need to + be relinked to the L{Properties} contained within the + I{next} value. + """ + pass + + +class Link(object): + """ + Property link object. + @ivar endpoints: A tuple of the (2) endpoints of the link. + @type endpoints: tuple(2) + """ + def __init__(self, a, b): + """ + @param a: Property (A) to link. + @type a: L{Property} + @param b: Property (B) to link. + @type b: L{Property} + """ + pA = Endpoint(self, a) + pB = Endpoint(self, b) + self.endpoints = (pA, pB) + self.validate(a, b) + a.links.append(pB) + b.links.append(pA) + + def validate(self, pA, pB): + """ + Validate that the two properties may be linked. + @param pA: Endpoint (A) to link. + @type pA: L{Endpoint} + @param pB: Endpoint (B) to link. + @type pB: L{Endpoint} + @return: self + @rtype: L{Link} + """ + if pA in pB.links or \ + pB in pA.links: + raise Exception, 'Already linked' + dA = pA.domains() + dB = pB.domains() + for d in dA: + if d in dB: + raise Exception, 'Duplicate domain "%s" found' % d + for d in dB: + if d in dA: + raise Exception, 'Duplicate domain "%s" found' % d + kA = pA.keys() + kB = pB.keys() + for k in kA: + if k in kB: + raise Exception, 'Duplicate key %s found' % k + for k in kB: + if k in kA: + raise Exception, 'Duplicate key %s found' % k + return self + + def teardown(self): + """ + Teardown the link. + Removes endpoints from properties I{links} collection. + @return: self + @rtype: L{Link} + """ + pA, pB = self.endpoints + if pA in pB.links: + pB.links.remove(pA) + if pB in pA.links: + pA.links.remove(pB) + return self + + +class Endpoint(object): + """ + Link endpoint (wrapper). + @ivar link: The associated link. + @type link: L{Link} + @ivar target: The properties object. + @type target: L{Property} + """ + def __init__(self, link, target): + self.link = link + self.target = target + + def teardown(self): + return self.link.teardown() + + def __eq__(self, rhs): + return ( self.target == rhs ) + + def __hash__(self): + return hash(self.target) + + def __getattr__(self, name): + return getattr(self.target, name) + + +class Definition: + """ + Property definition. + @ivar name: The property name. + @type name: str + @ivar classes: The (class) list of permitted values + @type classes: tuple + @ivar default: The default value. + @ivar type: any + """ + def __init__(self, name, classes, default, linker=AutoLinker()): + """ + @param name: The property name. + @type name: str + @param classes: The (class) list of permitted values + @type classes: tuple + @param default: The default value. + @type default: any + """ + if not isinstance(classes, (list, tuple)): + classes = (classes,) + self.name = name + self.classes = classes + self.default = default + self.linker = linker + + def nvl(self, value=None): + """ + Convert the I{value} into the default when I{None}. + @param value: The proposed value. + @type value: any + @return: The I{default} when I{value} is I{None}, else I{value}. + @rtype: any + """ + if value is None: + return self.default + else: + return value + + def validate(self, value): + """ + Validate the I{value} is of the correct class. + @param value: The value to validate. + @type value: any + @raise AttributeError: When I{value} is invalid. + """ + if value is None: + return + if len(self.classes) and \ + not isinstance(value, self.classes): + msg = '"%s" must be: %s' % (self.name, self.classes) + raise AttributeError,msg + + + def __repr__(self): + return '%s: %s' % (self.name, str(self)) + + def __str__(self): + s = [] + if len(self.classes): + s.append('classes=%s' % str(self.classes)) + else: + s.append('classes=*') + s.append("default=%s" % str(self.default)) + return ', '.join(s) + + +class Properties: + """ + Represents basic application properties. + Provides basic type validation, default values and + link/synchronization behavior. + @ivar domain: The domain name. + @type domain: str + @ivar definitions: A table of property definitions. + @type definitions: {name: L{Definition}} + @ivar links: A list of linked property objects used to create + a network of properties. + @type links: [L{Property},..] + @ivar defined: A dict of property values. + @type defined: dict + """ + def __init__(self, domain, definitions, kwargs): + """ + @param domain: The property domain name. + @type domain: str + @param definitions: A table of property definitions. + @type definitions: {name: L{Definition}} + @param kwargs: A list of property name/values to set. + @type kwargs: dict + """ + self.definitions = {} + for d in definitions: + self.definitions[d.name] = d + self.domain = domain + self.links = [] + self.defined = {} + self.modified = set() + self.prime() + self.update(kwargs) + + def definition(self, name): + """ + Get the definition for the property I{name}. + @param name: The property I{name} to find the definition for. + @type name: str + @return: The property definition + @rtype: L{Definition} + @raise AttributeError: On not found. + """ + d = self.definitions.get(name) + if d is None: + raise AttributeError(name) + return d + + def update(self, other): + """ + Update the property values as specified by keyword/value. + @param other: An object to update from. + @type other: (dict|L{Properties}) + @return: self + @rtype: L{Properties} + """ + if isinstance(other, Properties): + other = other.defined + for n,v in other.items(): + self.set(n, v) + return self + + def notset(self, name): + """ + Get whether a property has never been set by I{name}. + @param name: A property name. + @type name: str + @return: True if never been set. + @rtype: bool + """ + self.provider(name).__notset(name) + + def set(self, name, value): + """ + Set the I{value} of a property by I{name}. + The value is validated against the definition and set + to the default when I{value} is None. + @param name: The property name. + @type name: str + @param value: The new property value. + @type value: any + @return: self + @rtype: L{Properties} + """ + self.provider(name).__set(name, value) + return self + + def unset(self, name): + """ + Unset a property by I{name}. + @param name: A property name. + @type name: str + @return: self + @rtype: L{Properties} + """ + self.provider(name).__set(name, None) + return self + + def get(self, name, *df): + """ + Get the value of a property by I{name}. + @param name: The property name. + @type name: str + @param df: An optional value to be returned when the value + is not set + @type df: [1]. + @return: The stored value, or I{df[0]} if not set. + @rtype: any + """ + return self.provider(name).__get(name, *df) + + def link(self, other): + """ + Link (associate) this object with anI{other} properties object + to create a network of properties. Links are bidirectional. + @param other: The object to link. + @type other: L{Properties} + @return: self + @rtype: L{Properties} + """ + Link(self, other) + return self + + def unlink(self, *others): + """ + Unlink (disassociate) the specified properties object. + @param others: The list object to unlink. Unspecified means unlink all. + @type others: [L{Properties},..] + @return: self + @rtype: L{Properties} + """ + if not len(others): + others = self.links[:] + for p in self.links[:]: + if p in others: + p.teardown() + return self + + def provider(self, name, history=None): + """ + Find the provider of the property by I{name}. + @param name: The property name. + @type name: str + @param history: A history of nodes checked to prevent + circular hunting. + @type history: [L{Properties},..] + @return: The provider when found. Otherwise, None (when nested) + and I{self} when not nested. + @rtype: L{Properties} + """ + if history is None: + history = [] + history.append(self) + if name in self.definitions: + return self + for x in self.links: + if x in history: + continue + provider = x.provider(name, history) + if provider is not None: + return provider + history.remove(self) + if len(history): + return None + return self + + def keys(self, history=None): + """ + Get the set of I{all} property names. + @param history: A history of nodes checked to prevent + circular hunting. + @type history: [L{Properties},..] + @return: A set of property names. + @rtype: list + """ + if history is None: + history = [] + history.append(self) + keys = set() + keys.update(self.definitions.keys()) + for x in self.links: + if x in history: + continue + keys.update(x.keys(history)) + history.remove(self) + return keys + + def domains(self, history=None): + """ + Get the set of I{all} domain names. + @param history: A history of nodes checked to prevent + circular hunting. + @type history: [L{Properties},..] + @return: A set of domain names. + @rtype: list + """ + if history is None: + history = [] + history.append(self) + domains = set() + domains.add(self.domain) + for x in self.links: + if x in history: + continue + domains.update(x.domains(history)) + history.remove(self) + return domains + + def prime(self): + """ + Prime the stored values based on default values + found in property definitions. + @return: self + @rtype: L{Properties} + """ + for d in self.definitions.values(): + self.defined[d.name] = d.default + return self + + def __notset(self, name): + return not (name in self.modified) + + def __set(self, name, value): + d = self.definition(name) + d.validate(value) + value = d.nvl(value) + prev = self.defined[name] + self.defined[name] = value + self.modified.add(name) + d.linker.updated(self, prev, value) + + def __get(self, name, *df): + d = self.definition(name) + value = self.defined.get(name) + if value == d.default and len(df): + value = df[0] + return value + + def str(self, history): + s = [] + s.append('Definitions:') + for d in self.definitions.values(): + s.append('\t%s' % repr(d)) + s.append('Content:') + for d in self.defined.items(): + s.append('\t%s' % str(d)) + if self not in history: + history.append(self) + s.append('Linked:') + for x in self.links: + s.append(x.str(history)) + history.remove(self) + return '\n'.join(s) + + def __repr__(self): + return str(self) + + def __str__(self): + return self.str([]) + + +class Skin(object): + """ + The meta-programming I{skin} around the L{Properties} object. + @ivar __pts__: The wrapped object. + @type __pts__: L{Properties}. + """ + def __init__(self, domain, definitions, kwargs): + self.__pts__ = Properties(domain, definitions, kwargs) + + def __setattr__(self, name, value): + builtin = name.startswith('__') and name.endswith('__') + if builtin: + self.__dict__[name] = value + return + self.__pts__.set(name, value) + + def __getattr__(self, name): + return self.__pts__.get(name) + + def __repr__(self): + return str(self) + + def __str__(self): + return str(self.__pts__) + + +class Unskin(object): + def __new__(self, *args, **kwargs): + return args[0].__pts__ + + +class Inspector: + """ + Wrapper inspector. + """ + def __init__(self, options): + self.properties = options.__pts__ + + def get(self, name, *df): + """ + Get the value of a property by I{name}. + @param name: The property name. + @type name: str + @param df: An optional value to be returned when the value + is not set + @type df: [1]. + @return: The stored value, or I{df[0]} if not set. + @rtype: any + """ + return self.properties.get(name, *df) + + def update(self, **kwargs): + """ + Update the property values as specified by keyword/value. + @param kwargs: A list of property name/values to set. + @type kwargs: dict + @return: self + @rtype: L{Properties} + """ + return self.properties.update(**kwargs) + + def link(self, other): + """ + Link (associate) this object with anI{other} properties object + to create a network of properties. Links are bidirectional. + @param other: The object to link. + @type other: L{Properties} + @return: self + @rtype: L{Properties} + """ + p = other.__pts__ + return self.properties.link(p) + + def unlink(self, other): + """ + Unlink (disassociate) the specified properties object. + @param other: The object to unlink. + @type other: L{Properties} + @return: self + @rtype: L{Properties} + """ + p = other.__pts__ + return self.properties.unlink(p) diff --git a/libs/suds/reader.py b/libs/suds/reader.py new file mode 100644 index 00000000..1184f127 --- /dev/null +++ b/libs/suds/reader.py @@ -0,0 +1,169 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains xml document reader classes. +""" + + +from suds.sax.parser import Parser +from suds.transport import Request +from suds.cache import Cache, NoCache +from suds.store import DocumentStore +from suds.plugin import PluginContainer +from logging import getLogger + + +log = getLogger(__name__) + + +class Reader: + """ + The reader provides integration with cache. + @ivar options: An options object. + @type options: I{Options} + """ + + def __init__(self, options): + """ + @param options: An options object. + @type options: I{Options} + """ + self.options = options + self.plugins = PluginContainer(options.plugins) + + def mangle(self, name, x): + """ + Mangle the name by hashing the I{name} and appending I{x}. + @return: the mangled name. + """ + h = abs(hash(name)) + return '%s-%s' % (h, x) + + +class DocumentReader(Reader): + """ + The XML document reader provides an integration + between the SAX L{Parser} and the document cache. + """ + + def open(self, url): + """ + Open an XML document at the specified I{url}. + First, the document attempted to be retrieved from + the I{object cache}. If not found, it is downloaded and + parsed using the SAX parser. The result is added to the + cache for the next open(). + @param url: A document url. + @type url: str. + @return: The specified XML document. + @rtype: I{Document} + """ + cache = self.cache() + id = self.mangle(url, 'document') + d = cache.get(id) + if d is None: + d = self.download(url) + cache.put(id, d) + self.plugins.document.parsed(url=url, document=d.root()) + return d + + def download(self, url): + """ + Download the docuemnt. + @param url: A document url. + @type url: str. + @return: A file pointer to the docuemnt. + @rtype: file-like + """ + store = DocumentStore() + fp = store.open(url) + if fp is None: + fp = self.options.transport.open(Request(url)) + content = fp.read() + fp.close() + ctx = self.plugins.document.loaded(url=url, document=content) + content = ctx.document + sax = Parser() + return sax.parse(string=content) + + def cache(self): + """ + Get the cache. + @return: The I{options} when I{cachingpolicy} = B{0}. + @rtype: L{Cache} + """ + if self.options.cachingpolicy == 0: + return self.options.cache + else: + return NoCache() + + +class DefinitionsReader(Reader): + """ + The WSDL definitions reader provides an integration + between the Definitions and the object cache. + @ivar fn: A factory function (constructor) used to + create the object not found in the cache. + @type fn: I{Constructor} + """ + + def __init__(self, options, fn): + """ + @param options: An options object. + @type options: I{Options} + @param fn: A factory function (constructor) used to + create the object not found in the cache. + @type fn: I{Constructor} + """ + Reader.__init__(self, options) + self.fn = fn + + def open(self, url): + """ + Open a WSDL at the specified I{url}. + First, the WSDL attempted to be retrieved from + the I{object cache}. After unpickled from the cache, the + I{options} attribute is restored. + If not found, it is downloaded and instantiated using the + I{fn} constructor and added to the cache for the next open(). + @param url: A WSDL url. + @type url: str. + @return: The WSDL object. + @rtype: I{Definitions} + """ + cache = self.cache() + id = self.mangle(url, 'wsdl') + d = cache.get(id) + if d is None: + d = self.fn(url, self.options) + cache.put(id, d) + else: + d.options = self.options + for imp in d.imports: + imp.imported.options = self.options + return d + + def cache(self): + """ + Get the cache. + @return: The I{options} when I{cachingpolicy} = B{1}. + @rtype: L{Cache} + """ + if self.options.cachingpolicy == 1: + return self.options.cache + else: + return NoCache() \ No newline at end of file diff --git a/libs/suds/resolver.py b/libs/suds/resolver.py new file mode 100644 index 00000000..278b5da6 --- /dev/null +++ b/libs/suds/resolver.py @@ -0,0 +1,496 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{resolver} module provides a collection of classes that +provide wsdl/xsd named type resolution. +""" + +import re +from logging import getLogger +from suds import * +from suds.sax import splitPrefix, Namespace +from suds.sudsobject import Object +from suds.xsd.query import BlindQuery, TypeQuery, qualify + +log = getLogger(__name__) + + +class Resolver: + """ + An I{abstract} schema-type resolver. + @ivar schema: A schema object. + @type schema: L{xsd.schema.Schema} + """ + + def __init__(self, schema): + """ + @param schema: A schema object. + @type schema: L{xsd.schema.Schema} + """ + self.schema = schema + + def find(self, name, resolved=True): + """ + Get the definition object for the schema object by name. + @param name: The name of a schema object. + @type name: basestring + @param resolved: A flag indicating that the fully resolved type + should be returned. + @type resolved: boolean + @return: The found schema I{type} + @rtype: L{xsd.sxbase.SchemaObject} + """ + log.debug('searching schema for (%s)', name) + qref = qualify(name, self.schema.root, self.schema.tns) + query = BlindQuery(qref) + result = query.execute(self.schema) + if result is None: + log.error('(%s) not-found', name) + return None + log.debug('found (%s) as (%s)', name, Repr(result)) + if resolved: + result = result.resolve() + return result + + +class PathResolver(Resolver): + """ + Resolveds the definition object for the schema type located at the specified path. + The path may contain (.) dot notation to specify nested types. + @ivar wsdl: A wsdl object. + @type wsdl: L{wsdl.Definitions} + """ + + def __init__(self, wsdl, ps='.'): + """ + @param wsdl: A schema object. + @type wsdl: L{wsdl.Definitions} + @param ps: The path separator character + @type ps: char + """ + Resolver.__init__(self, wsdl.schema) + self.wsdl = wsdl + self.altp = re.compile('({)(.+)(})(.+)') + self.splitp = re.compile('({.+})*[^\%s]+' % ps[0]) + + def find(self, path, resolved=True): + """ + Get the definition object for the schema type located at the specified path. + The path may contain (.) dot notation to specify nested types. + Actually, the path separator is usually a (.) but can be redefined + during contruction. + @param path: A (.) separated path to a schema type. + @type path: basestring + @param resolved: A flag indicating that the fully resolved type + should be returned. + @type resolved: boolean + @return: The found schema I{type} + @rtype: L{xsd.sxbase.SchemaObject} + """ + result = None + parts = self.split(path) + try: + result = self.root(parts) + if len(parts) > 1: + result = result.resolve(nobuiltin=True) + result = self.branch(result, parts) + result = self.leaf(result, parts) + if resolved: + result = result.resolve(nobuiltin=True) + except PathResolver.BadPath: + log.error('path: "%s", not-found' % path) + return result + + def root(self, parts): + """ + Find the path root. + @param parts: A list of path parts. + @type parts: [str,..] + @return: The root. + @rtype: L{xsd.sxbase.SchemaObject} + """ + result = None + name = parts[0] + log.debug('searching schema for (%s)', name) + qref = self.qualify(parts[0]) + query = BlindQuery(qref) + result = query.execute(self.schema) + if result is None: + log.error('(%s) not-found', name) + raise PathResolver.BadPath(name) + else: + log.debug('found (%s) as (%s)', name, Repr(result)) + return result + + def branch(self, root, parts): + """ + Traverse the path until the leaf is reached. + @param parts: A list of path parts. + @type parts: [str,..] + @param root: The root. + @type root: L{xsd.sxbase.SchemaObject} + @return: The end of the branch. + @rtype: L{xsd.sxbase.SchemaObject} + """ + result = root + for part in parts[1:-1]: + name = splitPrefix(part)[1] + log.debug('searching parent (%s) for (%s)', Repr(result), name) + result, ancestry = result.get_child(name) + if result is None: + log.error('(%s) not-found', name) + raise PathResolver.BadPath(name) + else: + result = result.resolve(nobuiltin=True) + log.debug('found (%s) as (%s)', name, Repr(result)) + return result + + def leaf(self, parent, parts): + """ + Find the leaf. + @param parts: A list of path parts. + @type parts: [str,..] + @param parent: The leaf's parent. + @type parent: L{xsd.sxbase.SchemaObject} + @return: The leaf. + @rtype: L{xsd.sxbase.SchemaObject} + """ + name = splitPrefix(parts[-1])[1] + if name.startswith('@'): + result, path = parent.get_attribute(name[1:]) + else: + result, ancestry = parent.get_child(name) + if result is None: + raise PathResolver.BadPath(name) + return result + + def qualify(self, name): + """ + Qualify the name as either: + - plain name + - ns prefixed name (eg: ns0:Person) + - fully ns qualified name (eg: {http://myns-uri}Person) + @param name: The name of an object in the schema. + @type name: str + @return: A qualifed name. + @rtype: qname + """ + m = self.altp.match(name) + if m is None: + return qualify(name, self.wsdl.root, self.wsdl.tns) + else: + return (m.group(4), m.group(2)) + + def split(self, s): + """ + Split the string on (.) while preserving any (.) inside the + '{}' alternalte syntax for full ns qualification. + @param s: A plain or qualifed name. + @type s: str + @return: A list of the name's parts. + @rtype: [str,..] + """ + parts = [] + b = 0 + while 1: + m = self.splitp.match(s, b) + if m is None: + break + b,e = m.span() + parts.append(s[b:e]) + b = e+1 + return parts + + class BadPath(Exception): pass + + +class TreeResolver(Resolver): + """ + The tree resolver is a I{stateful} tree resolver + used to resolve each node in a tree. As such, it mirrors + the tree structure to ensure that nodes are resolved in + context. + @ivar stack: The context stack. + @type stack: list + """ + + def __init__(self, schema): + """ + @param schema: A schema object. + @type schema: L{xsd.schema.Schema} + """ + Resolver.__init__(self, schema) + self.stack = Stack() + + def reset(self): + """ + Reset the resolver's state. + """ + self.stack = Stack() + + def push(self, x): + """ + Push an I{object} onto the stack. + @param x: An object to push. + @type x: L{Frame} + @return: The pushed frame. + @rtype: L{Frame} + """ + if isinstance(x, Frame): + frame = x + else: + frame = Frame(x) + self.stack.append(frame) + log.debug('push: (%s)\n%s', Repr(frame), Repr(self.stack)) + return frame + + def top(self): + """ + Get the I{frame} at the top of the stack. + @return: The top I{frame}, else None. + @rtype: L{Frame} + """ + if len(self.stack): + return self.stack[-1] + else: + return Frame.Empty() + + def pop(self): + """ + Pop the frame at the top of the stack. + @return: The popped frame, else None. + @rtype: L{Frame} + """ + if len(self.stack): + popped = self.stack.pop() + log.debug('pop: (%s)\n%s', Repr(popped), Repr(self.stack)) + return popped + else: + log.debug('stack empty, not-popped') + return None + + def depth(self): + """ + Get the current stack depth. + @return: The current stack depth. + @rtype: int + """ + return len(self.stack) + + def getchild(self, name, parent): + """ get a child by name """ + log.debug('searching parent (%s) for (%s)', Repr(parent), name) + if name.startswith('@'): + return parent.get_attribute(name[1:]) + else: + return parent.get_child(name) + + +class NodeResolver(TreeResolver): + """ + The node resolver is a I{stateful} XML document resolver + used to resolve each node in a tree. As such, it mirrors + the tree structure to ensure that nodes are resolved in + context. + """ + + def __init__(self, schema): + """ + @param schema: A schema object. + @type schema: L{xsd.schema.Schema} + """ + TreeResolver.__init__(self, schema) + + def find(self, node, resolved=False, push=True): + """ + @param node: An xml node to be resolved. + @type node: L{sax.element.Element} + @param resolved: A flag indicating that the fully resolved type should be + returned. + @type resolved: boolean + @param push: Indicates that the resolved type should be + pushed onto the stack. + @type push: boolean + @return: The found schema I{type} + @rtype: L{xsd.sxbase.SchemaObject} + """ + name = node.name + parent = self.top().resolved + if parent is None: + result, ancestry = self.query(name, node) + else: + result, ancestry = self.getchild(name, parent) + known = self.known(node) + if result is None: + return result + if push: + frame = Frame(result, resolved=known, ancestry=ancestry) + pushed = self.push(frame) + if resolved: + result = result.resolve() + return result + + def findattr(self, name, resolved=True): + """ + Find an attribute type definition. + @param name: An attribute name. + @type name: basestring + @param resolved: A flag indicating that the fully resolved type should be + returned. + @type resolved: boolean + @return: The found schema I{type} + @rtype: L{xsd.sxbase.SchemaObject} + """ + name = '@%s'%name + parent = self.top().resolved + if parent is None: + result, ancestry = self.query(name, node) + else: + result, ancestry = self.getchild(name, parent) + if result is None: + return result + if resolved: + result = result.resolve() + return result + + def query(self, name, node): + """ blindly query the schema by name """ + log.debug('searching schema for (%s)', name) + qref = qualify(name, node, node.namespace()) + query = BlindQuery(qref) + result = query.execute(self.schema) + return (result, []) + + def known(self, node): + """ resolve type referenced by @xsi:type """ + ref = node.get('type', Namespace.xsins) + if ref is None: + return None + qref = qualify(ref, node, node.namespace()) + query = BlindQuery(qref) + return query.execute(self.schema) + + +class GraphResolver(TreeResolver): + """ + The graph resolver is a I{stateful} L{Object} graph resolver + used to resolve each node in a tree. As such, it mirrors + the tree structure to ensure that nodes are resolved in + context. + """ + + def __init__(self, schema): + """ + @param schema: A schema object. + @type schema: L{xsd.schema.Schema} + """ + TreeResolver.__init__(self, schema) + + def find(self, name, object, resolved=False, push=True): + """ + @param name: The name of the object to be resolved. + @type name: basestring + @param object: The name's value. + @type object: (any|L{Object}) + @param resolved: A flag indicating that the fully resolved type + should be returned. + @type resolved: boolean + @param push: Indicates that the resolved type should be + pushed onto the stack. + @type push: boolean + @return: The found schema I{type} + @rtype: L{xsd.sxbase.SchemaObject} + """ + known = None + parent = self.top().resolved + if parent is None: + result, ancestry = self.query(name) + else: + result, ancestry = self.getchild(name, parent) + if result is None: + return None + if isinstance(object, Object): + known = self.known(object) + if push: + frame = Frame(result, resolved=known, ancestry=ancestry) + pushed = self.push(frame) + if resolved: + if known is None: + result = result.resolve() + else: + result = known + return result + + def query(self, name): + """ blindly query the schema by name """ + log.debug('searching schema for (%s)', name) + schema = self.schema + wsdl = self.wsdl() + if wsdl is None: + qref = qualify(name, schema.root, schema.tns) + else: + qref = qualify(name, wsdl.root, wsdl.tns) + query = BlindQuery(qref) + result = query.execute(schema) + return (result, []) + + def wsdl(self): + """ get the wsdl """ + container = self.schema.container + if container is None: + return None + else: + return container.wsdl + + def known(self, object): + """ get the type specified in the object's metadata """ + try: + md = object.__metadata__ + known = md.sxtype + return known + except: + pass + + +class Frame: + def __init__(self, type, resolved=None, ancestry=()): + self.type = type + if resolved is None: + resolved = type.resolve() + self.resolved = resolved.resolve() + self.ancestry = ancestry + + def __str__(self): + return '%s\n%s\n%s' % \ + (Repr(self.type), + Repr(self.resolved), + [Repr(t) for t in self.ancestry]) + + class Empty: + def __getattr__(self, name): + if name == 'ancestry': + return () + else: + return None + + +class Stack(list): + def __repr__(self): + result = [] + for item in self: + result.append(repr(item)) + return '\n'.join(result) \ No newline at end of file diff --git a/libs/suds/sax/__init__.py b/libs/suds/sax/__init__.py new file mode 100644 index 00000000..3d71432f --- /dev/null +++ b/libs/suds/sax/__init__.py @@ -0,0 +1,109 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The sax module contains a collection of classes that provide a +(D)ocument (O)bject (M)odel representation of an XML document. +The goal is to provide an easy, intuative interface for managing XML +documents. Although, the term, DOM, is used above, this model is +B{far} better. + +XML namespaces in suds are represented using a (2) element tuple +containing the prefix and the URI. Eg: I{('tns', 'http://myns')} + +@var encoder: A I{pluggable} XML special character processor used to + encode/decode strings. +@type encoder: L{Encoder} +""" + +from suds.sax.enc import Encoder + +# +# pluggable XML special character encoder. +# +encoder = Encoder() + + +def splitPrefix(name): + """ + Split the name into a tuple (I{prefix}, I{name}). The first element in + the tuple is I{None} when the name does't have a prefix. + @param name: A node name containing an optional prefix. + @type name: basestring + @return: A tuple containing the (2) parts of I{name} + @rtype: (I{prefix}, I{name}) + """ + if isinstance(name, basestring) \ + and ':' in name: + return tuple(name.split(':', 1)) + else: + return (None, name) + + +class Namespace: + """ + The namespace class represents XML namespaces. + """ + + default = (None, None) + xmlns = ('xml', 'http://www.w3.org/XML/1998/namespace') + xsdns = ('xs', 'http://www.w3.org/2001/XMLSchema') + xsins = ('xsi', 'http://www.w3.org/2001/XMLSchema-instance') + all = (xsdns, xsins) + + @classmethod + def create(cls, p=None, u=None): + return (p, u) + + @classmethod + def none(cls, ns): + return ( ns == cls.default ) + + @classmethod + def xsd(cls, ns): + try: + return cls.w3(ns) and ns[1].endswith('XMLSchema') + except: + pass + return False + + @classmethod + def xsi(cls, ns): + try: + return cls.w3(ns) and ns[1].endswith('XMLSchema-instance') + except: + pass + return False + + @classmethod + def xs(cls, ns): + return ( cls.xsd(ns) or cls.xsi(ns) ) + + @classmethod + def w3(cls, ns): + try: + return ns[1].startswith('http://www.w3.org') + except: + pass + return False + + @classmethod + def isns(cls, ns): + try: + return isinstance(ns, tuple) and len(ns) == len(cls.default) + except: + pass + return False diff --git a/libs/suds/sax/attribute.py b/libs/suds/sax/attribute.py new file mode 100644 index 00000000..86dfb111 --- /dev/null +++ b/libs/suds/sax/attribute.py @@ -0,0 +1,181 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides XML I{attribute} classes. +""" + +import suds.sax +from logging import getLogger +from suds import * +from suds.sax import * +from suds.sax.text import Text + +log = getLogger(__name__) + +class Attribute: + """ + An XML attribute object. + @ivar parent: The node containing this attribute + @type parent: L{element.Element} + @ivar prefix: The I{optional} namespace prefix. + @type prefix: basestring + @ivar name: The I{unqualified} name of the attribute + @type name: basestring + @ivar value: The attribute's value + @type value: basestring + """ + def __init__(self, name, value=None): + """ + @param name: The attribute's name with I{optional} namespace prefix. + @type name: basestring + @param value: The attribute's value + @type value: basestring + """ + self.parent = None + self.prefix, self.name = splitPrefix(name) + self.setValue(value) + + def clone(self, parent=None): + """ + Clone this object. + @param parent: The parent for the clone. + @type parent: L{element.Element} + @return: A copy of this object assigned to the new parent. + @rtype: L{Attribute} + """ + a = Attribute(self.qname(), self.value) + a.parent = parent + return a + + def qname(self): + """ + Get the B{fully} qualified name of this attribute + @return: The fully qualified name. + @rtype: basestring + """ + if self.prefix is None: + return self.name + else: + return ':'.join((self.prefix, self.name)) + + def setValue(self, value): + """ + Set the attributes value + @param value: The new value (may be None) + @type value: basestring + @return: self + @rtype: L{Attribute} + """ + if isinstance(value, Text): + self.value = value + else: + self.value = Text(value) + return self + + def getValue(self, default=Text('')): + """ + Get the attributes value with optional default. + @param default: An optional value to be return when the + attribute's has not been set. + @type default: basestring + @return: The attribute's value, or I{default} + @rtype: L{Text} + """ + if self.hasText(): + return self.value + else: + return default + + def hasText(self): + """ + Get whether the attribute has I{text} and that it is not an empty + (zero length) string. + @return: True when has I{text}. + @rtype: boolean + """ + return ( self.value is not None and len(self.value) ) + + def namespace(self): + """ + Get the attributes namespace. This may either be the namespace + defined by an optional prefix, or its parent's namespace. + @return: The attribute's namespace + @rtype: (I{prefix}, I{name}) + """ + if self.prefix is None: + return Namespace.default + else: + return self.resolvePrefix(self.prefix) + + def resolvePrefix(self, prefix): + """ + Resolve the specified prefix to a known namespace. + @param prefix: A declared prefix + @type prefix: basestring + @return: The namespace that has been mapped to I{prefix} + @rtype: (I{prefix}, I{name}) + """ + ns = Namespace.default + if self.parent is not None: + ns = self.parent.resolvePrefix(prefix) + return ns + + def match(self, name=None, ns=None): + """ + Match by (optional) name and/or (optional) namespace. + @param name: The optional attribute tag name. + @type name: str + @param ns: An optional namespace. + @type ns: (I{prefix}, I{name}) + @return: True if matched. + @rtype: boolean + """ + if name is None: + byname = True + else: + byname = ( self.name == name ) + if ns is None: + byns = True + else: + byns = ( self.namespace()[1] == ns[1] ) + return ( byname and byns ) + + def __eq__(self, rhs): + """ equals operator """ + return rhs is not None and \ + isinstance(rhs, Attribute) and \ + self.prefix == rhs.name and \ + self.name == rhs.name + + def __repr__(self): + """ get a string representation """ + return \ + 'attr (prefix=%s, name=%s, value=(%s))' %\ + (self.prefix, self.name, self.value) + + def __str__(self): + """ get an xml string representation """ + return unicode(self).encode('utf-8') + + def __unicode__(self): + """ get an xml string representation """ + n = self.qname() + if self.hasText(): + v = self.value.escape() + else: + v = self.value + return u'%s="%s"' % (n, v) diff --git a/libs/suds/sax/date.py b/libs/suds/sax/date.py new file mode 100644 index 00000000..6e31c4c7 --- /dev/null +++ b/libs/suds/sax/date.py @@ -0,0 +1,378 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Nathan Van Gheem (vangheem@gmail.com) + +""" +The I{xdate} module provides classes for converstion +between XML dates and python objects. +""" + +from logging import getLogger +from suds import * +from suds.xsd import * +import time +import datetime as dt +import re + +log = getLogger(__name__) + + +class Date: + """ + An XML date object. + Supported formats: + - YYYY-MM-DD + - YYYY-MM-DD(z|Z) + - YYYY-MM-DD+06:00 + - YYYY-MM-DD-06:00 + @ivar date: The object value. + @type date: B{datetime}.I{date} + """ + def __init__(self, date): + """ + @param date: The value of the object. + @type date: (date|str) + @raise ValueError: When I{date} is invalid. + """ + if isinstance(date, dt.date): + self.date = date + return + if isinstance(date, basestring): + self.date = self.__parse(date) + return + raise ValueError, type(date) + + def year(self): + """ + Get the I{year} component. + @return: The year. + @rtype: int + """ + return self.date.year + + def month(self): + """ + Get the I{month} component. + @return: The month. + @rtype: int + """ + return self.date.month + + def day(self): + """ + Get the I{day} component. + @return: The day. + @rtype: int + """ + return self.date.day + + def __parse(self, s): + """ + Parse the string date. + Supported formats: + - YYYY-MM-DD + - YYYY-MM-DD(z|Z) + - YYYY-MM-DD+06:00 + - YYYY-MM-DD-06:00 + Although, the TZ is ignored because it's meaningless + without the time, right? + @param s: A date string. + @type s: str + @return: A date object. + @rtype: I{date} + """ + try: + year, month, day = s[:10].split('-', 2) + year = int(year) + month = int(month) + day = int(day) + return dt.date(year, month, day) + except: + log.debug(s, exec_info=True) + raise ValueError, 'Invalid format "%s"' % s + + def __str__(self): + return unicode(self) + + def __unicode__(self): + return self.date.isoformat() + + +class Time: + """ + An XML time object. + Supported formats: + - HH:MI:SS + - HH:MI:SS(z|Z) + - HH:MI:SS.ms + - HH:MI:SS.ms(z|Z) + - HH:MI:SS(+|-)06:00 + - HH:MI:SS.ms(+|-)06:00 + @ivar tz: The timezone + @type tz: L{Timezone} + @ivar date: The object value. + @type date: B{datetime}.I{time} + """ + + def __init__(self, time, adjusted=True): + """ + @param time: The value of the object. + @type time: (time|str) + @param adjusted: Adjust for I{local} Timezone. + @type adjusted: boolean + @raise ValueError: When I{time} is invalid. + """ + self.tz = Timezone() + if isinstance(time, dt.time): + self.time = time + return + if isinstance(time, basestring): + self.time = self.__parse(time) + if adjusted: + self.__adjust() + return + raise ValueError, type(time) + + def hour(self): + """ + Get the I{hour} component. + @return: The hour. + @rtype: int + """ + return self.time.hour + + def minute(self): + """ + Get the I{minute} component. + @return: The minute. + @rtype: int + """ + return self.time.minute + + def second(self): + """ + Get the I{seconds} component. + @return: The seconds. + @rtype: int + """ + return self.time.second + + def microsecond(self): + """ + Get the I{microsecond} component. + @return: The microsecond. + @rtype: int + """ + return self.time.microsecond + + def __adjust(self): + """ + Adjust for TZ offset. + """ + if hasattr(self, 'offset'): + today = dt.date.today() + delta = self.tz.adjustment(self.offset) + d = dt.datetime.combine(today, self.time) + d = ( d + delta ) + self.time = d.time() + + def __parse(self, s): + """ + Parse the string date. + Patterns: + - HH:MI:SS + - HH:MI:SS(z|Z) + - HH:MI:SS.ms + - HH:MI:SS.ms(z|Z) + - HH:MI:SS(+|-)06:00 + - HH:MI:SS.ms(+|-)06:00 + @param s: A time string. + @type s: str + @return: A time object. + @rtype: B{datetime}.I{time} + """ + try: + offset = None + part = Timezone.split(s) + hour, minute, second = part[0].split(':', 2) + hour = int(hour) + minute = int(minute) + second, ms = self.__second(second) + if len(part) == 2: + self.offset = self.__offset(part[1]) + if ms is None: + return dt.time(hour, minute, second) + else: + return dt.time(hour, minute, second, ms) + except: + log.debug(s, exec_info=True) + raise ValueError, 'Invalid format "%s"' % s + + def __second(self, s): + """ + Parse the seconds and microseconds. + The microseconds are truncated to 999999 due to a restriction in + the python datetime.datetime object. + @param s: A string representation of the seconds. + @type s: str + @return: Tuple of (sec,ms) + @rtype: tuple. + """ + part = s.split('.') + if len(part) > 1: + return (int(part[0]), int(part[1][:6])) + else: + return (int(part[0]), None) + + def __offset(self, s): + """ + Parse the TZ offset. + @param s: A string representation of the TZ offset. + @type s: str + @return: The signed offset in hours. + @rtype: str + """ + if len(s) == len('-00:00'): + return int(s[:3]) + if len(s) == 0: + return self.tz.local + if len(s) == 1: + return 0 + raise Exception() + + def __str__(self): + return unicode(self) + + def __unicode__(self): + time = self.time.isoformat() + if self.tz.local: + return '%s%+.2d:00' % (time, self.tz.local) + else: + return '%sZ' % time + + +class DateTime(Date,Time): + """ + An XML time object. + Supported formats: + - YYYY-MM-DDB{T}HH:MI:SS + - YYYY-MM-DDB{T}HH:MI:SS(z|Z) + - YYYY-MM-DDB{T}HH:MI:SS.ms + - YYYY-MM-DDB{T}HH:MI:SS.ms(z|Z) + - YYYY-MM-DDB{T}HH:MI:SS(+|-)06:00 + - YYYY-MM-DDB{T}HH:MI:SS.ms(+|-)06:00 + @ivar datetime: The object value. + @type datetime: B{datetime}.I{datedate} + """ + def __init__(self, date): + """ + @param date: The value of the object. + @type date: (datetime|str) + @raise ValueError: When I{tm} is invalid. + """ + if isinstance(date, dt.datetime): + Date.__init__(self, date.date()) + Time.__init__(self, date.time()) + self.datetime = \ + dt.datetime.combine(self.date, self.time) + return + if isinstance(date, basestring): + part = date.split('T') + Date.__init__(self, part[0]) + Time.__init__(self, part[1], 0) + self.datetime = \ + dt.datetime.combine(self.date, self.time) + self.__adjust() + return + raise ValueError, type(date) + + def __adjust(self): + """ + Adjust for TZ offset. + """ + if not hasattr(self, 'offset'): + return + delta = self.tz.adjustment(self.offset) + try: + d = ( self.datetime + delta ) + self.datetime = d + self.date = d.date() + self.time = d.time() + except OverflowError: + log.warn('"%s" caused overflow, not-adjusted', self.datetime) + + def __str__(self): + return unicode(self) + + def __unicode__(self): + s = [] + s.append(Date.__unicode__(self)) + s.append(Time.__unicode__(self)) + return 'T'.join(s) + + +class UTC(DateTime): + """ + Represents current UTC time. + """ + + def __init__(self, date=None): + if date is None: + date = dt.datetime.utcnow() + DateTime.__init__(self, date) + self.tz.local = 0 + + +class Timezone: + """ + Timezone object used to do TZ conversions + @cvar local: The (A) local TZ offset. + @type local: int + @cvar patten: The regex patten to match TZ. + @type patten: re.Pattern + """ + + pattern = re.compile('([zZ])|([\-\+][0-9]{2}:[0-9]{2})') + + LOCAL = ( 0-time.timezone/60/60 ) + + def __init__(self, offset=None): + if offset is None: + offset = self.LOCAL + self.local = offset + + @classmethod + def split(cls, s): + """ + Split the TZ from string. + @param s: A string containing a timezone + @type s: basestring + @return: The split parts. + @rtype: tuple + """ + m = cls.pattern.search(s) + if m is None: + return (s,) + x = m.start(0) + return (s[:x], s[x:]) + + def adjustment(self, offset): + """ + Get the adjustment to the I{local} TZ. + @return: The delta between I{offset} and local TZ. + @rtype: B{datetime}.I{timedelta} + """ + delta = ( self.local - offset ) + return dt.timedelta(hours=delta) diff --git a/libs/suds/sax/document.py b/libs/suds/sax/document.py new file mode 100644 index 00000000..5a004eb5 --- /dev/null +++ b/libs/suds/sax/document.py @@ -0,0 +1,61 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides XML I{document} classes. +""" + +from logging import getLogger +from suds import * +from suds.sax import * +from suds.sax.element import Element + +log = getLogger(__name__) + +class Document(Element): + """ simple document """ + + DECL = '' + + def __init__(self, root=None): + Element.__init__(self, 'document') + if root is not None: + self.append(root) + + def root(self): + if len(self.children): + return self.children[0] + else: + return None + + def str(self): + s = [] + s.append(self.DECL) + s.append('\n') + s.append(self.root().str()) + return ''.join(s) + + def plain(self): + s = [] + s.append(self.DECL) + s.append(self.root().plain()) + return ''.join(s) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + return self.str() \ No newline at end of file diff --git a/libs/suds/sax/element.py b/libs/suds/sax/element.py new file mode 100644 index 00000000..9dec1f94 --- /dev/null +++ b/libs/suds/sax/element.py @@ -0,0 +1,1147 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides XML I{element} classes. +""" + +from logging import getLogger +from suds import * +from suds.sax import * +from suds.sax.text import Text +from suds.sax.attribute import Attribute +import sys +if sys.version_info < (2, 4, 0): + from sets import Set as set + del sys + +log = getLogger(__name__) + +class Element: + """ + An XML element object. + @ivar parent: The node containing this attribute + @type parent: L{Element} + @ivar prefix: The I{optional} namespace prefix. + @type prefix: basestring + @ivar name: The I{unqualified} name of the attribute + @type name: basestring + @ivar expns: An explicit namespace (xmlns="..."). + @type expns: (I{prefix}, I{name}) + @ivar nsprefixes: A mapping of prefixes to namespaces. + @type nsprefixes: dict + @ivar attributes: A list of XML attributes. + @type attributes: [I{Attribute},] + @ivar text: The element's I{text} content. + @type text: basestring + @ivar children: A list of child elements. + @type children: [I{Element},] + @cvar matcher: A collection of I{lambda} for string matching. + @cvar specialprefixes: A dictionary of builtin-special prefixes. + """ + + matcher = \ + { + 'eq': lambda a,b: a == b, + 'startswith' : lambda a,b: a.startswith(b), + 'endswith' : lambda a,b: a.endswith(b), + 'contains' : lambda a,b: b in a + } + + specialprefixes = { Namespace.xmlns[0] : Namespace.xmlns[1] } + + @classmethod + def buildPath(self, parent, path): + """ + Build the specifed pat as a/b/c where missing intermediate nodes are built + automatically. + @param parent: A parent element on which the path is built. + @type parent: I{Element} + @param path: A simple path separated by (/). + @type path: basestring + @return: The leaf node of I{path}. + @rtype: L{Element} + """ + for tag in path.split('/'): + child = parent.getChild(tag) + if child is None: + child = Element(tag, parent) + parent = child + return child + + def __init__(self, name, parent=None, ns=None): + """ + @param name: The element's (tag) name. May cotain a prefix. + @type name: basestring + @param parent: An optional parent element. + @type parent: I{Element} + @param ns: An optional namespace + @type ns: (I{prefix}, I{name}) + """ + + self.rename(name) + self.expns = None + self.nsprefixes = {} + self.attributes = [] + self.text = None + if parent is not None: + if isinstance(parent, Element): + self.parent = parent + else: + raise Exception('parent (%s) not-valid', parent.__class__.__name__) + else: + self.parent = None + self.children = [] + self.applyns(ns) + + def rename(self, name): + """ + Rename the element. + @param name: A new name for the element. + @type name: basestring + """ + if name is None: + raise Exception('name (%s) not-valid' % name) + else: + self.prefix, self.name = splitPrefix(name) + + def setPrefix(self, p, u=None): + """ + Set the element namespace prefix. + @param p: A new prefix for the element. + @type p: basestring + @param u: A namespace URI to be mapped to the prefix. + @type u: basestring + @return: self + @rtype: L{Element} + """ + self.prefix = p + if p is not None and u is not None: + self.addPrefix(p, u) + return self + + def qname(self): + """ + Get the B{fully} qualified name of this element + @return: The fully qualified name. + @rtype: basestring + """ + if self.prefix is None: + return self.name + else: + return '%s:%s' % (self.prefix, self.name) + + def getRoot(self): + """ + Get the root (top) node of the tree. + @return: The I{top} node of this tree. + @rtype: I{Element} + """ + if self.parent is None: + return self + else: + return self.parent.getRoot() + + def clone(self, parent=None): + """ + Deep clone of this element and children. + @param parent: An optional parent for the copied fragment. + @type parent: I{Element} + @return: A deep copy parented by I{parent} + @rtype: I{Element} + """ + root = Element(self.qname(), parent, self.namespace()) + for a in self.attributes: + root.append(a.clone(self)) + for c in self.children: + root.append(c.clone(self)) + for item in self.nsprefixes.items(): + root.addPrefix(item[0], item[1]) + return root + + def detach(self): + """ + Detach from parent. + @return: This element removed from its parent's + child list and I{parent}=I{None} + @rtype: L{Element} + """ + if self.parent is not None: + if self in self.parent.children: + self.parent.children.remove(self) + self.parent = None + return self + + def set(self, name, value): + """ + Set an attribute's value. + @param name: The name of the attribute. + @type name: basestring + @param value: The attribute value. + @type value: basestring + @see: __setitem__() + """ + attr = self.getAttribute(name) + if attr is None: + attr = Attribute(name, value) + self.append(attr) + else: + attr.setValue(value) + + def unset(self, name): + """ + Unset (remove) an attribute. + @param name: The attribute name. + @type name: str + @return: self + @rtype: L{Element} + """ + try: + attr = self.getAttribute(name) + self.attributes.remove(attr) + except: + pass + return self + + + def get(self, name, ns=None, default=None): + """ + Get the value of an attribute by name. + @param name: The name of the attribute. + @type name: basestring + @param ns: The optional attribute's namespace. + @type ns: (I{prefix}, I{name}) + @param default: An optional value to be returned when either + the attribute does not exist of has not value. + @type default: basestring + @return: The attribute's value or I{default} + @rtype: basestring + @see: __getitem__() + """ + attr = self.getAttribute(name, ns) + if attr is None or attr.value is None: + return default + else: + return attr.getValue() + + def setText(self, value): + """ + Set the element's L{Text} content. + @param value: The element's text value. + @type value: basestring + @return: self + @rtype: I{Element} + """ + if isinstance(value, Text): + self.text = value + else: + self.text = Text(value) + return self + + def getText(self, default=None): + """ + Get the element's L{Text} content with optional default + @param default: A value to be returned when no text content exists. + @type default: basestring + @return: The text content, or I{default} + @rtype: L{Text} + """ + if self.hasText(): + return self.text + else: + return default + + def trim(self): + """ + Trim leading and trailing whitespace. + @return: self + @rtype: L{Element} + """ + if self.hasText(): + self.text = self.text.trim() + return self + + def hasText(self): + """ + Get whether the element has I{text} and that it is not an empty + (zero length) string. + @return: True when has I{text}. + @rtype: boolean + """ + return ( self.text is not None and len(self.text) ) + + def namespace(self): + """ + Get the element's namespace. + @return: The element's namespace by resolving the prefix, the explicit + namespace or the inherited namespace. + @rtype: (I{prefix}, I{name}) + """ + if self.prefix is None: + return self.defaultNamespace() + else: + return self.resolvePrefix(self.prefix) + + def defaultNamespace(self): + """ + Get the default (unqualified namespace). + This is the expns of the first node (looking up the tree) + that has it set. + @return: The namespace of a node when not qualified. + @rtype: (I{prefix}, I{name}) + """ + p = self + while p is not None: + if p.expns is not None: + return (None, p.expns) + else: + p = p.parent + return Namespace.default + + def append(self, objects): + """ + Append the specified child based on whether it is an + element or an attrbuite. + @param objects: A (single|collection) of attribute(s) or element(s) + to be added as children. + @type objects: (L{Element}|L{Attribute}) + @return: self + @rtype: L{Element} + """ + if not isinstance(objects, (list, tuple)): + objects = (objects,) + for child in objects: + if isinstance(child, Element): + self.children.append(child) + child.parent = self + continue + if isinstance(child, Attribute): + self.attributes.append(child) + child.parent = self + continue + raise Exception('append %s not-valid' % child.__class__.__name__) + return self + + def insert(self, objects, index=0): + """ + Insert an L{Element} content at the specified index. + @param objects: A (single|collection) of attribute(s) or element(s) + to be added as children. + @type objects: (L{Element}|L{Attribute}) + @param index: The position in the list of children to insert. + @type index: int + @return: self + @rtype: L{Element} + """ + objects = (objects,) + for child in objects: + if isinstance(child, Element): + self.children.insert(index, child) + child.parent = self + else: + raise Exception('append %s not-valid' % child.__class__.__name__) + return self + + def remove(self, child): + """ + Remove the specified child element or attribute. + @param child: A child to remove. + @type child: L{Element}|L{Attribute} + @return: The detached I{child} when I{child} is an element, else None. + @rtype: L{Element}|None + """ + if isinstance(child, Element): + return child.detach() + if isinstance(child, Attribute): + self.attributes.remove(child) + return None + + def replaceChild(self, child, content): + """ + Replace I{child} with the specified I{content}. + @param child: A child element. + @type child: L{Element} + @param content: An element or collection of elements. + @type content: L{Element} or [L{Element},] + """ + if child not in self.children: + raise Exception('child not-found') + index = self.children.index(child) + self.remove(child) + if not isinstance(content, (list, tuple)): + content = (content,) + for node in content: + self.children.insert(index, node.detach()) + node.parent = self + index += 1 + + def getAttribute(self, name, ns=None, default=None): + """ + Get an attribute by name and (optional) namespace + @param name: The name of a contained attribute (may contain prefix). + @type name: basestring + @param ns: An optional namespace + @type ns: (I{prefix}, I{name}) + @param default: Returned when attribute not-found. + @type default: L{Attribute} + @return: The requested attribute object. + @rtype: L{Attribute} + """ + if ns is None: + prefix, name = splitPrefix(name) + if prefix is None: + ns = None + else: + ns = self.resolvePrefix(prefix) + for a in self.attributes: + if a.match(name, ns): + return a + return default + + def getChild(self, name, ns=None, default=None): + """ + Get a child by (optional) name and/or (optional) namespace. + @param name: The name of a child element (may contain prefix). + @type name: basestring + @param ns: An optional namespace used to match the child. + @type ns: (I{prefix}, I{name}) + @param default: Returned when child not-found. + @type default: L{Element} + @return: The requested child, or I{default} when not-found. + @rtype: L{Element} + """ + if ns is None: + prefix, name = splitPrefix(name) + if prefix is None: + ns = None + else: + ns = self.resolvePrefix(prefix) + for c in self.children: + if c.match(name, ns): + return c + return default + + def childAtPath(self, path): + """ + Get a child at I{path} where I{path} is a (/) separated + list of element names that are expected to be children. + @param path: A (/) separated list of element names. + @type path: basestring + @return: The leaf node at the end of I{path} + @rtype: L{Element} + """ + result = None + node = self + for name in [p for p in path.split('/') if len(p) > 0]: + ns = None + prefix, name = splitPrefix(name) + if prefix is not None: + ns = node.resolvePrefix(prefix) + result = node.getChild(name, ns) + if result is None: + break; + else: + node = result + return result + + def childrenAtPath(self, path): + """ + Get a list of children at I{path} where I{path} is a (/) separated + list of element names that are expected to be children. + @param path: A (/) separated list of element names. + @type path: basestring + @return: The collection leaf nodes at the end of I{path} + @rtype: [L{Element},...] + """ + parts = [p for p in path.split('/') if len(p) > 0] + if len(parts) == 1: + result = self.getChildren(path) + else: + result = self.__childrenAtPath(parts) + return result + + def getChildren(self, name=None, ns=None): + """ + Get a list of children by (optional) name and/or (optional) namespace. + @param name: The name of a child element (may contain prefix). + @type name: basestring + @param ns: An optional namespace used to match the child. + @type ns: (I{prefix}, I{name}) + @return: The list of matching children. + @rtype: [L{Element},...] + """ + if ns is None: + if name is None: + return self.children + prefix, name = splitPrefix(name) + if prefix is None: + ns = None + else: + ns = self.resolvePrefix(prefix) + return [c for c in self.children if c.match(name, ns)] + + def detachChildren(self): + """ + Detach and return this element's children. + @return: The element's children (detached). + @rtype: [L{Element},...] + """ + detached = self.children + self.children = [] + for child in detached: + child.parent = None + return detached + + def resolvePrefix(self, prefix, default=Namespace.default): + """ + Resolve the specified prefix to a namespace. The I{nsprefixes} is + searched. If not found, it walks up the tree until either resolved or + the top of the tree is reached. Searching up the tree provides for + inherited mappings. + @param prefix: A namespace prefix to resolve. + @type prefix: basestring + @param default: An optional value to be returned when the prefix + cannot be resolved. + @type default: (I{prefix},I{URI}) + @return: The namespace that is mapped to I{prefix} in this context. + @rtype: (I{prefix},I{URI}) + """ + n = self + while n is not None: + if prefix in n.nsprefixes: + return (prefix, n.nsprefixes[prefix]) + if prefix in self.specialprefixes: + return (prefix, self.specialprefixes[prefix]) + n = n.parent + return default + + def addPrefix(self, p, u): + """ + Add or update a prefix mapping. + @param p: A prefix. + @type p: basestring + @param u: A namespace URI. + @type u: basestring + @return: self + @rtype: L{Element} + """ + self.nsprefixes[p] = u + return self + + def updatePrefix(self, p, u): + """ + Update (redefine) a prefix mapping for the branch. + @param p: A prefix. + @type p: basestring + @param u: A namespace URI. + @type u: basestring + @return: self + @rtype: L{Element} + @note: This method traverses down the entire branch! + """ + if p in self.nsprefixes: + self.nsprefixes[p] = u + for c in self.children: + c.updatePrefix(p, u) + return self + + def clearPrefix(self, prefix): + """ + Clear the specified prefix from the prefix mappings. + @param prefix: A prefix to clear. + @type prefix: basestring + @return: self + @rtype: L{Element} + """ + if prefix in self.nsprefixes: + del self.nsprefixes[prefix] + return self + + def findPrefix(self, uri, default=None): + """ + Find the first prefix that has been mapped to a namespace URI. + The local mapping is searched, then it walks up the tree until + it reaches the top or finds a match. + @param uri: A namespace URI. + @type uri: basestring + @param default: A default prefix when not found. + @type default: basestring + @return: A mapped prefix. + @rtype: basestring + """ + for item in self.nsprefixes.items(): + if item[1] == uri: + prefix = item[0] + return prefix + for item in self.specialprefixes.items(): + if item[1] == uri: + prefix = item[0] + return prefix + if self.parent is not None: + return self.parent.findPrefix(uri, default) + else: + return default + + def findPrefixes(self, uri, match='eq'): + """ + Find all prefixes that has been mapped to a namespace URI. + The local mapping is searched, then it walks up the tree until + it reaches the top collecting all matches. + @param uri: A namespace URI. + @type uri: basestring + @param match: A matching function L{Element.matcher}. + @type match: basestring + @return: A list of mapped prefixes. + @rtype: [basestring,...] + """ + result = [] + for item in self.nsprefixes.items(): + if self.matcher[match](item[1], uri): + prefix = item[0] + result.append(prefix) + for item in self.specialprefixes.items(): + if self.matcher[match](item[1], uri): + prefix = item[0] + result.append(prefix) + if self.parent is not None: + result += self.parent.findPrefixes(uri, match) + return result + + def promotePrefixes(self): + """ + Push prefix declarations up the tree as far as possible. Prefix + mapping are pushed to its parent unless the parent has the + prefix mapped to another URI or the parent has the prefix. + This is propagated up the tree until the top is reached. + @return: self + @rtype: L{Element} + """ + for c in self.children: + c.promotePrefixes() + if self.parent is None: + return + for p,u in self.nsprefixes.items(): + if p in self.parent.nsprefixes: + pu = self.parent.nsprefixes[p] + if pu == u: + del self.nsprefixes[p] + continue + if p != self.parent.prefix: + self.parent.nsprefixes[p] = u + del self.nsprefixes[p] + return self + + def refitPrefixes(self): + """ + Refit namespace qualification by replacing prefixes + with explicit namespaces. Also purges prefix mapping table. + @return: self + @rtype: L{Element} + """ + for c in self.children: + c.refitPrefixes() + if self.prefix is not None: + ns = self.resolvePrefix(self.prefix) + if ns[1] is not None: + self.expns = ns[1] + self.prefix = None + self.nsprefixes = {} + return self + + def normalizePrefixes(self): + """ + Normalize the namespace prefixes. + This generates unique prefixes for all namespaces. Then retrofits all + prefixes and prefix mappings. Further, it will retrofix attribute values + that have values containing (:). + @return: self + @rtype: L{Element} + """ + PrefixNormalizer.apply(self) + return self + + def isempty(self, content=True): + """ + Get whether the element has no children. + @param content: Test content (children & text) only. + @type content: boolean + @return: True when element has not children. + @rtype: boolean + """ + noattrs = not len(self.attributes) + nochildren = not len(self.children) + notext = ( self.text is None ) + nocontent = ( nochildren and notext ) + if content: + return nocontent + else: + return ( nocontent and noattrs ) + + + def isnil(self): + """ + Get whether the element is I{nil} as defined by having + an attribute in the I{xsi:nil="true"} + @return: True if I{nil}, else False + @rtype: boolean + """ + nilattr = self.getAttribute('nil', ns=Namespace.xsins) + if nilattr is None: + return False + else: + return ( nilattr.getValue().lower() == 'true' ) + + def setnil(self, flag=True): + """ + Set this node to I{nil} as defined by having an + attribute I{xsi:nil}=I{flag}. + @param flag: A flag inidcating how I{xsi:nil} will be set. + @type flag: boolean + @return: self + @rtype: L{Element} + """ + p, u = Namespace.xsins + name = ':'.join((p, 'nil')) + self.set(name, str(flag).lower()) + self.addPrefix(p, u) + if flag: + self.text = None + return self + + def applyns(self, ns): + """ + Apply the namespace to this node. If the prefix is I{None} then + this element's explicit namespace I{expns} is set to the + URI defined by I{ns}. Otherwise, the I{ns} is simply mapped. + @param ns: A namespace. + @type ns: (I{prefix},I{URI}) + """ + if ns is None: + return + if not isinstance(ns, (tuple,list)): + raise Exception('namespace must be tuple') + if ns[0] is None: + self.expns = ns[1] + else: + self.prefix = ns[0] + self.nsprefixes[ns[0]] = ns[1] + + def str(self, indent=0): + """ + Get a string representation of this XML fragment. + @param indent: The indent to be used in formatting the output. + @type indent: int + @return: A I{pretty} string. + @rtype: basestring + """ + tab = '%*s'%(indent*3,'') + result = [] + result.append('%s<%s' % (tab, self.qname())) + result.append(self.nsdeclarations()) + for a in [unicode(a) for a in self.attributes]: + result.append(' %s' % a) + if self.isempty(): + result.append('/>') + return ''.join(result) + result.append('>') + if self.hasText(): + result.append(self.text.escape()) + for c in self.children: + result.append('\n') + result.append(c.str(indent+1)) + if len(self.children): + result.append('\n%s' % tab) + result.append('' % self.qname()) + result = ''.join(result) + return result + + def plain(self): + """ + Get a string representation of this XML fragment. + @return: A I{plain} string. + @rtype: basestring + """ + result = [] + result.append('<%s' % self.qname()) + result.append(self.nsdeclarations()) + for a in [unicode(a) for a in self.attributes]: + result.append(' %s' % a) + if self.isempty(): + result.append('/>') + return ''.join(result) + result.append('>') + if self.hasText(): + result.append(self.text.escape()) + for c in self.children: + result.append(c.plain()) + result.append('' % self.qname()) + result = ''.join(result) + return result + + def nsdeclarations(self): + """ + Get a string representation for all namespace declarations + as xmlns="" and xmlns:p="". + @return: A separated list of declarations. + @rtype: basestring + """ + s = [] + myns = (None, self.expns) + if self.parent is None: + pns = Namespace.default + else: + pns = (None, self.parent.expns) + if myns[1] != pns[1]: + if self.expns is not None: + d = ' xmlns="%s"' % self.expns + s.append(d) + for item in self.nsprefixes.items(): + (p,u) = item + if self.parent is not None: + ns = self.parent.resolvePrefix(p) + if ns[1] == u: continue + d = ' xmlns:%s="%s"' % (p, u) + s.append(d) + return ''.join(s) + + def match(self, name=None, ns=None): + """ + Match by (optional) name and/or (optional) namespace. + @param name: The optional element tag name. + @type name: str + @param ns: An optional namespace. + @type ns: (I{prefix}, I{name}) + @return: True if matched. + @rtype: boolean + """ + if name is None: + byname = True + else: + byname = ( self.name == name ) + if ns is None: + byns = True + else: + byns = ( self.namespace()[1] == ns[1] ) + return ( byname and byns ) + + def branch(self): + """ + Get a flattened representation of the branch. + @return: A flat list of nodes. + @rtype: [L{Element},..] + """ + branch = [self] + for c in self.children: + branch += c.branch() + return branch + + def ancestors(self): + """ + Get a list of ancestors. + @return: A list of ancestors. + @rtype: [L{Element},..] + """ + ancestors = [] + p = self.parent + while p is not None: + ancestors.append(p) + p = p.parent + return ancestors + + def walk(self, visitor): + """ + Walk the branch and call the visitor function + on each node. + @param visitor: A function. + @return: self + @rtype: L{Element} + """ + visitor(self) + for c in self.children: + c.walk(visitor) + return self + + def prune(self): + """ + Prune the branch of empty nodes. + """ + pruned = [] + for c in self.children: + c.prune() + if c.isempty(False): + pruned.append(c) + for p in pruned: + self.children.remove(p) + + + def __childrenAtPath(self, parts): + result = [] + node = self + last = len(parts)-1 + ancestors = parts[:last] + leaf = parts[last] + for name in ancestors: + ns = None + prefix, name = splitPrefix(name) + if prefix is not None: + ns = node.resolvePrefix(prefix) + child = node.getChild(name, ns) + if child is None: + break + else: + node = child + if child is not None: + ns = None + prefix, leaf = splitPrefix(leaf) + if prefix is not None: + ns = node.resolvePrefix(prefix) + result = child.getChildren(leaf) + return result + + def __len__(self): + return len(self.children) + + def __getitem__(self, index): + if isinstance(index, basestring): + return self.get(index) + else: + if index < len(self.children): + return self.children[index] + else: + return None + + def __setitem__(self, index, value): + if isinstance(index, basestring): + self.set(index, value) + else: + if index < len(self.children) and \ + isinstance(value, Element): + self.children.insert(index, value) + + def __eq__(self, rhs): + return rhs is not None and \ + isinstance(rhs, Element) and \ + self.name == rhs.name and \ + self.namespace()[1] == rhs.namespace()[1] + + def __repr__(self): + return \ + 'Element (prefix=%s, name=%s)' % (self.prefix, self.name) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + return self.str() + + def __iter__(self): + return NodeIterator(self) + + +class NodeIterator: + """ + The L{Element} child node iterator. + @ivar pos: The current position + @type pos: int + @ivar children: A list of a child nodes. + @type children: [L{Element},..] + """ + + def __init__(self, parent): + """ + @param parent: An element to iterate. + @type parent: L{Element} + """ + self.pos = 0 + self.children = parent.children + + def next(self): + """ + Get the next child. + @return: The next child. + @rtype: L{Element} + @raise StopIterator: At the end. + """ + try: + child = self.children[self.pos] + self.pos += 1 + return child + except: + raise StopIteration() + + +class PrefixNormalizer: + """ + The prefix normalizer provides namespace prefix normalization. + @ivar node: A node to normalize. + @type node: L{Element} + @ivar branch: The nodes flattened branch. + @type branch: [L{Element},..] + @ivar namespaces: A unique list of namespaces (URI). + @type namespaces: [str,] + @ivar prefixes: A reverse dict of prefixes. + @type prefixes: {u, p} + """ + + @classmethod + def apply(cls, node): + """ + Normalize the specified node. + @param node: A node to normalize. + @type node: L{Element} + @return: The normalized node. + @rtype: L{Element} + """ + pn = PrefixNormalizer(node) + return pn.refit() + + def __init__(self, node): + """ + @param node: A node to normalize. + @type node: L{Element} + """ + self.node = node + self.branch = node.branch() + self.namespaces = self.getNamespaces() + self.prefixes = self.genPrefixes() + + def getNamespaces(self): + """ + Get the I{unique} set of namespaces referenced in the branch. + @return: A set of namespaces. + @rtype: set + """ + s = set() + for n in self.branch + self.node.ancestors(): + if self.permit(n.expns): + s.add(n.expns) + s = s.union(self.pset(n)) + return s + + def pset(self, n): + """ + Convert the nodes nsprefixes into a set. + @param n: A node. + @type n: L{Element} + @return: A set of namespaces. + @rtype: set + """ + s = set() + for ns in n.nsprefixes.items(): + if self.permit(ns): + s.add(ns[1]) + return s + + def genPrefixes(self): + """ + Generate a I{reverse} mapping of unique prefixes for all namespaces. + @return: A referse dict of prefixes. + @rtype: {u, p} + """ + prefixes = {} + n = 0 + for u in self.namespaces: + p = 'ns%d' % n + prefixes[u] = p + n += 1 + return prefixes + + def refit(self): + """ + Refit (normalize) the prefixes in the node. + """ + self.refitNodes() + self.refitMappings() + + def refitNodes(self): + """ + Refit (normalize) all of the nodes in the branch. + """ + for n in self.branch: + if n.prefix is not None: + ns = n.namespace() + if self.permit(ns): + n.prefix = self.prefixes[ns[1]] + self.refitAttrs(n) + + def refitAttrs(self, n): + """ + Refit (normalize) all of the attributes in the node. + @param n: A node. + @type n: L{Element} + """ + for a in n.attributes: + self.refitAddr(a) + + def refitAddr(self, a): + """ + Refit (normalize) the attribute. + @param a: An attribute. + @type a: L{Attribute} + """ + if a.prefix is not None: + ns = a.namespace() + if self.permit(ns): + a.prefix = self.prefixes[ns[1]] + self.refitValue(a) + + def refitValue(self, a): + """ + Refit (normalize) the attribute's value. + @param a: An attribute. + @type a: L{Attribute} + """ + p,name = splitPrefix(a.getValue()) + if p is None: return + ns = a.resolvePrefix(p) + if self.permit(ns): + u = ns[1] + p = self.prefixes[u] + a.setValue(':'.join((p, name))) + + def refitMappings(self): + """ + Refit (normalize) all of the nsprefix mappings. + """ + for n in self.branch: + n.nsprefixes = {} + n = self.node + for u, p in self.prefixes.items(): + n.addPrefix(p, u) + + def permit(self, ns): + """ + Get whether the I{ns} is to be normalized. + @param ns: A namespace. + @type ns: (p,u) + @return: True if to be included. + @rtype: boolean + """ + return not self.skip(ns) + + def skip(self, ns): + """ + Get whether the I{ns} is to B{not} be normalized. + @param ns: A namespace. + @type ns: (p,u) + @return: True if to be skipped. + @rtype: boolean + """ + return ns is None or \ + ( ns == Namespace.default ) or \ + ( ns == Namespace.xsdns ) or \ + ( ns == Namespace.xsins) or \ + ( ns == Namespace.xmlns ) \ No newline at end of file diff --git a/libs/suds/sax/enc.py b/libs/suds/sax/enc.py new file mode 100644 index 00000000..efc72744 --- /dev/null +++ b/libs/suds/sax/enc.py @@ -0,0 +1,79 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides XML I{special character} encoder classes. +""" + +import re + +class Encoder: + """ + An XML special character encoder/decoder. + @cvar encodings: A mapping of special characters encoding. + @type encodings: [(str,str)] + @cvar decodings: A mapping of special characters decoding. + @type decodings: [(str,str)] + @cvar special: A list of special characters + @type special: [char] + """ + + encodings = \ + (( '&(?!(amp|lt|gt|quot|apos);)', '&' ),( '<', '<' ),( '>', '>' ),( '"', '"' ),("'", ''' )) + decodings = \ + (( '<', '<' ),( '>', '>' ),( '"', '"' ),( ''', "'" ),( '&', '&' )) + special = \ + ('&', '<', '>', '"', "'") + + def needsEncoding(self, s): + """ + Get whether string I{s} contains special characters. + @param s: A string to check. + @type s: str + @return: True if needs encoding. + @rtype: boolean + """ + if isinstance(s, basestring): + for c in self.special: + if c in s: + return True + return False + + def encode(self, s): + """ + Encode special characters found in string I{s}. + @param s: A string to encode. + @type s: str + @return: The encoded string. + @rtype: str + """ + if isinstance(s, basestring) and self.needsEncoding(s): + for x in self.encodings: + s = re.sub(x[0], x[1], s) + return s + + def decode(self, s): + """ + Decode special characters encodings found in string I{s}. + @param s: A string to decode. + @type s: str + @return: The decoded string. + @rtype: str + """ + if isinstance(s, basestring) and '&' in s: + for x in self.decodings: + s = s.replace(x[0], x[1]) + return s diff --git a/libs/suds/sax/parser.py b/libs/suds/sax/parser.py new file mode 100644 index 00000000..69f871b4 --- /dev/null +++ b/libs/suds/sax/parser.py @@ -0,0 +1,139 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The sax module contains a collection of classes that provide a +(D)ocument (O)bject (M)odel representation of an XML document. +The goal is to provide an easy, intuative interface for managing XML +documents. Although, the term, DOM, is used above, this model is +B{far} better. + +XML namespaces in suds are represented using a (2) element tuple +containing the prefix and the URI. Eg: I{('tns', 'http://myns')} + +""" + +from logging import getLogger +import suds.metrics +from suds import * +from suds.sax import * +from suds.sax.document import Document +from suds.sax.element import Element +from suds.sax.text import Text +from suds.sax.attribute import Attribute +from xml.sax import make_parser, InputSource, ContentHandler +from xml.sax.handler import feature_external_ges +from cStringIO import StringIO + +log = getLogger(__name__) + + +class Handler(ContentHandler): + """ sax hanlder """ + + def __init__(self): + self.nodes = [Document()] + + def startElement(self, name, attrs): + top = self.top() + node = Element(unicode(name), parent=top) + for a in attrs.getNames(): + n = unicode(a) + v = unicode(attrs.getValue(a)) + attribute = Attribute(n,v) + if self.mapPrefix(node, attribute): + continue + node.append(attribute) + node.charbuffer = [] + top.append(node) + self.push(node) + + def mapPrefix(self, node, attribute): + skip = False + if attribute.name == 'xmlns': + if len(attribute.value): + node.expns = unicode(attribute.value) + skip = True + elif attribute.prefix == 'xmlns': + prefix = attribute.name + node.nsprefixes[prefix] = unicode(attribute.value) + skip = True + return skip + + def endElement(self, name): + name = unicode(name) + current = self.top() + if len(current.charbuffer): + current.text = Text(u''.join(current.charbuffer)) + del current.charbuffer + if len(current): + current.trim() + currentqname = current.qname() + if name == currentqname: + self.pop() + else: + raise Exception('malformed document') + + def characters(self, content): + text = unicode(content) + node = self.top() + node.charbuffer.append(text) + + def push(self, node): + self.nodes.append(node) + return node + + def pop(self): + return self.nodes.pop() + + def top(self): + return self.nodes[len(self.nodes)-1] + + +class Parser: + """ SAX Parser """ + + @classmethod + def saxparser(cls): + p = make_parser() + p.setFeature(feature_external_ges, 0) + h = Handler() + p.setContentHandler(h) + return (p, h) + + def parse(self, file=None, string=None): + """ + SAX parse XML text. + @param file: Parse a python I{file-like} object. + @type file: I{file-like} object. + @param string: Parse string XML. + @type string: str + """ + timer = metrics.Timer() + timer.start() + sax, handler = self.saxparser() + if file is not None: + sax.parse(file) + timer.stop() + metrics.log.debug('sax (%s) duration: %s', file, timer) + return handler.nodes[0] + if string is not None: + source = InputSource(None) + source.setByteStream(StringIO(string)) + sax.parse(source) + timer.stop() + metrics.log.debug('%s\nsax duration: %s', string, timer) + return handler.nodes[0] \ No newline at end of file diff --git a/libs/suds/sax/text.py b/libs/suds/sax/text.py new file mode 100644 index 00000000..0d58ee82 --- /dev/null +++ b/libs/suds/sax/text.py @@ -0,0 +1,116 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains XML text classes. +""" + +from suds import * +from suds.sax import * + + +class Text(unicode): + """ + An XML text object used to represent text content. + @ivar lang: The (optional) language flag. + @type lang: bool + @ivar escaped: The (optional) XML special character escaped flag. + @type escaped: bool + """ + __slots__ = ('lang', 'escaped',) + + @classmethod + def __valid(cls, *args): + return ( len(args) and args[0] is not None ) + + def __new__(cls, *args, **kwargs): + if cls.__valid(*args): + lang = kwargs.pop('lang', None) + escaped = kwargs.pop('escaped', False) + result = super(Text, cls).__new__(cls, *args, **kwargs) + result.lang = lang + result.escaped = escaped + else: + result = None + return result + + def escape(self): + """ + Encode (escape) special XML characters. + @return: The text with XML special characters escaped. + @rtype: L{Text} + """ + if not self.escaped: + post = sax.encoder.encode(self) + escaped = ( post != self ) + return Text(post, lang=self.lang, escaped=escaped) + return self + + def unescape(self): + """ + Decode (unescape) special XML characters. + @return: The text with escaped XML special characters decoded. + @rtype: L{Text} + """ + if self.escaped: + post = sax.encoder.decode(self) + return Text(post, lang=self.lang) + return self + + def trim(self): + post = self.strip() + return Text(post, lang=self.lang, escaped=self.escaped) + + def __add__(self, other): + joined = u''.join((self, other)) + result = Text(joined, lang=self.lang, escaped=self.escaped) + if isinstance(other, Text): + result.escaped = ( self.escaped or other.escaped ) + return result + + def __repr__(self): + s = [self] + if self.lang is not None: + s.append(' [%s]' % self.lang) + if self.escaped: + s.append(' ') + return ''.join(s) + + def __getstate__(self): + state = {} + for k in self.__slots__: + state[k] = getattr(self, k) + return state + + def __setstate__(self, state): + for k in self.__slots__: + setattr(self, k, state[k]) + + +class Raw(Text): + """ + Raw text which is not XML escaped. + This may include I{string} XML. + """ + def escape(self): + return self + + def unescape(self): + return self + + def __add__(self, other): + joined = u''.join((self, other)) + return Raw(joined, lang=self.lang) diff --git a/libs/suds/servicedefinition.py b/libs/suds/servicedefinition.py new file mode 100644 index 00000000..81b5a0d5 --- /dev/null +++ b/libs/suds/servicedefinition.py @@ -0,0 +1,248 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{service definition} provides a textual representation of a service. +""" + +from logging import getLogger +from suds import * +import suds.metrics as metrics +from suds.sax import Namespace + +log = getLogger(__name__) + +class ServiceDefinition: + """ + A service definition provides an object used to generate a textual description + of a service. + @ivar wsdl: A wsdl. + @type wsdl: L{wsdl.Definitions} + @ivar service: The service object. + @type service: L{suds.wsdl.Service} + @ivar ports: A list of port-tuple: (port, [(method-name, pdef)]) + @type ports: [port-tuple,..] + @ivar prefixes: A list of remapped prefixes. + @type prefixes: [(prefix,uri),..] + @ivar types: A list of type definitions + @type types: [I{Type},..] + """ + + def __init__(self, wsdl, service): + """ + @param wsdl: A wsdl object + @type wsdl: L{Definitions} + @param service: A service B{name}. + @type service: str + """ + self.wsdl = wsdl + self.service = service + self.ports = [] + self.params = [] + self.types = [] + self.prefixes = [] + self.addports() + self.paramtypes() + self.publictypes() + self.getprefixes() + self.pushprefixes() + + def pushprefixes(self): + """ + Add our prefixes to the wsdl so that when users invoke methods + and reference the prefixes, the will resolve properly. + """ + for ns in self.prefixes: + self.wsdl.root.addPrefix(ns[0], ns[1]) + + def addports(self): + """ + Look through the list of service ports and construct a list of tuples where + each tuple is used to describe a port and it's list of methods as: + (port, [method]). Each method is tuple: (name, [pdef,..] where each pdef is + a tuple: (param-name, type). + """ + timer = metrics.Timer() + timer.start() + for port in self.service.ports: + p = self.findport(port) + for op in port.binding.operations.values(): + m = p[0].method(op.name) + binding = m.binding.input + method = (m.name, binding.param_defs(m)) + p[1].append(method) + metrics.log.debug("method '%s' created: %s", m.name, timer) + p[1].sort() + timer.stop() + + def findport(self, port): + """ + Find and return a port tuple for the specified port. + Created and added when not found. + @param port: A port. + @type port: I{service.Port} + @return: A port tuple. + @rtype: (port, [method]) + """ + for p in self.ports: + if p[0] == p: return p + p = (port, []) + self.ports.append(p) + return p + + def getprefixes(self): + """ + Add prefixes foreach namespace referenced by parameter types. + """ + namespaces = [] + for l in (self.params, self.types): + for t,r in l: + ns = r.namespace() + if ns[1] is None: continue + if ns[1] in namespaces: continue + if Namespace.xs(ns) or Namespace.xsd(ns): + continue + namespaces.append(ns[1]) + if t == r: continue + ns = t.namespace() + if ns[1] is None: continue + if ns[1] in namespaces: continue + namespaces.append(ns[1]) + i = 0 + namespaces.sort() + for u in namespaces: + p = self.nextprefix() + ns = (p, u) + self.prefixes.append(ns) + + def paramtypes(self): + """ get all parameter types """ + for m in [p[1] for p in self.ports]: + for p in [p[1] for p in m]: + for pd in p: + if pd[1] in self.params: continue + item = (pd[1], pd[1].resolve()) + self.params.append(item) + + def publictypes(self): + """ get all public types """ + for t in self.wsdl.schema.types.values(): + if t in self.params: continue + if t in self.types: continue + item = (t, t) + self.types.append(item) + tc = lambda x,y: cmp(x[0].name, y[0].name) + self.types.sort(cmp=tc) + + def nextprefix(self): + """ + Get the next available prefix. This means a prefix starting with 'ns' with + a number appended as (ns0, ns1, ..) that is not already defined on the + wsdl document. + """ + used = [ns[0] for ns in self.prefixes] + used += [ns[0] for ns in self.wsdl.root.nsprefixes.items()] + for n in range(0,1024): + p = 'ns%d'%n + if p not in used: + return p + raise Exception('prefixes exhausted') + + def getprefix(self, u): + """ + Get the prefix for the specified namespace (uri) + @param u: A namespace uri. + @type u: str + @return: The namspace. + @rtype: (prefix, uri). + """ + for ns in Namespace.all: + if u == ns[1]: return ns[0] + for ns in self.prefixes: + if u == ns[1]: return ns[0] + raise Exception('ns (%s) not mapped' % u) + + def xlate(self, type): + """ + Get a (namespace) translated I{qualified} name for specified type. + @param type: A schema type. + @type type: I{suds.xsd.sxbasic.SchemaObject} + @return: A translated I{qualified} name. + @rtype: str + """ + resolved = type.resolve() + name = resolved.name + if type.unbounded(): + name += '[]' + ns = resolved.namespace() + if ns[1] == self.wsdl.tns[1]: + return name + prefix = self.getprefix(ns[1]) + return ':'.join((prefix, name)) + + def description(self): + """ + Get a textual description of the service for which this object represents. + @return: A textual description. + @rtype: str + """ + s = [] + indent = (lambda n : '\n%*s'%(n*3,' ')) + s.append('Service ( %s ) tns="%s"' % (self.service.name, self.wsdl.tns[1])) + s.append(indent(1)) + s.append('Prefixes (%d)' % len(self.prefixes)) + for p in self.prefixes: + s.append(indent(2)) + s.append('%s = "%s"' % p) + s.append(indent(1)) + s.append('Ports (%d):' % len(self.ports)) + for p in self.ports: + s.append(indent(2)) + s.append('(%s)' % p[0].name) + s.append(indent(3)) + s.append('Methods (%d):' % len(p[1])) + for m in p[1]: + sig = [] + s.append(indent(4)) + sig.append(m[0]) + sig.append('(') + for p in m[1]: + sig.append(self.xlate(p[1])) + sig.append(' ') + sig.append(p[0]) + sig.append(', ') + sig.append(')') + try: + s.append(''.join(sig)) + except: + pass + s.append(indent(3)) + s.append('Types (%d):' % len(self.types)) + for t in self.types: + s.append(indent(4)) + s.append(self.xlate(t[0])) + s.append('\n\n') + return ''.join(s) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + try: + return self.description() + except Exception, e: + log.exception(e) + return tostr(e) \ No newline at end of file diff --git a/libs/suds/serviceproxy.py b/libs/suds/serviceproxy.py new file mode 100644 index 00000000..6e710509 --- /dev/null +++ b/libs/suds/serviceproxy.py @@ -0,0 +1,86 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The service proxy provides access to web services. + +Replaced by: L{client.Client} +""" + +from logging import getLogger +from suds import * +from suds.client import Client + +log = getLogger(__name__) + + +class ServiceProxy(object): + + """ + A lightweight soap based web service proxy. + @ivar __client__: A client. + Everything is delegated to the 2nd generation API. + @type __client__: L{Client} + @note: Deprecated, replaced by L{Client}. + """ + + def __init__(self, url, **kwargs): + """ + @param url: The URL for the WSDL. + @type url: str + @param kwargs: keyword arguments. + @keyword faults: Raise faults raised by server (default:True), + else return tuple from service method invocation as (http code, object). + @type faults: boolean + @keyword proxy: An http proxy to be specified on requests (default:{}). + The proxy is defined as {protocol:proxy,} + @type proxy: dict + """ + client = Client(url, **kwargs) + self.__client__ = client + + def get_instance(self, name): + """ + Get an instance of a WSDL type by name + @param name: The name of a type defined in the WSDL. + @type name: str + @return: An instance on success, else None + @rtype: L{sudsobject.Object} + """ + return self.__client__.factory.create(name) + + def get_enum(self, name): + """ + Get an instance of an enumeration defined in the WSDL by name. + @param name: The name of a enumeration defined in the WSDL. + @type name: str + @return: An instance on success, else None + @rtype: L{sudsobject.Object} + """ + return self.__client__.factory.create(name) + + def __str__(self): + return str(self.__client__) + + def __unicode__(self): + return unicode(self.__client__) + + def __getattr__(self, name): + builtin = name.startswith('__') and name.endswith('__') + if builtin: + return self.__dict__[name] + else: + return getattr(self.__client__.service, name) \ No newline at end of file diff --git a/libs/suds/soaparray.py b/libs/suds/soaparray.py new file mode 100644 index 00000000..04847d50 --- /dev/null +++ b/libs/suds/soaparray.py @@ -0,0 +1,72 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{soaparray} module provides XSD extensions for handling +soap (section 5) encoded arrays. +""" + +from suds import * +from logging import getLogger +from suds.xsd.sxbasic import Factory as SXFactory +from suds.xsd.sxbasic import Attribute as SXAttribute + + +class Attribute(SXAttribute): + """ + Represents an XSD that handles special + attributes that are extensions for WSDLs. + @ivar aty: Array type information. + @type aty: The value of wsdl:arrayType. + """ + + def __init__(self, schema, root, aty): + """ + @param aty: Array type information. + @type aty: The value of wsdl:arrayType. + """ + SXAttribute.__init__(self, schema, root) + if aty.endswith('[]'): + self.aty = aty[:-2] + else: + self.aty = aty + + def autoqualified(self): + aqs = SXAttribute.autoqualified(self) + aqs.append('aty') + return aqs + + def description(self): + d = SXAttribute.description(self) + d = d+('aty',) + return d + +# +# Builder function, only builds Attribute when arrayType +# attribute is defined on root. +# +def __fn(x, y): + ns = (None, "http://schemas.xmlsoap.org/wsdl/") + aty = y.get('arrayType', ns=ns) + if aty is None: + return SXAttribute(x, y) + else: + return Attribute(x, y, aty) + +# +# Remap tags to __fn() builder. +# +SXFactory.maptag('attribute', __fn) \ No newline at end of file diff --git a/libs/suds/store.py b/libs/suds/store.py new file mode 100644 index 00000000..85e09439 --- /dev/null +++ b/libs/suds/store.py @@ -0,0 +1,594 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains XML text for documents to be distributed +with the suds lib. Also, contains classes for accessing +these documents. +""" + +from StringIO import StringIO +from logging import getLogger + +log = getLogger(__name__) + + +# +# Soap section 5 encoding schema. +# +encoding = \ +""" + + + + + + 'root' can be used to distinguish serialization roots from other + elements that are present in a serialization but are not roots of + a serialized value graph + + + + + + + + + + + + + Attributes common to all elements that function as accessors or + represent independent (multi-ref) values. The href attribute is + intended to be used in a manner like CONREF. That is, the element + content should be empty iff the href attribute appears + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 'Array' is a complex type for accessors identified by position + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +class DocumentStore: + """ + The I{suds} document store provides a local repository + for xml documnts. + @cvar protocol: The URL protocol for the store. + @type protocol: str + @cvar store: The mapping of URL location to documents. + @type store: dict + """ + + protocol = 'suds' + + store = { + 'schemas.xmlsoap.org/soap/encoding/' : encoding + } + + def open(self, url): + """ + Open a document at the specified url. + @param url: A document URL. + @type url: str + @return: A file pointer to the document. + @rtype: StringIO + """ + protocol, location = self.split(url) + if protocol == self.protocol: + return self.find(location) + else: + return None + + def find(self, location): + """ + Find the specified location in the store. + @param location: The I{location} part of a URL. + @type location: str + @return: An input stream to the document. + @rtype: StringIO + """ + try: + content = self.store[location] + return StringIO(content) + except: + reason = 'location "%s" not in document store' % location + raise Exception, reason + + def split(self, url): + """ + Split the url into I{protocol} and I{location} + @param url: A URL. + @param url: str + @return: (I{url}, I{location}) + @rtype: tuple + """ + parts = url.split('://', 1) + if len(parts) == 2: + return parts + else: + return (None, url) \ No newline at end of file diff --git a/libs/suds/sudsobject.py b/libs/suds/sudsobject.py new file mode 100644 index 00000000..1f6168d1 --- /dev/null +++ b/libs/suds/sudsobject.py @@ -0,0 +1,390 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{sudsobject} module provides a collection of suds objects +that are primarily used for the highly dynamic interactions with +wsdl/xsd defined types. +""" + +from logging import getLogger +from suds import * +from new import classobj + +log = getLogger(__name__) + + +def items(sobject): + """ + Extract the I{items} from a suds object much like the + items() method works on I{dict}. + @param sobject: A suds object + @type sobject: L{Object} + @return: A list of items contained in I{sobject}. + @rtype: [(key, value),...] + """ + for item in sobject: + yield item + + +def asdict(sobject): + """ + Convert a sudsobject into a dictionary. + @param sobject: A suds object + @type sobject: L{Object} + @return: A python dictionary containing the + items contained in I{sobject}. + @rtype: dict + """ + return dict(items(sobject)) + +def merge(a, b): + """ + Merge all attributes and metadata from I{a} to I{b}. + @param a: A I{source} object + @type a: L{Object} + @param b: A I{destination} object + @type b: L{Object} + """ + for item in a: + setattr(b, item[0], item[1]) + b.__metadata__ = b.__metadata__ + return b + +def footprint(sobject): + """ + Get the I{virtual footprint} of the object. + This is really a count of the attributes in the branch with a significant value. + @param sobject: A suds object. + @type sobject: L{Object} + @return: The branch footprint. + @rtype: int + """ + n = 0 + for a in sobject.__keylist__: + v = getattr(sobject, a) + if v is None: continue + if isinstance(v, Object): + n += footprint(v) + continue + if hasattr(v, '__len__'): + if len(v): n += 1 + continue + n +=1 + return n + + +class Factory: + + cache = {} + + @classmethod + def subclass(cls, name, bases, dict={}): + if not isinstance(bases, tuple): + bases = (bases,) + name = name.encode('utf-8') + key = '.'.join((name, str(bases))) + subclass = cls.cache.get(key) + if subclass is None: + subclass = classobj(name, bases, dict) + cls.cache[key] = subclass + return subclass + + @classmethod + def object(cls, classname=None, dict={}): + if classname is not None: + subclass = cls.subclass(classname, Object) + inst = subclass() + else: + inst = Object() + for a in dict.items(): + setattr(inst, a[0], a[1]) + return inst + + @classmethod + def metadata(cls): + return Metadata() + + @classmethod + def property(cls, name, value=None): + subclass = cls.subclass(name, Property) + return subclass(value) + + +class Object: + + def __init__(self): + self.__keylist__ = [] + self.__printer__ = Printer() + self.__metadata__ = Metadata() + + def __setattr__(self, name, value): + builtin = name.startswith('__') and name.endswith('__') + if not builtin and \ + name not in self.__keylist__: + self.__keylist__.append(name) + self.__dict__[name] = value + + def __delattr__(self, name): + try: + del self.__dict__[name] + builtin = name.startswith('__') and name.endswith('__') + if not builtin: + self.__keylist__.remove(name) + except: + cls = self.__class__.__name__ + raise AttributeError, "%s has no attribute '%s'" % (cls, name) + + def __getitem__(self, name): + if isinstance(name, int): + name = self.__keylist__[int(name)] + return getattr(self, name) + + def __setitem__(self, name, value): + setattr(self, name, value) + + def __iter__(self): + return Iter(self) + + def __len__(self): + return len(self.__keylist__) + + def __contains__(self, name): + return name in self.__keylist__ + + def __repr__(self): + return str(self) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + return self.__printer__.tostr(self) + + +class Iter: + + def __init__(self, sobject): + self.sobject = sobject + self.keylist = self.__keylist(sobject) + self.index = 0 + + def next(self): + keylist = self.keylist + nkeys = len(self.keylist) + while self.index < nkeys: + k = keylist[self.index] + self.index += 1 + if hasattr(self.sobject, k): + v = getattr(self.sobject, k) + return (k, v) + raise StopIteration() + + def __keylist(self, sobject): + keylist = sobject.__keylist__ + try: + keyset = set(keylist) + ordering = sobject.__metadata__.ordering + ordered = set(ordering) + if not ordered.issuperset(keyset): + log.debug( + '%s must be superset of %s, ordering ignored', + keylist, + ordering) + raise KeyError() + return ordering + except: + return keylist + + def __iter__(self): + return self + + +class Metadata(Object): + def __init__(self): + self.__keylist__ = [] + self.__printer__ = Printer() + + +class Facade(Object): + def __init__(self, name): + Object.__init__(self) + md = self.__metadata__ + md.facade = name + + +class Property(Object): + + def __init__(self, value): + Object.__init__(self) + self.value = value + + def items(self): + for item in self: + if item[0] != 'value': + yield item + + def get(self): + return self.value + + def set(self, value): + self.value = value + return self + + +class Printer: + """ + Pretty printing of a Object object. + """ + + @classmethod + def indent(cls, n): return '%*s'%(n*3,' ') + + def tostr(self, object, indent=-2): + """ get s string representation of object """ + history = [] + return self.process(object, history, indent) + + def process(self, object, h, n=0, nl=False): + """ print object using the specified indent (n) and newline (nl). """ + if object is None: + return 'None' + if isinstance(object, Object): + if len(object) == 0: + return '' + else: + return self.print_object(object, h, n+2, nl) + if isinstance(object, dict): + if len(object) == 0: + return '' + else: + return self.print_dictionary(object, h, n+2, nl) + if isinstance(object, (list,tuple)): + if len(object) == 0: + return '' + else: + return self.print_collection(object, h, n+2) + if isinstance(object, basestring): + return '"%s"' % tostr(object) + return '%s' % tostr(object) + + def print_object(self, d, h, n, nl=False): + """ print complex using the specified indent (n) and newline (nl). """ + s = [] + cls = d.__class__ + md = d.__metadata__ + if d in h: + s.append('(') + s.append(cls.__name__) + s.append(')') + s.append('...') + return ''.join(s) + h.append(d) + if nl: + s.append('\n') + s.append(self.indent(n)) + if cls != Object: + s.append('(') + if isinstance(d, Facade): + s.append(md.facade) + else: + s.append(cls.__name__) + s.append(')') + s.append('{') + for item in d: + if self.exclude(d, item): + continue + item = self.unwrap(d, item) + s.append('\n') + s.append(self.indent(n+1)) + if isinstance(item[1], (list,tuple)): + s.append(item[0]) + s.append('[]') + else: + s.append(item[0]) + s.append(' = ') + s.append(self.process(item[1], h, n, True)) + s.append('\n') + s.append(self.indent(n)) + s.append('}') + h.pop() + return ''.join(s) + + def print_dictionary(self, d, h, n, nl=False): + """ print complex using the specified indent (n) and newline (nl). """ + if d in h: return '{}...' + h.append(d) + s = [] + if nl: + s.append('\n') + s.append(self.indent(n)) + s.append('{') + for item in d.items(): + s.append('\n') + s.append(self.indent(n+1)) + if isinstance(item[1], (list,tuple)): + s.append(tostr(item[0])) + s.append('[]') + else: + s.append(tostr(item[0])) + s.append(' = ') + s.append(self.process(item[1], h, n, True)) + s.append('\n') + s.append(self.indent(n)) + s.append('}') + h.pop() + return ''.join(s) + + def print_collection(self, c, h, n): + """ print collection using the specified indent (n) and newline (nl). """ + if c in h: return '[]...' + h.append(c) + s = [] + for item in c: + s.append('\n') + s.append(self.indent(n)) + s.append(self.process(item, h, n-2)) + s.append(',') + h.pop() + return ''.join(s) + + def unwrap(self, d, item): + """ translate (unwrap) using an optional wrapper function """ + nopt = ( lambda x: x ) + try: + md = d.__metadata__ + pmd = getattr(md, '__print__', None) + if pmd is None: + return item + wrappers = getattr(pmd, 'wrappers', {}) + fn = wrappers.get(item[0], nopt) + return (item[0], fn(item[1])) + except: + pass + return item + + def exclude(self, d, item): + """ check metadata for excluded items """ + try: + md = d.__metadata__ + pmd = getattr(md, '__print__', None) + if pmd is None: + return False + excludes = getattr(pmd, 'excludes', []) + return ( item[0] in excludes ) + except: + pass + return False \ No newline at end of file diff --git a/libs/suds/transport/__init__.py b/libs/suds/transport/__init__.py new file mode 100644 index 00000000..e1e00d73 --- /dev/null +++ b/libs/suds/transport/__init__.py @@ -0,0 +1,130 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains transport interface (classes). +""" + + +class TransportError(Exception): + def __init__(self, reason, httpcode, fp=None): + Exception.__init__(self, reason) + self.httpcode = httpcode + self.fp = fp + +class Request: + """ + A transport request + @ivar url: The url for the request. + @type url: str + @ivar message: The message to be sent in a POST request. + @type message: str + @ivar headers: The http headers to be used for the request. + @type headers: dict + """ + + def __init__(self, url, message=None): + """ + @param url: The url for the request. + @type url: str + @param message: The (optional) message to be send in the request. + @type message: str + """ + self.url = url + self.headers = {} + self.message = message + + def __str__(self): + s = [] + s.append('URL:%s' % self.url) + s.append('HEADERS: %s' % self.headers) + s.append('MESSAGE:') + s.append(self.message) + return '\n'.join(s) + + +class Reply: + """ + A transport reply + @ivar code: The http code returned. + @type code: int + @ivar message: The message to be sent in a POST request. + @type message: str + @ivar headers: The http headers to be used for the request. + @type headers: dict + """ + + def __init__(self, code, headers, message): + """ + @param code: The http code returned. + @type code: int + @param headers: The http returned headers. + @type headers: dict + @param message: The (optional) reply message received. + @type message: str + """ + self.code = code + self.headers = headers + self.message = message + + def __str__(self): + s = [] + s.append('CODE: %s' % self.code) + s.append('HEADERS: %s' % self.headers) + s.append('MESSAGE:') + s.append(self.message) + return '\n'.join(s) + + +class Transport: + """ + The transport I{interface}. + """ + + def __init__(self): + """ + Constructor. + """ + from suds.transport.options import Options + self.options = Options() + del Options + + def open(self, request): + """ + Open the url in the specified request. + @param request: A transport request. + @type request: L{Request} + @return: An input stream. + @rtype: stream + @raise TransportError: On all transport errors. + """ + raise Exception('not-implemented') + + def send(self, request): + """ + Send soap message. Implementations are expected to handle: + - proxies + - I{http} headers + - cookies + - sending message + - brokering exceptions into L{TransportError} + @param request: A transport request. + @type request: L{Request} + @return: The reply + @rtype: L{Reply} + @raise TransportError: On all transport errors. + """ + raise Exception('not-implemented') diff --git a/libs/suds/transport/http.py b/libs/suds/transport/http.py new file mode 100644 index 00000000..6d85b09c --- /dev/null +++ b/libs/suds/transport/http.py @@ -0,0 +1,187 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains classes for basic HTTP transport implementations. +""" + +import urllib2 as u2 +import base64 +import socket +from suds.transport import * +from suds.properties import Unskin +from urlparse import urlparse +from cookielib import CookieJar +from logging import getLogger + +log = getLogger(__name__) + + +class HttpTransport(Transport): + """ + HTTP transport using urllib2. Provided basic http transport + that provides for cookies, proxies but no authentication. + """ + + def __init__(self, **kwargs): + """ + @param kwargs: Keyword arguments. + - B{proxy} - An http proxy to be specified on requests. + The proxy is defined as {protocol:proxy,} + - type: I{dict} + - default: {} + - B{timeout} - Set the url open timeout (seconds). + - type: I{float} + - default: 90 + """ + Transport.__init__(self) + Unskin(self.options).update(kwargs) + self.cookiejar = CookieJar() + self.proxy = {} + self.urlopener = None + + def open(self, request): + try: + url = request.url + log.debug('opening (%s)', url) + u2request = u2.Request(url) + self.proxy = self.options.proxy + return self.u2open(u2request) + except u2.HTTPError, e: + raise TransportError(str(e), e.code, e.fp) + + def send(self, request): + result = None + url = request.url + msg = request.message + headers = request.headers + try: + u2request = u2.Request(url, msg, headers) + self.addcookies(u2request) + self.proxy = self.options.proxy + request.headers.update(u2request.headers) + log.debug('sending:\n%s', request) + fp = self.u2open(u2request) + self.getcookies(fp, u2request) + result = Reply(200, fp.headers.dict, fp.read()) + log.debug('received:\n%s', result) + except u2.HTTPError, e: + if e.code in (202,204): + result = None + else: + raise TransportError(e.msg, e.code, e.fp) + return result + + def addcookies(self, u2request): + """ + Add cookies in the cookiejar to the request. + @param u2request: A urllib2 request. + @rtype: u2request: urllib2.Requet. + """ + self.cookiejar.add_cookie_header(u2request) + + def getcookies(self, fp, u2request): + """ + Add cookies in the request to the cookiejar. + @param u2request: A urllib2 request. + @rtype: u2request: urllib2.Requet. + """ + self.cookiejar.extract_cookies(fp, u2request) + + def u2open(self, u2request): + """ + Open a connection. + @param u2request: A urllib2 request. + @type u2request: urllib2.Requet. + @return: The opened file-like urllib2 object. + @rtype: fp + """ + tm = self.options.timeout + url = self.u2opener() + if self.u2ver() < 2.6: + socket.setdefaulttimeout(tm) + return url.open(u2request) + else: + return url.open(u2request, timeout=tm) + + def u2opener(self): + """ + Create a urllib opener. + @return: An opener. + @rtype: I{OpenerDirector} + """ + if self.urlopener is None: + return u2.build_opener(*self.u2handlers()) + else: + return self.urlopener + + def u2handlers(self): + """ + Get a collection of urllib handlers. + @return: A list of handlers to be installed in the opener. + @rtype: [Handler,...] + """ + handlers = [] + handlers.append(u2.ProxyHandler(self.proxy)) + return handlers + + def u2ver(self): + """ + Get the major/minor version of the urllib2 lib. + @return: The urllib2 version. + @rtype: float + """ + try: + part = u2.__version__.split('.', 1) + n = float('.'.join(part)) + return n + except Exception, e: + log.exception(e) + return 0 + + def __deepcopy__(self, memo={}): + clone = self.__class__() + p = Unskin(self.options) + cp = Unskin(clone.options) + cp.update(p) + return clone + + +class HttpAuthenticated(HttpTransport): + """ + Provides basic http authentication for servers that don't follow + the specified challenge / response model. This implementation + appends the I{Authorization} http header with base64 encoded + credentials on every http request. + """ + + def open(self, request): + self.addcredentials(request) + return HttpTransport.open(self, request) + + def send(self, request): + self.addcredentials(request) + return HttpTransport.send(self, request) + + def addcredentials(self, request): + credentials = self.credentials() + if not (None in credentials): + encoded = base64.encodestring(':'.join(credentials)) + basic = 'Basic %s' % encoded[:-1] + request.headers['Authorization'] = basic + + def credentials(self): + return (self.options.username, self.options.password) \ No newline at end of file diff --git a/libs/suds/transport/https.py b/libs/suds/transport/https.py new file mode 100644 index 00000000..ed23fd55 --- /dev/null +++ b/libs/suds/transport/https.py @@ -0,0 +1,98 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains classes for basic HTTP (authenticated) transport implementations. +""" + +import urllib2 as u2 +from suds.transport import * +from suds.transport.http import HttpTransport +from logging import getLogger + +log = getLogger(__name__) + + +class HttpAuthenticated(HttpTransport): + """ + Provides basic http authentication that follows the RFC-2617 specification. + As defined by specifications, credentials are provided to the server + upon request (HTTP/1.0 401 Authorization Required) by the server only. + @ivar pm: The password manager. + @ivar handler: The authentication handler. + """ + + def __init__(self, **kwargs): + """ + @param kwargs: Keyword arguments. + - B{proxy} - An http proxy to be specified on requests. + The proxy is defined as {protocol:proxy,} + - type: I{dict} + - default: {} + - B{timeout} - Set the url open timeout (seconds). + - type: I{float} + - default: 90 + - B{username} - The username used for http authentication. + - type: I{str} + - default: None + - B{password} - The password used for http authentication. + - type: I{str} + - default: None + """ + HttpTransport.__init__(self, **kwargs) + self.pm = u2.HTTPPasswordMgrWithDefaultRealm() + + def open(self, request): + self.addcredentials(request) + return HttpTransport.open(self, request) + + def send(self, request): + self.addcredentials(request) + return HttpTransport.send(self, request) + + def addcredentials(self, request): + credentials = self.credentials() + if not (None in credentials): + u = credentials[0] + p = credentials[1] + self.pm.add_password(None, request.url, u, p) + + def credentials(self): + return (self.options.username, self.options.password) + + def u2handlers(self): + handlers = HttpTransport.u2handlers(self) + handlers.append(u2.HTTPBasicAuthHandler(self.pm)) + return handlers + + +class WindowsHttpAuthenticated(HttpAuthenticated): + """ + Provides Windows (NTLM) http authentication. + @ivar pm: The password manager. + @ivar handler: The authentication handler. + @author: Christopher Bess + """ + + def u2handlers(self): + # try to import ntlm support + try: + from ntlm import HTTPNtlmAuthHandler + except ImportError: + raise Exception("Cannot import python-ntlm module") + handlers = HttpTransport.u2handlers(self) + handlers.append(HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(self.pm)) + return handlers diff --git a/libs/suds/transport/options.py b/libs/suds/transport/options.py new file mode 100644 index 00000000..8b0d194a --- /dev/null +++ b/libs/suds/transport/options.py @@ -0,0 +1,57 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Contains classes for transport options. +""" + + +from suds.transport import * +from suds.properties import * + + +class Options(Skin): + """ + Options: + - B{proxy} - An http proxy to be specified on requests. + The proxy is defined as {protocol:proxy,} + - type: I{dict} + - default: {} + - B{timeout} - Set the url open timeout (seconds). + - type: I{float} + - default: 90 + - B{headers} - Extra HTTP headers. + - type: I{dict} + - I{str} B{http} - The I{http} protocol proxy URL. + - I{str} B{https} - The I{https} protocol proxy URL. + - default: {} + - B{username} - The username used for http authentication. + - type: I{str} + - default: None + - B{password} - The password used for http authentication. + - type: I{str} + - default: None + """ + def __init__(self, **kwargs): + domain = __name__ + definitions = [ + Definition('proxy', dict, {}), + Definition('timeout', (int,float), 90), + Definition('headers', dict, {}), + Definition('username', basestring, None), + Definition('password', basestring, None), + ] + Skin.__init__(self, domain, definitions, kwargs) \ No newline at end of file diff --git a/libs/suds/umx/__init__.py b/libs/suds/umx/__init__.py new file mode 100644 index 00000000..9d06b408 --- /dev/null +++ b/libs/suds/umx/__init__.py @@ -0,0 +1,56 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides modules containing classes to support +unmarshalling (XML). +""" + +from suds.sudsobject import Object + + + +class Content(Object): + """ + @ivar node: The content source node. + @type node: L{sax.element.Element} + @ivar data: The (optional) content data. + @type data: L{Object} + @ivar text: The (optional) content (xml) text. + @type text: basestring + """ + + extensions = [] + + def __init__(self, node, **kwargs): + Object.__init__(self) + self.node = node + self.data = None + self.text = None + for k,v in kwargs.items(): + setattr(self, k, v) + + def __getattr__(self, name): + if name not in self.__dict__: + if name in self.extensions: + v = None + setattr(self, name, v) + else: + raise AttributeError, \ + 'Content has no attribute %s' % name + else: + v = self.__dict__[name] + return v \ No newline at end of file diff --git a/libs/suds/umx/attrlist.py b/libs/suds/umx/attrlist.py new file mode 100644 index 00000000..36943274 --- /dev/null +++ b/libs/suds/umx/attrlist.py @@ -0,0 +1,88 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides filtered attribute list classes. +""" + +from suds import * +from suds.umx import * +from suds.sax import Namespace + + +class AttrList: + """ + A filtered attribute list. + Items are included during iteration if they are in either the (xs) or + (xml) namespaces. + @ivar raw: The I{raw} attribute list. + @type raw: list + """ + def __init__(self, attributes): + """ + @param attributes: A list of attributes + @type attributes: list + """ + self.raw = attributes + + def real(self): + """ + Get list of I{real} attributes which exclude xs and xml attributes. + @return: A list of I{real} attributes. + @rtype: I{generator} + """ + for a in self.raw: + if self.skip(a): continue + yield a + + def rlen(self): + """ + Get the number of I{real} attributes which exclude xs and xml attributes. + @return: A count of I{real} attributes. + @rtype: L{int} + """ + n = 0 + for a in self.real(): + n += 1 + return n + + def lang(self): + """ + Get list of I{filtered} attributes which exclude xs. + @return: A list of I{filtered} attributes. + @rtype: I{generator} + """ + for a in self.raw: + if a.qname() == 'xml:lang': + return a.value + return None + + def skip(self, attr): + """ + Get whether to skip (filter-out) the specified attribute. + @param attr: An attribute. + @type attr: I{Attribute} + @return: True if should be skipped. + @rtype: bool + """ + ns = attr.namespace() + skip = ( + Namespace.xmlns[1], + 'http://schemas.xmlsoap.org/soap/encoding/', + 'http://schemas.xmlsoap.org/soap/envelope/', + 'http://www.w3.org/2003/05/soap-envelope', + ) + return ( Namespace.xs(ns) or ns[1] in skip ) diff --git a/libs/suds/umx/basic.py b/libs/suds/umx/basic.py new file mode 100644 index 00000000..cdc1e66e --- /dev/null +++ b/libs/suds/umx/basic.py @@ -0,0 +1,41 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides basic unmarshaller classes. +""" + +from logging import getLogger +from suds import * +from suds.umx import * +from suds.umx.core import Core + + +class Basic(Core): + """ + A object builder (unmarshaller). + """ + + def process(self, node): + """ + Process an object graph representation of the xml I{node}. + @param node: An XML tree. + @type node: L{sax.element.Element} + @return: A suds object. + @rtype: L{Object} + """ + content = Content(node) + return Core.process(self, content) \ No newline at end of file diff --git a/libs/suds/umx/core.py b/libs/suds/umx/core.py new file mode 100644 index 00000000..07d33c48 --- /dev/null +++ b/libs/suds/umx/core.py @@ -0,0 +1,216 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides base classes for XML->object I{unmarshalling}. +""" + +from logging import getLogger +from suds import * +from suds.umx import * +from suds.umx.attrlist import AttrList +from suds.sax.text import Text +from suds.sudsobject import Factory, merge + + +log = getLogger(__name__) + +reserved = { 'class':'cls', 'def':'dfn', } + +class Core: + """ + The abstract XML I{node} unmarshaller. This class provides the + I{core} unmarshalling functionality. + """ + + def process(self, content): + """ + Process an object graph representation of the xml I{node}. + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: A suds object. + @rtype: L{Object} + """ + self.reset() + return self.append(content) + + def append(self, content): + """ + Process the specified node and convert the XML document into + a I{suds} L{object}. + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: A I{append-result} tuple as: (L{Object}, I{value}) + @rtype: I{append-result} + @note: This is not the proper entry point. + @see: L{process()} + """ + self.start(content) + self.append_attributes(content) + self.append_children(content) + self.append_text(content) + self.end(content) + return self.postprocess(content) + + def postprocess(self, content): + """ + Perform final processing of the resulting data structure as follows: + - Mixed values (children and text) will have a result of the I{content.node}. + - Simi-simple values (attributes, no-children and text) will have a result of a + property object. + - Simple values (no-attributes, no-children with text nodes) will have a string + result equal to the value of the content.node.getText(). + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: The post-processed result. + @rtype: I{any} + """ + node = content.node + if len(node.children) and node.hasText(): + return node + attributes = AttrList(node.attributes) + if attributes.rlen() and \ + not len(node.children) and \ + node.hasText(): + p = Factory.property(node.name, node.getText()) + return merge(content.data, p) + if len(content.data): + return content.data + lang = attributes.lang() + if content.node.isnil(): + return None + if not len(node.children) and content.text is None: + if self.nillable(content): + return None + else: + return Text('', lang=lang) + if isinstance(content.text, basestring): + return Text(content.text, lang=lang) + else: + return content.text + + def append_attributes(self, content): + """ + Append attribute nodes into L{Content.data}. + Attributes in the I{schema} or I{xml} namespaces are skipped. + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + attributes = AttrList(content.node.attributes) + for attr in attributes.real(): + name = attr.name + value = attr.value + self.append_attribute(name, value, content) + + def append_attribute(self, name, value, content): + """ + Append an attribute name/value into L{Content.data}. + @param name: The attribute name + @type name: basestring + @param value: The attribute's value + @type value: basestring + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + key = name + key = '_%s' % reserved.get(key, key) + setattr(content.data, key, value) + + def append_children(self, content): + """ + Append child nodes into L{Content.data} + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + for child in content.node: + cont = Content(child) + cval = self.append(cont) + key = reserved.get(child.name, child.name) + if key in content.data: + v = getattr(content.data, key) + if isinstance(v, list): + v.append(cval) + else: + setattr(content.data, key, [v, cval]) + continue + if self.unbounded(cont): + if cval is None: + setattr(content.data, key, []) + else: + setattr(content.data, key, [cval,]) + else: + setattr(content.data, key, cval) + + def append_text(self, content): + """ + Append text nodes into L{Content.data} + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + if content.node.hasText(): + content.text = content.node.getText() + + def reset(self): + pass + + def start(self, content): + """ + Processing on I{node} has started. Build and return + the proper object. + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: A subclass of Object. + @rtype: L{Object} + """ + content.data = Factory.object(content.node.name) + + def end(self, content): + """ + Processing on I{node} has ended. + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + pass + + def bounded(self, content): + """ + Get whether the content is bounded (not a list). + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: True if bounded, else False + @rtype: boolean + '""" + return ( not self.unbounded(content) ) + + def unbounded(self, content): + """ + Get whether the object is unbounded (a list). + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: True if unbounded, else False + @rtype: boolean + '""" + return False + + def nillable(self, content): + """ + Get whether the object is nillable. + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: True if nillable, else False + @rtype: boolean + '""" + return False \ No newline at end of file diff --git a/libs/suds/umx/encoded.py b/libs/suds/umx/encoded.py new file mode 100644 index 00000000..afe7374c --- /dev/null +++ b/libs/suds/umx/encoded.py @@ -0,0 +1,128 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides soap encoded unmarshaller classes. +""" + +from logging import getLogger +from suds import * +from suds.umx import * +from suds.umx.typed import Typed +from suds.sax import splitPrefix, Namespace + +log = getLogger(__name__) + +# +# Add encoded extensions +# aty = The soap (section 5) encoded array type. +# +Content.extensions.append('aty') + + +class Encoded(Typed): + """ + A SOAP section (5) encoding unmarshaller. + This marshaller supports rpc/encoded soap styles. + """ + + def start(self, content): + # + # Grab the array type and continue + # + self.setaty(content) + Typed.start(self, content) + + def end(self, content): + # + # Squash soap encoded arrays into python lists. This is + # also where we insure that empty arrays are represented + # as empty python lists. + # + aty = content.aty + if aty is not None: + self.promote(content) + return Typed.end(self, content) + + def postprocess(self, content): + # + # Ensure proper rendering of empty arrays. + # + if content.aty is None: + return Typed.postprocess(self, content) + else: + return content.data + + def setaty(self, content): + """ + Grab the (aty) soap-enc:arrayType and attach it to the + content for proper array processing later in end(). + @param content: The current content being unmarshalled. + @type content: L{Content} + @return: self + @rtype: L{Encoded} + """ + name = 'arrayType' + ns = (None, 'http://schemas.xmlsoap.org/soap/encoding/') + aty = content.node.get(name, ns) + if aty is not None: + content.aty = aty + parts = aty.split('[') + ref = parts[0] + if len(parts) == 2: + self.applyaty(content, ref) + else: + pass # (2) dimensional array + return self + + def applyaty(self, content, xty): + """ + Apply the type referenced in the I{arrayType} to the content + (child nodes) of the array. Each element (node) in the array + that does not have an explicit xsi:type attribute is given one + based on the I{arrayType}. + @param content: An array content. + @type content: L{Content} + @param xty: The XSI type reference. + @type xty: str + @return: self + @rtype: L{Encoded} + """ + name = 'type' + ns = Namespace.xsins + parent = content.node + for child in parent.getChildren(): + ref = child.get(name, ns) + if ref is None: + parent.addPrefix(ns[0], ns[1]) + attr = ':'.join((ns[0], name)) + child.set(attr, xty) + return self + + def promote(self, content): + """ + Promote (replace) the content.data with the first attribute + of the current content.data that is a I{list}. Note: the + content.data may be empty or contain only _x attributes. + In either case, the content.data is assigned an empty list. + @param content: An array content. + @type content: L{Content} + """ + for n,v in content.data: + if isinstance(v, list): + content.data = v + return + content.data = [] \ No newline at end of file diff --git a/libs/suds/umx/typed.py b/libs/suds/umx/typed.py new file mode 100644 index 00000000..f272a259 --- /dev/null +++ b/libs/suds/umx/typed.py @@ -0,0 +1,141 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +Provides typed unmarshaller classes. +""" + +from logging import getLogger +from suds import * +from suds.umx import * +from suds.umx.core import Core +from suds.resolver import NodeResolver, Frame +from suds.sudsobject import Factory + +log = getLogger(__name__) + + +# +# Add typed extensions +# type = The expected xsd type +# real = The 'true' XSD type +# +Content.extensions.append('type') +Content.extensions.append('real') + + +class Typed(Core): + """ + A I{typed} XML unmarshaller + @ivar resolver: A schema type resolver. + @type resolver: L{NodeResolver} + """ + + def __init__(self, schema): + """ + @param schema: A schema object. + @type schema: L{xsd.schema.Schema} + """ + self.resolver = NodeResolver(schema) + + def process(self, node, type): + """ + Process an object graph representation of the xml L{node}. + @param node: An XML tree. + @type node: L{sax.element.Element} + @param type: The I{optional} schema type. + @type type: L{xsd.sxbase.SchemaObject} + @return: A suds object. + @rtype: L{Object} + """ + content = Content(node) + content.type = type + return Core.process(self, content) + + def reset(self): + log.debug('reset') + self.resolver.reset() + + def start(self, content): + # + # Resolve to the schema type; build an object and setup metadata. + # + if content.type is None: + found = self.resolver.find(content.node) + if found is None: + log.error(self.resolver.schema) + raise TypeNotFound(content.node.qname()) + content.type = found + else: + known = self.resolver.known(content.node) + frame = Frame(content.type, resolved=known) + self.resolver.push(frame) + real = self.resolver.top().resolved + content.real = real + cls_name = real.name + if cls_name is None: + cls_name = content.node.name + content.data = Factory.object(cls_name) + md = content.data.__metadata__ + md.sxtype = real + + def end(self, content): + self.resolver.pop() + + def unbounded(self, content): + return content.type.unbounded() + + def nillable(self, content): + resolved = content.type.resolve() + return ( content.type.nillable or \ + (resolved.builtin() and resolved.nillable ) ) + + def append_attribute(self, name, value, content): + """ + Append an attribute name/value into L{Content.data}. + @param name: The attribute name + @type name: basestring + @param value: The attribute's value + @type value: basestring + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + type = self.resolver.findattr(name) + if type is None: + log.warn('attribute (%s) type, not-found', name) + else: + value = self.translated(value, type) + Core.append_attribute(self, name, value, content) + + def append_text(self, content): + """ + Append text nodes into L{Content.data} + Here is where the I{true} type is used to translate the value + into the proper python type. + @param content: The current content being unmarshalled. + @type content: L{Content} + """ + Core.append_text(self, content) + known = self.resolver.top().resolved + content.text = self.translated(content.text, known) + + def translated(self, value, type): + """ translate using the schema type """ + if value is not None: + resolved = type.resolve() + return resolved.translate(value) + else: + return value \ No newline at end of file diff --git a/libs/suds/wsdl.py b/libs/suds/wsdl.py new file mode 100644 index 00000000..8bba88f9 --- /dev/null +++ b/libs/suds/wsdl.py @@ -0,0 +1,922 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{wsdl} module provides an objectification of the WSDL. +The primary class is I{Definitions} as it represends the root element +found in the document. +""" + +from logging import getLogger +from suds import * +from suds.sax import splitPrefix +from suds.sax.element import Element +from suds.bindings.document import Document +from suds.bindings.rpc import RPC, Encoded +from suds.xsd import qualify, Namespace +from suds.xsd.schema import Schema, SchemaCollection +from suds.xsd.query import ElementQuery +from suds.sudsobject import Object, Facade, Metadata +from suds.reader import DocumentReader, DefinitionsReader +from urlparse import urljoin +import re, soaparray + +log = getLogger(__name__) + +wsdlns = (None, "http://schemas.xmlsoap.org/wsdl/") +soapns = (None, 'http://schemas.xmlsoap.org/wsdl/soap/') +soap12ns = (None, 'http://schemas.xmlsoap.org/wsdl/soap12/') + + +class WObject(Object): + """ + Base object for wsdl types. + @ivar root: The XML I{root} element. + @type root: L{Element} + """ + + def __init__(self, root, definitions=None): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + Object.__init__(self) + self.root = root + pmd = Metadata() + pmd.excludes = ['root'] + pmd.wrappers = dict(qname=repr) + self.__metadata__.__print__ = pmd + + def resolve(self, definitions): + """ + Resolve named references to other WSDL objects. + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + pass + + +class NamedObject(WObject): + """ + A B{named} WSDL object. + @ivar name: The name of the object. + @type name: str + @ivar qname: The I{qualified} name of the object. + @type qname: (name, I{namespace-uri}). + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + WObject.__init__(self, root, definitions) + self.name = root.get('name') + self.qname = (self.name, definitions.tns[1]) + pmd = self.__metadata__.__print__ + pmd.wrappers['qname'] = repr + + +class Definitions(WObject): + """ + Represents the I{root} container of the WSDL objects as defined + by + @ivar id: The object id. + @type id: str + @ivar options: An options dictionary. + @type options: L{options.Options} + @ivar url: The URL used to load the object. + @type url: str + @ivar tns: The target namespace for the WSDL. + @type tns: str + @ivar schema: The collective WSDL schema object. + @type schema: L{SchemaCollection} + @ivar children: The raw list of child objects. + @type children: [L{WObject},...] + @ivar imports: The list of L{Import} children. + @type imports: [L{Import},...] + @ivar messages: The dictionary of L{Message} children key'd by I{qname} + @type messages: [L{Message},...] + @ivar port_types: The dictionary of L{PortType} children key'd by I{qname} + @type port_types: [L{PortType},...] + @ivar bindings: The dictionary of L{Binding} children key'd by I{qname} + @type bindings: [L{Binding},...] + @ivar service: The service object. + @type service: L{Service} + """ + + Tag = 'definitions' + + def __init__(self, url, options): + """ + @param url: A URL to the WSDL. + @type url: str + @param options: An options dictionary. + @type options: L{options.Options} + """ + log.debug('reading wsdl at: %s ...', url) + reader = DocumentReader(options) + d = reader.open(url) + root = d.root() + WObject.__init__(self, root) + self.id = objid(self) + self.options = options + self.url = url + self.tns = self.mktns(root) + self.types = [] + self.schema = None + self.children = [] + self.imports = [] + self.messages = {} + self.port_types = {} + self.bindings = {} + self.services = [] + self.add_children(self.root) + self.children.sort() + pmd = self.__metadata__.__print__ + pmd.excludes.append('children') + pmd.excludes.append('wsdl') + pmd.wrappers['schema'] = repr + self.open_imports() + self.resolve() + self.build_schema() + self.set_wrapped() + for s in self.services: + self.add_methods(s) + log.debug("wsdl at '%s' loaded:\n%s", url, self) + + def mktns(self, root): + """ Get/create the target namespace """ + tns = root.get('targetNamespace') + prefix = root.findPrefix(tns) + if prefix is None: + log.debug('warning: tns (%s), not mapped to prefix', tns) + prefix = 'tns' + return (prefix, tns) + + def add_children(self, root): + """ Add child objects using the factory """ + for c in root.getChildren(ns=wsdlns): + child = Factory.create(c, self) + if child is None: continue + self.children.append(child) + if isinstance(child, Import): + self.imports.append(child) + continue + if isinstance(child, Types): + self.types.append(child) + continue + if isinstance(child, Message): + self.messages[child.qname] = child + continue + if isinstance(child, PortType): + self.port_types[child.qname] = child + continue + if isinstance(child, Binding): + self.bindings[child.qname] = child + continue + if isinstance(child, Service): + self.services.append(child) + continue + + def open_imports(self): + """ Import the I{imported} WSDLs. """ + for imp in self.imports: + imp.load(self) + + def resolve(self): + """ Tell all children to resolve themselves """ + for c in self.children: + c.resolve(self) + + def build_schema(self): + """ Process L{Types} objects and create the schema collection """ + container = SchemaCollection(self) + for t in [t for t in self.types if t.local()]: + for root in t.contents(): + schema = Schema(root, self.url, self.options, container) + container.add(schema) + if not len(container): # empty + root = Element.buildPath(self.root, 'types/schema') + schema = Schema(root, self.url, self.options, container) + container.add(schema) + self.schema = container.load(self.options) + for s in [t.schema() for t in self.types if t.imported()]: + self.schema.merge(s) + return self.schema + + def add_methods(self, service): + """ Build method view for service """ + bindings = { + 'document/literal' : Document(self), + 'rpc/literal' : RPC(self), + 'rpc/encoded' : Encoded(self) + } + for p in service.ports: + binding = p.binding + ptype = p.binding.type + operations = p.binding.type.operations.values() + for name in [op.name for op in operations]: + m = Facade('Method') + m.name = name + m.location = p.location + m.binding = Facade('binding') + op = binding.operation(name) + m.soap = op.soap + key = '/'.join((op.soap.style, op.soap.input.body.use)) + m.binding.input = bindings.get(key) + key = '/'.join((op.soap.style, op.soap.output.body.use)) + m.binding.output = bindings.get(key) + op = ptype.operation(name) + p.methods[name] = m + + def set_wrapped(self): + """ set (wrapped|bare) flag on messages """ + for b in self.bindings.values(): + for op in b.operations.values(): + for body in (op.soap.input.body, op.soap.output.body): + body.wrapped = False + if len(body.parts) != 1: + continue + for p in body.parts: + if p.element is None: + continue + query = ElementQuery(p.element) + pt = query.execute(self.schema) + if pt is None: + raise TypeNotFound(query.ref) + resolved = pt.resolve() + if resolved.builtin(): + continue + body.wrapped = True + + def __getstate__(self): + nopickle = ('options',) + state = self.__dict__.copy() + for k in nopickle: + if k in state: + del state[k] + return state + + def __repr__(self): + return 'Definitions (id=%s)' % self.id + + +class Import(WObject): + """ + Represents the . + @ivar location: The value of the I{location} attribute. + @type location: str + @ivar ns: The value of the I{namespace} attribute. + @type ns: str + @ivar imported: The imported object. + @type imported: L{Definitions} + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + WObject.__init__(self, root, definitions) + self.location = root.get('location') + self.ns = root.get('namespace') + self.imported = None + pmd = self.__metadata__.__print__ + pmd.wrappers['imported'] = repr + + def load(self, definitions): + """ Load the object by opening the URL """ + url = self.location + log.debug('importing (%s)', url) + if '://' not in url: + url = urljoin(definitions.url, url) + options = definitions.options + d = Definitions(url, options) + if d.root.match(Definitions.Tag, wsdlns): + self.import_definitions(definitions, d) + return + if d.root.match(Schema.Tag, Namespace.xsdns): + self.import_schema(definitions, d) + return + raise Exception('document at "%s" is unknown' % url) + + def import_definitions(self, definitions, d): + """ import/merge wsdl definitions """ + definitions.types += d.types + definitions.messages.update(d.messages) + definitions.port_types.update(d.port_types) + definitions.bindings.update(d.bindings) + self.imported = d + log.debug('imported (WSDL):\n%s', d) + + def import_schema(self, definitions, d): + """ import schema as content """ + if not len(definitions.types): + types = Types.create(definitions) + definitions.types.append(types) + else: + types = definitions.types[-1] + types.root.append(d.root) + log.debug('imported (XSD):\n%s', d.root) + + def __gt__(self, other): + return False + + +class Types(WObject): + """ + Represents . + """ + + @classmethod + def create(cls, definitions): + root = Element('types', ns=wsdlns) + definitions.root.insert(root) + return Types(root, definitions) + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + WObject.__init__(self, root, definitions) + self.definitions = definitions + + def contents(self): + return self.root.getChildren('schema', Namespace.xsdns) + + def schema(self): + return self.definitions.schema + + def local(self): + return ( self.definitions.schema is None ) + + def imported(self): + return ( not self.local() ) + + def __gt__(self, other): + return isinstance(other, Import) + + +class Part(NamedObject): + """ + Represents . + @ivar element: The value of the {element} attribute. + Stored as a I{qref} as converted by L{suds.xsd.qualify}. + @type element: str + @ivar type: The value of the {type} attribute. + Stored as a I{qref} as converted by L{suds.xsd.qualify}. + @type type: str + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + NamedObject.__init__(self, root, definitions) + pmd = Metadata() + pmd.wrappers = dict(element=repr, type=repr) + self.__metadata__.__print__ = pmd + tns = definitions.tns + self.element = self.__getref('element', tns) + self.type = self.__getref('type', tns) + + def __getref(self, a, tns): + """ Get the qualified value of attribute named 'a'.""" + s = self.root.get(a) + if s is None: + return s + else: + return qualify(s, self.root, tns) + + +class Message(NamedObject): + """ + Represents . + @ivar parts: A list of message parts. + @type parts: [I{Part},...] + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + NamedObject.__init__(self, root, definitions) + self.parts = [] + for p in root.getChildren('part'): + part = Part(p, definitions) + self.parts.append(part) + + def __gt__(self, other): + return isinstance(other, (Import, Types)) + + +class PortType(NamedObject): + """ + Represents . + @ivar operations: A list of contained operations. + @type operations: list + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + NamedObject.__init__(self, root, definitions) + self.operations = {} + for c in root.getChildren('operation'): + op = Facade('Operation') + op.name = c.get('name') + op.tns = definitions.tns + input = c.getChild('input') + if input is None: + op.input = None + else: + op.input = input.get('message') + output = c.getChild('output') + if output is None: + op.output = None + else: + op.output = output.get('message') + faults = [] + for fault in c.getChildren('fault'): + f = Facade('Fault') + f.name = fault.get('name') + f.message = fault.get('message') + faults.append(f) + op.faults = faults + self.operations[op.name] = op + + def resolve(self, definitions): + """ + Resolve named references to other WSDL objects. + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + for op in self.operations.values(): + if op.input is None: + op.input = Message(Element('no-input'), definitions) + else: + qref = qualify(op.input, self.root, definitions.tns) + msg = definitions.messages.get(qref) + if msg is None: + raise Exception("msg '%s', not-found" % op.input) + else: + op.input = msg + if op.output is None: + op.output = Message(Element('no-output'), definitions) + else: + qref = qualify(op.output, self.root, definitions.tns) + msg = definitions.messages.get(qref) + if msg is None: + raise Exception("msg '%s', not-found" % op.output) + else: + op.output = msg + for f in op.faults: + qref = qualify(f.message, self.root, definitions.tns) + msg = definitions.messages.get(qref) + if msg is None: + raise Exception, "msg '%s', not-found" % f.message + f.message = msg + + def operation(self, name): + """ + Shortcut used to get a contained operation by name. + @param name: An operation name. + @type name: str + @return: The named operation. + @rtype: Operation + @raise L{MethodNotFound}: When not found. + """ + try: + return self.operations[name] + except Exception, e: + raise MethodNotFound(name) + + def __gt__(self, other): + return isinstance(other, (Import, Types, Message)) + + +class Binding(NamedObject): + """ + Represents + @ivar operations: A list of contained operations. + @type operations: list + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + NamedObject.__init__(self, root, definitions) + self.operations = {} + self.type = root.get('type') + sr = self.soaproot() + if sr is None: + self.soap = None + log.debug('binding: "%s" not a soap binding', self.name) + return + soap = Facade('soap') + self.soap = soap + self.soap.style = sr.get('style', default='document') + self.add_operations(self.root, definitions) + + def soaproot(self): + """ get the soap:binding """ + for ns in (soapns, soap12ns): + sr = self.root.getChild('binding', ns=ns) + if sr is not None: + return sr + return None + + def add_operations(self, root, definitions): + """ Add children """ + dsop = Element('operation', ns=soapns) + for c in root.getChildren('operation'): + op = Facade('Operation') + op.name = c.get('name') + sop = c.getChild('operation', default=dsop) + soap = Facade('soap') + soap.action = '"%s"' % sop.get('soapAction', default='') + soap.style = sop.get('style', default=self.soap.style) + soap.input = Facade('Input') + soap.input.body = Facade('Body') + soap.input.headers = [] + soap.output = Facade('Output') + soap.output.body = Facade('Body') + soap.output.headers = [] + op.soap = soap + input = c.getChild('input') + if input is None: + input = Element('input', ns=wsdlns) + body = input.getChild('body') + self.body(definitions, soap.input.body, body) + for header in input.getChildren('header'): + self.header(definitions, soap.input, header) + output = c.getChild('output') + if output is None: + output = Element('output', ns=wsdlns) + body = output.getChild('body') + self.body(definitions, soap.output.body, body) + for header in output.getChildren('header'): + self.header(definitions, soap.output, header) + faults = [] + for fault in c.getChildren('fault'): + sf = fault.getChild('fault') + if sf is None: + continue + fn = fault.get('name') + f = Facade('Fault') + f.name = sf.get('name', default=fn) + f.use = sf.get('use', default='literal') + faults.append(f) + soap.faults = faults + self.operations[op.name] = op + + def body(self, definitions, body, root): + """ add the input/output body properties """ + if root is None: + body.use = 'literal' + body.namespace = definitions.tns + body.parts = () + return + parts = root.get('parts') + if parts is None: + body.parts = () + else: + body.parts = re.split('[\s,]', parts) + body.use = root.get('use', default='literal') + ns = root.get('namespace') + if ns is None: + body.namespace = definitions.tns + else: + prefix = root.findPrefix(ns, 'b0') + body.namespace = (prefix, ns) + + def header(self, definitions, parent, root): + """ add the input/output header properties """ + if root is None: + return + header = Facade('Header') + parent.headers.append(header) + header.use = root.get('use', default='literal') + ns = root.get('namespace') + if ns is None: + header.namespace = definitions.tns + else: + prefix = root.findPrefix(ns, 'h0') + header.namespace = (prefix, ns) + msg = root.get('message') + if msg is not None: + header.message = msg + part = root.get('part') + if part is not None: + header.part = part + + def resolve(self, definitions): + """ + Resolve named references to other WSDL objects. This includes + cross-linking information (from) the portType (to) the I{soap} + protocol information on the binding for each operation. + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + self.resolveport(definitions) + for op in self.operations.values(): + self.resolvesoapbody(definitions, op) + self.resolveheaders(definitions, op) + self.resolvefaults(definitions, op) + + def resolveport(self, definitions): + """ + Resolve port_type reference. + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + ref = qualify(self.type, self.root, definitions.tns) + port_type = definitions.port_types.get(ref) + if port_type is None: + raise Exception("portType '%s', not-found" % self.type) + else: + self.type = port_type + + def resolvesoapbody(self, definitions, op): + """ + Resolve soap body I{message} parts by + cross-referencing with operation defined in port type. + @param definitions: A definitions object. + @type definitions: L{Definitions} + @param op: An I{operation} object. + @type op: I{operation} + """ + ptop = self.type.operation(op.name) + if ptop is None: + raise Exception, \ + "operation '%s' not defined in portType" % op.name + soap = op.soap + parts = soap.input.body.parts + if len(parts): + pts = [] + for p in ptop.input.parts: + if p.name in parts: + pts.append(p) + soap.input.body.parts = pts + else: + soap.input.body.parts = ptop.input.parts + parts = soap.output.body.parts + if len(parts): + pts = [] + for p in ptop.output.parts: + if p.name in parts: + pts.append(p) + soap.output.body.parts = pts + else: + soap.output.body.parts = ptop.output.parts + + def resolveheaders(self, definitions, op): + """ + Resolve soap header I{message} references. + @param definitions: A definitions object. + @type definitions: L{Definitions} + @param op: An I{operation} object. + @type op: I{operation} + """ + soap = op.soap + headers = soap.input.headers + soap.output.headers + for header in headers: + mn = header.message + ref = qualify(mn, self.root, definitions.tns) + message = definitions.messages.get(ref) + if message is None: + raise Exception, "message'%s', not-found" % mn + pn = header.part + for p in message.parts: + if p.name == pn: + header.part = p + break + if pn == header.part: + raise Exception, \ + "message '%s' has not part named '%s'" % (ref, pn) + + def resolvefaults(self, definitions, op): + """ + Resolve soap fault I{message} references by + cross-referencing with operation defined in port type. + @param definitions: A definitions object. + @type definitions: L{Definitions} + @param op: An I{operation} object. + @type op: I{operation} + """ + ptop = self.type.operation(op.name) + if ptop is None: + raise Exception, \ + "operation '%s' not defined in portType" % op.name + soap = op.soap + for fault in soap.faults: + for f in ptop.faults: + if f.name == fault.name: + fault.parts = f.message.parts + continue + if hasattr(fault, 'parts'): + continue + raise Exception, \ + "fault '%s' not defined in portType '%s'" % (fault.name, self.type.name) + + def operation(self, name): + """ + Shortcut used to get a contained operation by name. + @param name: An operation name. + @type name: str + @return: The named operation. + @rtype: Operation + @raise L{MethodNotFound}: When not found. + """ + try: + return self.operations[name] + except: + raise MethodNotFound(name) + + def __gt__(self, other): + return ( not isinstance(other, Service) ) + + +class Port(NamedObject): + """ + Represents a service port. + @ivar service: A service. + @type service: L{Service} + @ivar binding: A binding name. + @type binding: str + @ivar location: The service location (url). + @type location: str + """ + + def __init__(self, root, definitions, service): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + @param service: A service object. + @type service: L{Service} + """ + NamedObject.__init__(self, root, definitions) + self.__service = service + self.binding = root.get('binding') + address = root.getChild('address') + if address is None: + self.location = None + else: + self.location = address.get('location').encode('utf-8') + self.methods = {} + + def method(self, name): + """ + Get a method defined in this portType by name. + @param name: A method name. + @type name: str + @return: The requested method object. + @rtype: I{Method} + """ + return self.methods.get(name) + + +class Service(NamedObject): + """ + Represents . + @ivar port: The contained ports. + @type port: [Port,..] + @ivar methods: The contained methods for all ports. + @type methods: [Method,..] + """ + + def __init__(self, root, definitions): + """ + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + NamedObject.__init__(self, root, definitions) + self.ports = [] + for p in root.getChildren('port'): + port = Port(p, definitions, self) + self.ports.append(port) + + def port(self, name): + """ + Locate a port by name. + @param name: A port name. + @type name: str + @return: The port object. + @rtype: L{Port} + """ + for p in self.ports: + if p.name == name: + return p + return None + + def setlocation(self, url, names=None): + """ + Override the invocation location (url) for service method. + @param url: A url location. + @type url: A url. + @param names: A list of method names. None=ALL + @type names: [str,..] + """ + for p in self.ports: + for m in p.methods.values(): + if names is None or m.name in names: + m.location = url + + def resolve(self, definitions): + """ + Resolve named references to other WSDL objects. + Ports without soap bindings are discarded. + @param definitions: A definitions object. + @type definitions: L{Definitions} + """ + filtered = [] + for p in self.ports: + ref = qualify(p.binding, self.root, definitions.tns) + binding = definitions.bindings.get(ref) + if binding is None: + raise Exception("binding '%s', not-found" % p.binding) + if binding.soap is None: + log.debug('binding "%s" - not a soap, discarded', binding.name) + continue + p.binding = binding + filtered.append(p) + self.ports = filtered + + def __gt__(self, other): + return True + + +class Factory: + """ + Simple WSDL object factory. + @cvar tags: Dictionary of tag->constructor mappings. + @type tags: dict + """ + + tags =\ + { + 'import' : Import, + 'types' : Types, + 'message' : Message, + 'portType' : PortType, + 'binding' : Binding, + 'service' : Service, + } + + @classmethod + def create(cls, root, definitions): + """ + Create an object based on the root tag name. + @param root: An XML root element. + @type root: L{Element} + @param definitions: A definitions object. + @type definitions: L{Definitions} + @return: The created object. + @rtype: L{WObject} + """ + fn = cls.tags.get(root.name) + if fn is not None: + return fn(root, definitions) + else: + return None diff --git a/libs/suds/wsse.py b/libs/suds/wsse.py new file mode 100644 index 00000000..2a697c1c --- /dev/null +++ b/libs/suds/wsse.py @@ -0,0 +1,212 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{wsse} module provides WS-Security. +""" + +from logging import getLogger +from suds import * +from suds.sudsobject import Object +from suds.sax.element import Element +from suds.sax.date import UTC +from datetime import datetime, timedelta + +try: + from hashlib import md5 +except ImportError: + # Python 2.4 compatibility + from md5 import md5 + + +dsns = \ + ('ds', + 'http://www.w3.org/2000/09/xmldsig#') +wssens = \ + ('wsse', + 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd') +wsuns = \ + ('wsu', + 'http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd') +wsencns = \ + ('wsenc', + 'http://www.w3.org/2001/04/xmlenc#') + + +class Security(Object): + """ + WS-Security object. + @ivar tokens: A list of security tokens + @type tokens: [L{Token},...] + @ivar signatures: A list of signatures. + @type signatures: TBD + @ivar references: A list of references. + @type references: TBD + @ivar keys: A list of encryption keys. + @type keys: TBD + """ + + def __init__(self): + """ """ + Object.__init__(self) + self.mustUnderstand = True + self.tokens = [] + self.signatures = [] + self.references = [] + self.keys = [] + + def xml(self): + """ + Get xml representation of the object. + @return: The root node. + @rtype: L{Element} + """ + root = Element('Security', ns=wssens) + root.set('mustUnderstand', str(self.mustUnderstand).lower()) + for t in self.tokens: + root.append(t.xml()) + return root + + +class Token(Object): + """ I{Abstract} security token. """ + + @classmethod + def now(cls): + return datetime.now() + + @classmethod + def utc(cls): + return datetime.utcnow() + + @classmethod + def sysdate(cls): + utc = UTC() + return str(utc) + + def __init__(self): + Object.__init__(self) + + +class UsernameToken(Token): + """ + Represents a basic I{UsernameToken} WS-Secuirty token. + @ivar username: A username. + @type username: str + @ivar password: A password. + @type password: str + @ivar nonce: A set of bytes to prevent reply attacks. + @type nonce: str + @ivar created: The token created. + @type created: L{datetime} + """ + + def __init__(self, username=None, password=None): + """ + @param username: A username. + @type username: str + @param password: A password. + @type password: str + """ + Token.__init__(self) + self.username = username + self.password = password + self.nonce = None + self.created = None + + def setnonce(self, text=None): + """ + Set I{nonce} which is arbitraty set of bytes to prevent + reply attacks. + @param text: The nonce text value. + Generated when I{None}. + @type text: str + """ + if text is None: + s = [] + s.append(self.username) + s.append(self.password) + s.append(Token.sysdate()) + m = md5() + m.update(':'.join(s)) + self.nonce = m.hexdigest() + else: + self.nonce = text + + def setcreated(self, dt=None): + """ + Set I{created}. + @param dt: The created date & time. + Set as datetime.utc() when I{None}. + @type dt: L{datetime} + """ + if dt is None: + self.created = Token.utc() + else: + self.created = dt + + + def xml(self): + """ + Get xml representation of the object. + @return: The root node. + @rtype: L{Element} + """ + root = Element('UsernameToken', ns=wssens) + u = Element('Username', ns=wssens) + u.setText(self.username) + root.append(u) + p = Element('Password', ns=wssens) + p.setText(self.password) + root.append(p) + if self.nonce is not None: + n = Element('Nonce', ns=wssens) + n.setText(self.nonce) + root.append(n) + if self.created is not None: + n = Element('Created', ns=wsuns) + n.setText(str(UTC(self.created))) + root.append(n) + return root + + +class Timestamp(Token): + """ + Represents the I{Timestamp} WS-Secuirty token. + @ivar created: The token created. + @type created: L{datetime} + @ivar expires: The token expires. + @type expires: L{datetime} + """ + + def __init__(self, validity=90): + """ + @param validity: The time in seconds. + @type validity: int + """ + Token.__init__(self) + self.created = Token.utc() + self.expires = self.created + timedelta(seconds=validity) + + def xml(self): + root = Element("Timestamp", ns=wsuns) + created = Element('Created', ns=wsuns) + created.setText(str(UTC(self.created))) + expires = Element('Expires', ns=wsuns) + expires.setText(str(UTC(self.expires))) + root.append(created) + root.append(expires) + return root \ No newline at end of file diff --git a/libs/suds/xsd/__init__.py b/libs/suds/xsd/__init__.py new file mode 100644 index 00000000..0917f3f3 --- /dev/null +++ b/libs/suds/xsd/__init__.py @@ -0,0 +1,86 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{schema} module provides a intelligent representation of +an XSD schema. The I{raw} model is the XML tree and the I{model} +is the denormalized, objectified and intelligent view of the schema. +Most of the I{value-add} provided by the model is centered around +tranparent referenced type resolution and targeted denormalization. +""" + +from logging import getLogger +from suds import * +from suds.sax import Namespace, splitPrefix + +log = getLogger(__name__) + + +def qualify(ref, resolvers, defns=Namespace.default): + """ + Get a reference that is I{qualified} by namespace. + @param ref: A referenced schema type name. + @type ref: str + @param resolvers: A list of objects to be used to resolve types. + @type resolvers: [L{sax.element.Element},] + @param defns: An optional target namespace used to qualify references + when no prefix is specified. + @type defns: A default namespace I{tuple: (prefix,uri)} used when ref not prefixed. + @return: A qualified reference. + @rtype: (name, namespace-uri) + """ + ns = None + p, n = splitPrefix(ref) + if p is not None: + if not isinstance(resolvers, (list, tuple)): + resolvers = (resolvers,) + for r in resolvers: + resolved = r.resolvePrefix(p) + if resolved[1] is not None: + ns = resolved + break + if ns is None: + raise Exception('prefix (%s) not resolved' % p) + else: + ns = defns + return (n, ns[1]) + +def isqref(object): + """ + Get whether the object is a I{qualified reference}. + @param object: An object to be tested. + @type object: I{any} + @rtype: boolean + @see: L{qualify} + """ + return (\ + isinstance(object, tuple) and \ + len(object) == 2 and \ + isinstance(object[0], basestring) and \ + isinstance(object[1], basestring)) + + +class Filter: + def __init__(self, inclusive=False, *items): + self.inclusive = inclusive + self.items = items + def __contains__(self, x): + if self.inclusive: + result = ( x in self.items ) + else: + result = ( x not in self.items ) + return result + diff --git a/libs/suds/xsd/deplist.py b/libs/suds/xsd/deplist.py new file mode 100644 index 00000000..14ae19c0 --- /dev/null +++ b/libs/suds/xsd/deplist.py @@ -0,0 +1,140 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{depsolve} module defines a class for performing dependancy solving. +""" + +from logging import getLogger +from suds import * + +log = getLogger(__name__) + + +class DepList: + """ + Dependancy solving list. + Items are tuples: (object, (deps,)) + @ivar raw: The raw (unsorted) items. + @type raw: list + @ivar index: The index of (unsorted) items. + @type index: list + @ivar stack: The sorting stack. + @type stack: list + @ivar pushed: The I{pushed} set tracks items that have been + processed. + @type pushed: set + @ivar sorted: The sorted list of items. + @type sorted: list + """ + + def __init__(self): + """ """ + self.unsorted = [] + self.index = {} + self.stack = [] + self.pushed = set() + self.sorted = None + + def add(self, *items): + """ + Add items to be sorted. + @param items: One or more items to be added. + @type items: I{item} + @return: self + @rtype: L{DepList} + """ + for item in items: + self.unsorted.append(item) + key = item[0] + self.index[key] = item + return self + + def sort(self): + """ + Sort the list based on dependancies. + @return: The sorted items. + @rtype: list + """ + self.sorted = list() + self.pushed = set() + for item in self.unsorted: + popped = [] + self.push(item) + while len(self.stack): + try: + top = self.top() + ref = top[1].next() + refd = self.index.get(ref) + if refd is None: + log.debug('"%s" not found, skipped', Repr(ref)) + continue + self.push(refd) + except StopIteration: + popped.append(self.pop()) + continue + for p in popped: + self.sorted.append(p) + self.unsorted = self.sorted + return self.sorted + + def top(self): + """ + Get the item at the top of the stack. + @return: The top item. + @rtype: (item, iter) + """ + return self.stack[-1] + + def push(self, item): + """ + Push and item onto the sorting stack. + @param item: An item to push. + @type item: I{item} + @return: The number of items pushed. + @rtype: int + """ + if item in self.pushed: + return + frame = (item, iter(item[1])) + self.stack.append(frame) + self.pushed.add(item) + + def pop(self): + """ + Pop the top item off the stack and append + it to the sorted list. + @return: The popped item. + @rtype: I{item} + """ + try: + frame = self.stack.pop() + return frame[0] + except: + pass + + +if __name__ == '__main__': + a = ('a', ('x',)) + b = ('b', ('a',)) + c = ('c', ('a','b')) + d = ('d', ('c',)) + e = ('e', ('d','a')) + f = ('f', ('e','c','d','a')) + x = ('x', ()) + L = DepList() + L.add(c, e, d, b, f, a, x) + print [x[0] for x in L.sort()] \ No newline at end of file diff --git a/libs/suds/xsd/doctor.py b/libs/suds/xsd/doctor.py new file mode 100644 index 00000000..d7bbc14e --- /dev/null +++ b/libs/suds/xsd/doctor.py @@ -0,0 +1,226 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{doctor} module provides classes for fixing broken (sick) +schema(s). +""" + +from logging import getLogger +from suds.sax import splitPrefix, Namespace +from suds.sax.element import Element +from suds.plugin import DocumentPlugin, DocumentContext + +log = getLogger(__name__) + + +class Doctor: + """ + Schema Doctor. + """ + def examine(self, root): + """ + Examine and repair the schema (if necessary). + @param root: A schema root element. + @type root: L{Element} + """ + pass + + +class Practice(Doctor): + """ + A collection of doctors. + @ivar doctors: A list of doctors. + @type doctors: list + """ + + def __init__(self): + self.doctors = [] + + def add(self, doctor): + """ + Add a doctor to the practice + @param doctor: A doctor to add. + @type doctor: L{Doctor} + """ + self.doctors.append(doctor) + + def examine(self, root): + for d in self.doctors: + d.examine(root) + return root + + +class TnsFilter: + """ + Target Namespace filter. + @ivar tns: A list of target namespaces. + @type tns: [str,...] + """ + + def __init__(self, *tns): + """ + @param tns: A list of target namespaces. + @type tns: [str,...] + """ + self.tns = [] + self.add(*tns) + + def add(self, *tns): + """ + Add I{targetNamesapces} to be added. + @param tns: A list of target namespaces. + @type tns: [str,...] + """ + self.tns += tns + + def match(self, root, ns): + """ + Match by I{targetNamespace} excluding those that + are equal to the specified namespace to prevent + adding an import to itself. + @param root: A schema root. + @type root: L{Element} + """ + tns = root.get('targetNamespace') + if len(self.tns): + matched = ( tns in self.tns ) + else: + matched = 1 + itself = ( ns == tns ) + return ( matched and not itself ) + + +class Import: + """ + An to be applied. + @cvar xsdns: The XSD namespace. + @type xsdns: (p,u) + @ivar ns: An import namespace. + @type ns: str + @ivar location: An optional I{schemaLocation}. + @type location: str + @ivar filter: A filter used to restrict application to + a particular schema. + @type filter: L{TnsFilter} + """ + + xsdns = Namespace.xsdns + + def __init__(self, ns, location=None): + """ + @param ns: An import namespace. + @type ns: str + @param location: An optional I{schemaLocation}. + @type location: str + """ + self.ns = ns + self.location = location + self.filter = TnsFilter() + + def setfilter(self, filter): + """ + Set the filter. + @param filter: A filter to set. + @type filter: L{TnsFilter} + """ + self.filter = filter + + def apply(self, root): + """ + Apply the import (rule) to the specified schema. + If the schema does not already contain an import for the + I{namespace} specified here, it is added. + @param root: A schema root. + @type root: L{Element} + """ + if not self.filter.match(root, self.ns): + return + if self.exists(root): + return + node = Element('import', ns=self.xsdns) + node.set('namespace', self.ns) + if self.location is not None: + node.set('schemaLocation', self.location) + log.debug('inserting: %s', node) + root.insert(node) + + def add(self, root): + """ + Add an to the specified schema root. + @param root: A schema root. + @type root: L{Element} + """ + node = Element('import', ns=self.xsdns) + node.set('namespace', self.ns) + if self.location is not None: + node.set('schemaLocation', self.location) + log.debug('%s inserted', node) + root.insert(node) + + def exists(self, root): + """ + Check to see if the already exists + in the specified schema root by matching I{namesapce}. + @param root: A schema root. + @type root: L{Element} + """ + for node in root.children: + if node.name != 'import': + continue + ns = node.get('namespace') + if self.ns == ns: + return 1 + return 0 + + +class ImportDoctor(Doctor, DocumentPlugin): + """ + Doctor used to fix missing imports. + @ivar imports: A list of imports to apply. + @type imports: [L{Import},...] + """ + + def __init__(self, *imports): + """ + """ + self.imports = [] + self.add(*imports) + + def add(self, *imports): + """ + Add a namesapce to be checked. + @param imports: A list of L{Import} objects. + @type imports: [L{Import},..] + """ + self.imports += imports + + def examine(self, node): + for imp in self.imports: + imp.apply(node) + + def parsed(self, context): + node = context.document + # xsd root + if node.name == 'schema' and Namespace.xsd(node.namespace()): + self.examine(node) + return + # look deeper + context = DocumentContext() + for child in node: + context.document = child + self.parsed(context) + \ No newline at end of file diff --git a/libs/suds/xsd/query.py b/libs/suds/xsd/query.py new file mode 100644 index 00000000..c88b2202 --- /dev/null +++ b/libs/suds/xsd/query.py @@ -0,0 +1,208 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{query} module defines a class for performing schema queries. +""" + +from logging import getLogger +from suds import * +from suds.sudsobject import * +from suds.xsd import qualify, isqref +from suds.xsd.sxbuiltin import Factory + +log = getLogger(__name__) + + +class Query(Object): + """ + Schema query base class. + """ + + def __init__(self, ref=None): + """ + @param ref: The schema reference being queried. + @type ref: qref + """ + Object.__init__(self) + self.id = objid(self) + self.ref = ref + self.history = [] + self.resolved = False + if not isqref(self.ref): + raise Exception('%s, must be qref' % tostr(self.ref)) + + def execute(self, schema): + """ + Execute this query using the specified schema. + @param schema: The schema associated with the query. The schema + is used by the query to search for items. + @type schema: L{schema.Schema} + @return: The item matching the search criteria. + @rtype: L{sxbase.SchemaObject} + """ + raise Exception, 'not-implemented by subclass' + + def filter(self, result): + """ + Filter the specified result based on query criteria. + @param result: A potential result. + @type result: L{sxbase.SchemaObject} + @return: True if result should be excluded. + @rtype: boolean + """ + if result is None: + return True + reject = ( result in self.history ) + if reject: + log.debug('result %s, rejected by\n%s', Repr(result), self) + return reject + + def result(self, result): + """ + Query result post processing. + @param result: A query result. + @type result: L{sxbase.SchemaObject} + """ + if result is None: + log.debug('%s, not-found', self.ref) + return + if self.resolved: + result = result.resolve() + log.debug('%s, found as: %s', self.ref, Repr(result)) + self.history.append(result) + return result + + +class BlindQuery(Query): + """ + Schema query class that I{blindly} searches for a reference in + the specified schema. It may be used to find Elements and Types but + will match on an Element first. This query will also find builtins. + """ + + def execute(self, schema): + if schema.builtin(self.ref): + name = self.ref[0] + b = Factory.create(schema, name) + log.debug('%s, found builtin (%s)', self.id, name) + return b + result = None + for d in (schema.elements, schema.types): + result = d.get(self.ref) + if self.filter(result): + result = None + else: + break + if result is None: + eq = ElementQuery(self.ref) + eq.history = self.history + result = eq.execute(schema) + return self.result(result) + + +class TypeQuery(Query): + """ + Schema query class that searches for Type references in + the specified schema. Matches on root types only. + """ + + def execute(self, schema): + if schema.builtin(self.ref): + name = self.ref[0] + b = Factory.create(schema, name) + log.debug('%s, found builtin (%s)', self.id, name) + return b + result = schema.types.get(self.ref) + if self.filter(result): + result = None + return self.result(result) + + +class GroupQuery(Query): + """ + Schema query class that searches for Group references in + the specified schema. + """ + + def execute(self, schema): + result = schema.groups.get(self.ref) + if self.filter(result): + result = None + return self.result(result) + + +class AttrQuery(Query): + """ + Schema query class that searches for Attribute references in + the specified schema. Matches on root Attribute by qname first, then searches + deep into the document. + """ + + def execute(self, schema): + result = schema.attributes.get(self.ref) + if self.filter(result): + result = self.__deepsearch(schema) + return self.result(result) + + def __deepsearch(self, schema): + from suds.xsd.sxbasic import Attribute + result = None + for e in schema.all: + result = e.find(self.ref, (Attribute,)) + if self.filter(result): + result = None + else: + break + return result + + +class AttrGroupQuery(Query): + """ + Schema query class that searches for attributeGroup references in + the specified schema. + """ + + def execute(self, schema): + result = schema.agrps.get(self.ref) + if self.filter(result): + result = None + return self.result(result) + + +class ElementQuery(Query): + """ + Schema query class that searches for Element references in + the specified schema. Matches on root Elements by qname first, then searches + deep into the document. + """ + + def execute(self, schema): + result = schema.elements.get(self.ref) + if self.filter(result): + result = self.__deepsearch(schema) + return self.result(result) + + def __deepsearch(self, schema): + from suds.xsd.sxbasic import Element + result = None + for e in schema.all: + result = e.find(self.ref, (Element,)) + if self.filter(result): + result = None + else: + break + return result \ No newline at end of file diff --git a/libs/suds/xsd/schema.py b/libs/suds/xsd/schema.py new file mode 100644 index 00000000..cb7d678b --- /dev/null +++ b/libs/suds/xsd/schema.py @@ -0,0 +1,422 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{schema} module provides a intelligent representation of +an XSD schema. The I{raw} model is the XML tree and the I{model} +is the denormalized, objectified and intelligent view of the schema. +Most of the I{value-add} provided by the model is centered around +tranparent referenced type resolution and targeted denormalization. +""" + + +import suds.metrics +from suds import * +from suds.xsd import * +from suds.xsd.sxbuiltin import * +from suds.xsd.sxbasic import Factory as BasicFactory +from suds.xsd.sxbuiltin import Factory as BuiltinFactory +from suds.xsd.sxbase import SchemaObject +from suds.xsd.deplist import DepList +from suds.sax.element import Element +from suds.sax import splitPrefix, Namespace +from logging import getLogger + +log = getLogger(__name__) + + +class SchemaCollection: + """ + A collection of schema objects. This class is needed because WSDLs + may contain more then one node. + @ivar wsdl: A wsdl object. + @type wsdl: L{suds.wsdl.Definitions} + @ivar children: A list contained schemas. + @type children: [L{Schema},...] + @ivar namespaces: A dictionary of contained schemas by namespace. + @type namespaces: {str:L{Schema}} + """ + + def __init__(self, wsdl): + """ + @param wsdl: A wsdl object. + @type wsdl: L{suds.wsdl.Definitions} + """ + self.wsdl = wsdl + self.children = [] + self.namespaces = {} + + def add(self, schema): + """ + Add a schema node to the collection. Schema(s) within the same target + namespace are consolidated. + @param schema: A schema object. + @type schema: (L{Schema}) + """ + key = schema.tns[1] + existing = self.namespaces.get(key) + if existing is None: + self.children.append(schema) + self.namespaces[key] = schema + else: + existing.root.children += schema.root.children + existing.root.nsprefixes.update(schema.root.nsprefixes) + + def load(self, options): + """ + Load the schema objects for the root nodes. + - de-references schemas + - merge schemas + @param options: An options dictionary. + @type options: L{options.Options} + @return: The merged schema. + @rtype: L{Schema} + """ + if options.autoblend: + self.autoblend() + for child in self.children: + child.build() + for child in self.children: + child.open_imports(options) + for child in self.children: + child.dereference() + log.debug('loaded:\n%s', self) + merged = self.merge() + log.debug('MERGED:\n%s', merged) + return merged + + def autoblend(self): + """ + Ensure that all schemas within the collection + import each other which has a blending effect. + @return: self + @rtype: L{SchemaCollection} + """ + namespaces = self.namespaces.keys() + for s in self.children: + for ns in namespaces: + tns = s.root.get('targetNamespace') + if tns == ns: + continue + for imp in s.root.getChildren('import'): + if imp.get('namespace') == ns: + continue + imp = Element('import', ns=Namespace.xsdns) + imp.set('namespace', ns) + s.root.append(imp) + return self + + def locate(self, ns): + """ + Find a schema by namespace. Only the URI portion of + the namespace is compared to each schema's I{targetNamespace} + @param ns: A namespace. + @type ns: (prefix,URI) + @return: The schema matching the namesapce, else None. + @rtype: L{Schema} + """ + return self.namespaces.get(ns[1]) + + def merge(self): + """ + Merge the contained schemas into one. + @return: The merged schema. + @rtype: L{Schema} + """ + if len(self): + schema = self.children[0] + for s in self.children[1:]: + schema.merge(s) + return schema + else: + return None + + def __len__(self): + return len(self.children) + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + result = ['\nschema collection'] + for s in self.children: + result.append(s.str(1)) + return '\n'.join(result) + + +class Schema: + """ + The schema is an objectification of a (xsd) definition. + It provides inspection, lookup and type resolution. + @ivar root: The root node. + @type root: L{sax.element.Element} + @ivar baseurl: The I{base} URL for this schema. + @type baseurl: str + @ivar container: A schema collection containing this schema. + @type container: L{SchemaCollection} + @ivar children: A list of direct top level children. + @type children: [L{SchemaObject},...] + @ivar all: A list of all (includes imported) top level children. + @type all: [L{SchemaObject},...] + @ivar types: A schema types cache. + @type types: {name:L{SchemaObject}} + @ivar imports: A list of import objects. + @type imports: [L{SchemaObject},...] + @ivar elements: A list of objects. + @type elements: [L{SchemaObject},...] + @ivar attributes: A list of objects. + @type attributes: [L{SchemaObject},...] + @ivar groups: A list of group objects. + @type groups: [L{SchemaObject},...] + @ivar agrps: A list of attribute group objects. + @type agrps: [L{SchemaObject},...] + @ivar form_qualified: The flag indicating: + (@elementFormDefault). + @type form_qualified: bool + """ + + Tag = 'schema' + + def __init__(self, root, baseurl, options, container=None): + """ + @param root: The xml root. + @type root: L{sax.element.Element} + @param baseurl: The base url used for importing. + @type baseurl: basestring + @param options: An options dictionary. + @type options: L{options.Options} + @param container: An optional container. + @type container: L{SchemaCollection} + """ + self.root = root + self.id = objid(self) + self.tns = self.mktns() + self.baseurl = baseurl + self.container = container + self.children = [] + self.all = [] + self.types = {} + self.imports = [] + self.elements = {} + self.attributes = {} + self.groups = {} + self.agrps = {} + if options.doctor is not None: + options.doctor.examine(root) + form = self.root.get('elementFormDefault') + if form is None: + self.form_qualified = False + else: + self.form_qualified = ( form == 'qualified' ) + if container is None: + self.build() + self.open_imports(options) + log.debug('built:\n%s', self) + self.dereference() + log.debug('dereferenced:\n%s', self) + + def mktns(self): + """ + Make the schema's target namespace. + @return: The namespace representation of the schema's + targetNamespace value. + @rtype: (prefix, uri) + """ + tns = [None, self.root.get('targetNamespace')] + if tns[1] is not None: + tns[0] = self.root.findPrefix(tns[1]) + return tuple(tns) + + def build(self): + """ + Build the schema (object graph) using the root node + using the factory. + - Build the graph. + - Collate the children. + """ + self.children = BasicFactory.build(self.root, self) + collated = BasicFactory.collate(self.children) + self.children = collated[0] + self.attributes = collated[2] + self.imports = collated[1] + self.elements = collated[3] + self.types = collated[4] + self.groups = collated[5] + self.agrps = collated[6] + + def merge(self, schema): + """ + Merge the contents from the schema. Only objects not already contained + in this schema's collections are merged. This is to provide for bidirectional + import which produce cyclic includes. + @returns: self + @rtype: L{Schema} + """ + for item in schema.attributes.items(): + if item[0] in self.attributes: + continue + self.all.append(item[1]) + self.attributes[item[0]] = item[1] + for item in schema.elements.items(): + if item[0] in self.elements: + continue + self.all.append(item[1]) + self.elements[item[0]] = item[1] + for item in schema.types.items(): + if item[0] in self.types: + continue + self.all.append(item[1]) + self.types[item[0]] = item[1] + for item in schema.groups.items(): + if item[0] in self.groups: + continue + self.all.append(item[1]) + self.groups[item[0]] = item[1] + for item in schema.agrps.items(): + if item[0] in self.agrps: + continue + self.all.append(item[1]) + self.agrps[item[0]] = item[1] + schema.merged = True + return self + + def open_imports(self, options): + """ + Instruct all contained L{sxbasic.Import} children to import + the schema's which they reference. The contents of the + imported schema are I{merged} in. + @param options: An options dictionary. + @type options: L{options.Options} + """ + for imp in self.imports: + imported = imp.open(options) + if imported is None: + continue + imported.open_imports(options) + log.debug('imported:\n%s', imported) + self.merge(imported) + + def dereference(self): + """ + Instruct all children to perform dereferencing. + """ + all = [] + indexes = {} + for child in self.children: + child.content(all) + deplist = DepList() + for x in all: + x.qualify() + midx, deps = x.dependencies() + item = (x, tuple(deps)) + deplist.add(item) + indexes[x] = midx + for x, deps in deplist.sort(): + midx = indexes.get(x) + if midx is None: continue + d = deps[midx] + log.debug('(%s) merging %s <== %s', self.tns[1], Repr(x), Repr(d)) + x.merge(d) + + def locate(self, ns): + """ + Find a schema by namespace. Only the URI portion of + the namespace is compared to each schema's I{targetNamespace}. + The request is passed to the container. + @param ns: A namespace. + @type ns: (prefix,URI) + @return: The schema matching the namesapce, else None. + @rtype: L{Schema} + """ + if self.container is not None: + return self.container.locate(ns) + else: + return None + + def custom(self, ref, context=None): + """ + Get whether the specified reference is B{not} an (xs) builtin. + @param ref: A str or qref. + @type ref: (str|qref) + @return: True if B{not} a builtin, else False. + @rtype: bool + """ + if ref is None: + return True + else: + return ( not self.builtin(ref, context) ) + + def builtin(self, ref, context=None): + """ + Get whether the specified reference is an (xs) builtin. + @param ref: A str or qref. + @type ref: (str|qref) + @return: True if builtin, else False. + @rtype: bool + """ + w3 = 'http://www.w3.org' + try: + if isqref(ref): + ns = ref[1] + return ( ref[0] in Factory.tags and ns.startswith(w3) ) + if context is None: + context = self.root + prefix = splitPrefix(ref)[0] + prefixes = context.findPrefixes(w3, 'startswith') + return ( prefix in prefixes and ref[0] in Factory.tags ) + except: + return False + + def instance(self, root, baseurl, options): + """ + Create and return an new schema object using the + specified I{root} and I{url}. + @param root: A schema root node. + @type root: L{sax.element.Element} + @param baseurl: A base URL. + @type baseurl: str + @param options: An options dictionary. + @type options: L{options.Options} + @return: The newly created schema object. + @rtype: L{Schema} + @note: This is only used by Import children. + """ + return Schema(root, baseurl, options) + + def str(self, indent=0): + tab = '%*s'%(indent*3, '') + result = [] + result.append('%s%s' % (tab, self.id)) + result.append('%s(raw)' % tab) + result.append(self.root.str(indent+1)) + result.append('%s(model)' % tab) + for c in self.children: + result.append(c.str(indent+1)) + result.append('') + return '\n'.join(result) + + def __repr__(self): + myrep = '<%s tns="%s"/>' % (self.id, self.tns[1]) + return myrep.encode('utf-8') + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + return self.str() + + + diff --git a/libs/suds/xsd/sxbase.py b/libs/suds/xsd/sxbase.py new file mode 100644 index 00000000..2577ffd5 --- /dev/null +++ b/libs/suds/xsd/sxbase.py @@ -0,0 +1,669 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{sxbase} module provides I{base} classes that represent +schema objects. +""" + +from logging import getLogger +from suds import * +from suds.xsd import * +from suds.sax.element import Element +from suds.sax import Namespace + +log = getLogger(__name__) + + +class SchemaObject(object): + """ + A schema object is an extension to object object with + with schema awareness. + @ivar root: The XML root element. + @type root: L{Element} + @ivar schema: The schema containing this object. + @type schema: L{schema.Schema} + @ivar form_qualified: A flag that inidcates that @elementFormDefault + has a value of I{qualified}. + @type form_qualified: boolean + @ivar nillable: A flag that inidcates that @nillable + has a value of I{true}. + @type nillable: boolean + @ivar default: The default value. + @type default: object + @ivar rawchildren: A list raw of all children. + @type rawchildren: [L{SchemaObject},...] + """ + + @classmethod + def prepend(cls, d, s, filter=Filter()): + """ + Prepend schema object's from B{s}ource list to + the B{d}estination list while applying the filter. + @param d: The destination list. + @type d: list + @param s: The source list. + @type s: list + @param filter: A filter that allows items to be prepended. + @type filter: L{Filter} + """ + i = 0 + for x in s: + if x in filter: + d.insert(i, x) + i += 1 + + @classmethod + def append(cls, d, s, filter=Filter()): + """ + Append schema object's from B{s}ource list to + the B{d}estination list while applying the filter. + @param d: The destination list. + @type d: list + @param s: The source list. + @type s: list + @param filter: A filter that allows items to be appended. + @type filter: L{Filter} + """ + for item in s: + if item in filter: + d.append(item) + + def __init__(self, schema, root): + """ + @param schema: The containing schema. + @type schema: L{schema.Schema} + @param root: The xml root node. + @type root: L{Element} + """ + self.schema = schema + self.root = root + self.id = objid(self) + self.name = root.get('name') + self.qname = (self.name, schema.tns[1]) + self.min = root.get('minOccurs') + self.max = root.get('maxOccurs') + self.type = root.get('type') + self.ref = root.get('ref') + self.form_qualified = schema.form_qualified + self.nillable = False + self.default = root.get('default') + self.rawchildren = [] + self.cache = {} + + def attributes(self, filter=Filter()): + """ + Get only the attribute content. + @param filter: A filter to constrain the result. + @type filter: L{Filter} + @return: A list of tuples (attr, ancestry) + @rtype: [(L{SchemaObject}, [L{SchemaObject},..]),..] + """ + result = [] + for child, ancestry in self: + if child.isattr() and child in filter: + result.append((child, ancestry)) + return result + + def children(self, filter=Filter()): + """ + Get only the I{direct} or non-attribute content. + @param filter: A filter to constrain the result. + @type filter: L{Filter} + @return: A list tuples: (child, ancestry) + @rtype: [(L{SchemaObject}, [L{SchemaObject},..]),..] + """ + result = [] + for child, ancestry in self: + if not child.isattr() and child in filter: + result.append((child, ancestry)) + return result + + def get_attribute(self, name): + """ + Get (find) a I{non-attribute} attribute by name. + @param name: A attribute name. + @type name: str + @return: A tuple: the requested (attribute, ancestry). + @rtype: (L{SchemaObject}, [L{SchemaObject},..]) + """ + for child, ancestry in self.attributes(): + if child.name == name: + return (child, ancestry) + return (None, []) + + def get_child(self, name): + """ + Get (find) a I{non-attribute} child by name. + @param name: A child name. + @type name: str + @return: A tuple: the requested (child, ancestry). + @rtype: (L{SchemaObject}, [L{SchemaObject},..]) + """ + for child, ancestry in self.children(): + if child.any() or child.name == name: + return (child, ancestry) + return (None, []) + + def namespace(self, prefix=None): + """ + Get this properties namespace + @param prefix: The default prefix. + @type prefix: str + @return: The schema's target namespace + @rtype: (I{prefix},I{URI}) + """ + ns = self.schema.tns + if ns[0] is None: + ns = (prefix, ns[1]) + return ns + + def default_namespace(self): + return self.root.defaultNamespace() + + def unbounded(self): + """ + Get whether this node is unbounded I{(a collection)} + @return: True if unbounded, else False. + @rtype: boolean + """ + max = self.max + if max is None: + max = '1' + if max.isdigit(): + return (int(max) > 1) + else: + return ( max == 'unbounded' ) + + def optional(self): + """ + Get whether this type is optional. + @return: True if optional, else False + @rtype: boolean + """ + min = self.min + if min is None: + min = '1' + return ( min == '0' ) + + def required(self): + """ + Get whether this type is required. + @return: True if required, else False + @rtype: boolean + """ + return ( not self.optional() ) + + + def resolve(self, nobuiltin=False): + """ + Resolve and return the nodes true self. + @param nobuiltin: Flag indicates that resolution must + not continue to include xsd builtins. + @return: The resolved (true) type. + @rtype: L{SchemaObject} + """ + return self.cache.get(nobuiltin, self) + + def sequence(self): + """ + Get whether this is an + @return: True if , else False + @rtype: boolean + """ + return False + + def xslist(self): + """ + Get whether this is an + @return: True if any, else False + @rtype: boolean + """ + return False + + def all(self): + """ + Get whether this is an + @return: True if any, else False + @rtype: boolean + """ + return False + + def choice(self): + """ + Get whether this is n + @return: True if any, else False + @rtype: boolean + """ + return False + + def any(self): + """ + Get whether this is an + @return: True if any, else False + @rtype: boolean + """ + return False + + def builtin(self): + """ + Get whether this is a schema-instance (xs) type. + @return: True if any, else False + @rtype: boolean + """ + return False + + def enum(self): + """ + Get whether this is a simple-type containing an enumeration. + @return: True if any, else False + @rtype: boolean + """ + return False + + def isattr(self): + """ + Get whether the object is a schema I{attribute} definition. + @return: True if an attribute, else False. + @rtype: boolean + """ + return False + + def extension(self): + """ + Get whether the object is an extension of another type. + @return: True if an extension, else False. + @rtype: boolean + """ + return False + + def restriction(self): + """ + Get whether the object is an restriction of another type. + @return: True if an restriction, else False. + @rtype: boolean + """ + return False + + def mixed(self): + """ + Get whether this I{mixed} content. + """ + return False + + def find(self, qref, classes=()): + """ + Find a referenced type in self or children. + @param qref: A qualified reference. + @type qref: qref + @param classes: A list of classes used to qualify the match. + @type classes: [I{class},...] + @return: The referenced type. + @rtype: L{SchemaObject} + @see: L{qualify()} + """ + if not len(classes): + classes = (self.__class__,) + if self.qname == qref and self.__class__ in classes: + return self + for c in self.rawchildren: + p = c.find(qref, classes) + if p is not None: + return p + return None + + def translate(self, value, topython=True): + """ + Translate a value (type) to/from a python type. + @param value: A value to translate. + @return: The converted I{language} type. + """ + return value + + def childtags(self): + """ + Get a list of valid child tag names. + @return: A list of child tag names. + @rtype: [str,...] + """ + return () + + def dependencies(self): + """ + Get a list of dependancies for dereferencing. + @return: A merge dependancy index and a list of dependancies. + @rtype: (int, [L{SchemaObject},...]) + """ + return (None, []) + + def autoqualified(self): + """ + The list of I{auto} qualified attribute values. + Qualification means to convert values into I{qref}. + @return: A list of attibute names. + @rtype: list + """ + return ['type', 'ref'] + + def qualify(self): + """ + Convert attribute values, that are references to other + objects, into I{qref}. Qualfied using default document namespace. + Since many wsdls are written improperly: when the document does + not define a default namespace, the schema target namespace is used + to qualify references. + """ + defns = self.root.defaultNamespace() + if Namespace.none(defns): + defns = self.schema.tns + for a in self.autoqualified(): + ref = getattr(self, a) + if ref is None: + continue + if isqref(ref): + continue + qref = qualify(ref, self.root, defns) + log.debug('%s, convert %s="%s" to %s', self.id, a, ref, qref) + setattr(self, a, qref) + + def merge(self, other): + """ + Merge another object as needed. + """ + other.qualify() + for n in ('name', + 'qname', + 'min', + 'max', + 'default', + 'type', + 'nillable', + 'form_qualified',): + if getattr(self, n) is not None: + continue + v = getattr(other, n) + if v is None: + continue + setattr(self, n, v) + + + def content(self, collection=None, filter=Filter(), history=None): + """ + Get a I{flattened} list of this nodes contents. + @param collection: A list to fill. + @type collection: list + @param filter: A filter used to constrain the result. + @type filter: L{Filter} + @param history: The history list used to prevent cyclic dependency. + @type history: list + @return: The filled list. + @rtype: list + """ + if collection is None: + collection = [] + if history is None: + history = [] + if self in history: + return collection + history.append(self) + if self in filter: + collection.append(self) + for c in self.rawchildren: + c.content(collection, filter, history[:]) + return collection + + def str(self, indent=0, history=None): + """ + Get a string representation of this object. + @param indent: The indent. + @type indent: int + @return: A string. + @rtype: str + """ + if history is None: + history = [] + if self in history: + return '%s ...' % Repr(self) + history.append(self) + tab = '%*s'%(indent*3, '') + result = [] + result.append('%s<%s' % (tab, self.id)) + for n in self.description(): + if not hasattr(self, n): + continue + v = getattr(self, n) + if v is None: + continue + result.append(' %s="%s"' % (n, v)) + if len(self): + result.append('>') + for c in self.rawchildren: + result.append('\n') + result.append(c.str(indent+1, history[:])) + if c.isattr(): + result.append('@') + result.append('\n%s' % tab) + result.append('' % self.__class__.__name__) + else: + result.append(' />') + return ''.join(result) + + def description(self): + """ + Get the names used for str() and repr() description. + @return: A dictionary of relavent attributes. + @rtype: [str,...] + """ + return () + + def __str__(self): + return unicode(self).encode('utf-8') + + def __unicode__(self): + return unicode(self.str()) + + def __repr__(self): + s = [] + s.append('<%s' % self.id) + for n in self.description(): + if not hasattr(self, n): + continue + v = getattr(self, n) + if v is None: + continue + s.append(' %s="%s"' % (n, v)) + s.append(' />') + myrep = ''.join(s) + return myrep.encode('utf-8') + + def __len__(self): + n = 0 + for x in self: n += 1 + return n + + def __iter__(self): + return Iter(self) + + def __getitem__(self, index): + i = 0 + for c in self: + if i == index: + return c + + +class Iter: + """ + The content iterator - used to iterate the L{Content} children. The iterator + provides a I{view} of the children that is free of container elements + such as and . + @ivar stack: A stack used to control nesting. + @type stack: list + """ + + class Frame: + """ A content iterator frame. """ + + def __init__(self, sx): + """ + @param sx: A schema object. + @type sx: L{SchemaObject} + """ + self.sx = sx + self.items = sx.rawchildren + self.index = 0 + + def next(self): + """ + Get the I{next} item in the frame's collection. + @return: The next item or None + @rtype: L{SchemaObject} + """ + if self.index < len(self.items): + result = self.items[self.index] + self.index += 1 + return result + + def __init__(self, sx): + """ + @param sx: A schema object. + @type sx: L{SchemaObject} + """ + self.stack = [] + self.push(sx) + + def push(self, sx): + """ + Create a frame and push the specified object. + @param sx: A schema object to push. + @type sx: L{SchemaObject} + """ + self.stack.append(Iter.Frame(sx)) + + def pop(self): + """ + Pop the I{top} frame. + @return: The popped frame. + @rtype: L{Frame} + @raise StopIteration: when stack is empty. + """ + if len(self.stack): + return self.stack.pop() + else: + raise StopIteration() + + def top(self): + """ + Get the I{top} frame. + @return: The top frame. + @rtype: L{Frame} + @raise StopIteration: when stack is empty. + """ + if len(self.stack): + return self.stack[-1] + else: + raise StopIteration() + + def next(self): + """ + Get the next item. + @return: A tuple: the next (child, ancestry). + @rtype: (L{SchemaObject}, [L{SchemaObject},..]) + @raise StopIteration: A the end. + """ + frame = self.top() + while True: + result = frame.next() + if result is None: + self.pop() + return self.next() + if isinstance(result, Content): + ancestry = [f.sx for f in self.stack] + return (result, ancestry) + self.push(result) + return self.next() + + def __iter__(self): + return self + + +class XBuiltin(SchemaObject): + """ + Represents an (xsd) schema node + """ + + def __init__(self, schema, name): + """ + @param schema: The containing schema. + @type schema: L{schema.Schema} + """ + root = Element(name) + SchemaObject.__init__(self, schema, root) + self.name = name + self.nillable = True + + def namespace(self, prefix=None): + return Namespace.xsdns + + def builtin(self): + return True + + def resolve(self, nobuiltin=False): + return self + + +class Content(SchemaObject): + """ + This class represents those schema objects that represent + real XML document content. + """ + pass + + +class NodeFinder: + """ + Find nodes based on flexable criteria. The I{matcher} is + may be any object that implements a match(n) method. + @ivar matcher: An object used as criteria for match. + @type matcher: I{any}.match(n) + @ivar limit: Limit the number of matches. 0=unlimited. + @type limit: int + """ + def __init__(self, matcher, limit=0): + """ + @param matcher: An object used as criteria for match. + @type matcher: I{any}.match(n) + @param limit: Limit the number of matches. 0=unlimited. + @type limit: int + """ + self.matcher = matcher + self.limit = limit + + def find(self, node, list): + """ + Traverse the tree looking for matches. + @param node: A node to match on. + @type node: L{SchemaObject} + @param list: A list to fill. + @type list: list + """ + if self.matcher.match(node): + list.append(node) + self.limit -= 1 + if self.limit == 0: + return + for c in node.rawchildren: + self.find(c, list) + return self \ No newline at end of file diff --git a/libs/suds/xsd/sxbasic.py b/libs/suds/xsd/sxbasic.py new file mode 100644 index 00000000..2506e041 --- /dev/null +++ b/libs/suds/xsd/sxbasic.py @@ -0,0 +1,825 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{sxbasic} module provides classes that represent +I{basic} schema objects. +""" + +from logging import getLogger +from suds import * +from suds.xsd import * +from suds.xsd.sxbase import * +from suds.xsd.query import * +from suds.sax import splitPrefix, Namespace +from suds.transport import TransportError +from suds.reader import DocumentReader +from urlparse import urljoin + + +log = getLogger(__name__) + + +class RestrictionMatcher: + """ + For use with L{NodeFinder} to match restriction. + """ + def match(self, n): + return isinstance(n, Restriction) + + +class TypedContent(Content): + """ + Represents any I{typed} content. + """ + def resolve(self, nobuiltin=False): + qref = self.qref() + if qref is None: + return self + key = 'resolved:nb=%s' % nobuiltin + cached = self.cache.get(key) + if cached is not None: + return cached + result = self + query = TypeQuery(qref) + query.history = [self] + log.debug('%s, resolving: %s\n using:%s', self.id, qref, query) + resolved = query.execute(self.schema) + if resolved is None: + log.debug(self.schema) + raise TypeNotFound(qref) + self.cache[key] = resolved + if resolved.builtin(): + if nobuiltin: + result = self + else: + result = resolved + else: + result = resolved.resolve(nobuiltin) + return result + + def qref(self): + """ + Get the I{type} qualified reference to the referenced xsd type. + This method takes into account simple types defined through + restriction with are detected by determining that self is simple + (len=0) and by finding a restriction child. + @return: The I{type} qualified reference. + @rtype: qref + """ + qref = self.type + if qref is None and len(self) == 0: + ls = [] + m = RestrictionMatcher() + finder = NodeFinder(m, 1) + finder.find(self, ls) + if len(ls): + return ls[0].ref + return qref + + +class Complex(SchemaObject): + """ + Represents an (xsd) schema node. + @cvar childtags: A list of valid child node names + @type childtags: (I{str},...) + """ + + def childtags(self): + return ( + 'attribute', + 'attributeGroup', + 'sequence', + 'all', + 'choice', + 'complexContent', + 'simpleContent', + 'any', + 'group') + + def description(self): + return ('name',) + + def extension(self): + for c in self.rawchildren: + if c.extension(): + return True + return False + + def mixed(self): + for c in self.rawchildren: + if isinstance(c, SimpleContent) and c.mixed(): + return True + return False + + +class Group(SchemaObject): + """ + Represents an (xsd) schema node. + @cvar childtags: A list of valid child node names + @type childtags: (I{str},...) + """ + + def childtags(self): + return ('sequence', 'all', 'choice') + + def dependencies(self): + deps = [] + midx = None + if self.ref is not None: + query = GroupQuery(self.ref) + g = query.execute(self.schema) + if g is None: + log.debug(self.schema) + raise TypeNotFound(self.ref) + deps.append(g) + midx = 0 + return (midx, deps) + + def merge(self, other): + SchemaObject.merge(self, other) + self.rawchildren = other.rawchildren + + def description(self): + return ('name', 'ref',) + + +class AttributeGroup(SchemaObject): + """ + Represents an (xsd) schema node. + @cvar childtags: A list of valid child node names + @type childtags: (I{str},...) + """ + + def childtags(self): + return ('attribute', 'attributeGroup') + + def dependencies(self): + deps = [] + midx = None + if self.ref is not None: + query = AttrGroupQuery(self.ref) + ag = query.execute(self.schema) + if ag is None: + log.debug(self.schema) + raise TypeNotFound(self.ref) + deps.append(ag) + midx = 0 + return (midx, deps) + + def merge(self, other): + SchemaObject.merge(self, other) + self.rawchildren = other.rawchildren + + def description(self): + return ('name', 'ref',) + + +class Simple(SchemaObject): + """ + Represents an (xsd) schema node + """ + + def childtags(self): + return ('restriction', 'any', 'list',) + + def enum(self): + for child, ancestry in self.children(): + if isinstance(child, Enumeration): + return True + return False + + def mixed(self): + return len(self) + + def description(self): + return ('name',) + + def extension(self): + for c in self.rawchildren: + if c.extension(): + return True + return False + + def restriction(self): + for c in self.rawchildren: + if c.restriction(): + return True + return False + + +class List(SchemaObject): + """ + Represents an (xsd) schema node + """ + + def childtags(self): + return () + + def description(self): + return ('name',) + + def xslist(self): + return True + + +class Restriction(SchemaObject): + """ + Represents an (xsd) schema node + """ + + def __init__(self, schema, root): + SchemaObject.__init__(self, schema, root) + self.ref = root.get('base') + + def childtags(self): + return ('enumeration', 'attribute', 'attributeGroup') + + def dependencies(self): + deps = [] + midx = None + if self.ref is not None: + query = TypeQuery(self.ref) + super = query.execute(self.schema) + if super is None: + log.debug(self.schema) + raise TypeNotFound(self.ref) + if not super.builtin(): + deps.append(super) + midx = 0 + return (midx, deps) + + def restriction(self): + return True + + def merge(self, other): + SchemaObject.merge(self, other) + filter = Filter(False, self.rawchildren) + self.prepend(self.rawchildren, other.rawchildren, filter) + + def description(self): + return ('ref',) + + +class Collection(SchemaObject): + """ + Represents an (xsd) schema collection node: + - sequence + - choice + - all + """ + + def childtags(self): + return ('element', 'sequence', 'all', 'choice', 'any', 'group') + + +class Sequence(Collection): + """ + Represents an (xsd) schema node. + """ + def sequence(self): + return True + + +class All(Collection): + """ + Represents an (xsd) schema node. + """ + def all(self): + return True + +class Choice(Collection): + """ + Represents an (xsd) schema node. + """ + def choice(self): + return True + + +class ComplexContent(SchemaObject): + """ + Represents an (xsd) schema node. + """ + + def childtags(self): + return ('attribute', 'attributeGroup', 'extension', 'restriction') + + def extension(self): + for c in self.rawchildren: + if c.extension(): + return True + return False + + def restriction(self): + for c in self.rawchildren: + if c.restriction(): + return True + return False + + +class SimpleContent(SchemaObject): + """ + Represents an (xsd) schema node. + """ + + def childtags(self): + return ('extension', 'restriction') + + def extension(self): + for c in self.rawchildren: + if c.extension(): + return True + return False + + def restriction(self): + for c in self.rawchildren: + if c.restriction(): + return True + return False + + def mixed(self): + return len(self) + + +class Enumeration(Content): + """ + Represents an (xsd) schema node + """ + + def __init__(self, schema, root): + Content.__init__(self, schema, root) + self.name = root.get('value') + + def enum(self): + return True + + +class Element(TypedContent): + """ + Represents an (xsd) schema node. + """ + + def __init__(self, schema, root): + TypedContent.__init__(self, schema, root) + a = root.get('form') + if a is not None: + self.form_qualified = ( a == 'qualified' ) + a = self.root.get('nillable') + if a is not None: + self.nillable = ( a in ('1', 'true') ) + self.implany() + + def implany(self): + """ + Set the type as any when implicit. + An implicit is when an element has not + body and no type defined. + @return: self + @rtype: L{Element} + """ + if self.type is None and \ + self.ref is None and \ + self.root.isempty(): + self.type = self.anytype() + return self + + def childtags(self): + return ('attribute', 'simpleType', 'complexType', 'any',) + + def extension(self): + for c in self.rawchildren: + if c.extension(): + return True + return False + + def restriction(self): + for c in self.rawchildren: + if c.restriction(): + return True + return False + + def dependencies(self): + deps = [] + midx = None + if self.ref is not None: + query = ElementQuery(self.ref) + e = query.execute(self.schema) + if e is None: + log.debug(self.schema) + raise TypeNotFound(self.ref) + deps.append(e) + midx = 0 + return (midx, deps) + + def merge(self, other): + SchemaObject.merge(self, other) + self.rawchildren = other.rawchildren + + def description(self): + return ('name', 'ref', 'type') + + def anytype(self): + """ create an xsd:anyType reference """ + p,u = Namespace.xsdns + mp = self.root.findPrefix(u) + if mp is None: + mp = p + self.root.addPrefix(p, u) + return ':'.join((mp, 'anyType')) + + +class Extension(SchemaObject): + """ + Represents an (xsd) schema node. + """ + + def __init__(self, schema, root): + SchemaObject.__init__(self, schema, root) + self.ref = root.get('base') + + def childtags(self): + return ('attribute', + 'attributeGroup', + 'sequence', + 'all', + 'choice', + 'group') + + def dependencies(self): + deps = [] + midx = None + if self.ref is not None: + query = TypeQuery(self.ref) + super = query.execute(self.schema) + if super is None: + log.debug(self.schema) + raise TypeNotFound(self.ref) + if not super.builtin(): + deps.append(super) + midx = 0 + return (midx, deps) + + def merge(self, other): + SchemaObject.merge(self, other) + filter = Filter(False, self.rawchildren) + self.prepend(self.rawchildren, other.rawchildren, filter) + + def extension(self): + return ( self.ref is not None ) + + def description(self): + return ('ref',) + + +class Import(SchemaObject): + """ + Represents an (xsd) schema node + @cvar locations: A dictionary of namespace locations. + @type locations: dict + @ivar ns: The imported namespace. + @type ns: str + @ivar location: The (optional) location. + @type location: namespace-uri + @ivar opened: Opened and I{imported} flag. + @type opened: boolean + """ + + locations = {} + + @classmethod + def bind(cls, ns, location=None): + """ + Bind a namespace to a schema location (URI). + This is used for imports that don't specify a schemaLocation. + @param ns: A namespace-uri. + @type ns: str + @param location: The (optional) schema location for the + namespace. (default=ns). + @type location: str + """ + if location is None: + location = ns + cls.locations[ns] = location + + def __init__(self, schema, root): + SchemaObject.__init__(self, schema, root) + self.ns = (None, root.get('namespace')) + self.location = root.get('schemaLocation') + if self.location is None: + self.location = self.locations.get(self.ns[1]) + self.opened = False + + def open(self, options): + """ + Open and import the refrenced schema. + @param options: An options dictionary. + @type options: L{options.Options} + @return: The referenced schema. + @rtype: L{Schema} + """ + if self.opened: + return + self.opened = True + log.debug('%s, importing ns="%s", location="%s"', self.id, self.ns[1], self.location) + result = self.locate() + if result is None: + if self.location is None: + log.debug('imported schema (%s) not-found', self.ns[1]) + else: + result = self.download(options) + log.debug('imported:\n%s', result) + return result + + def locate(self): + """ find the schema locally """ + if self.ns[1] == self.schema.tns[1]: + return None + else: + return self.schema.locate(self.ns) + + def download(self, options): + """ download the schema """ + url = self.location + try: + if '://' not in url: + url = urljoin(self.schema.baseurl, url) + reader = DocumentReader(options) + d = reader.open(url) + root = d.root() + root.set('url', url) + return self.schema.instance(root, url, options) + except TransportError: + msg = 'imported schema (%s) at (%s), failed' % (self.ns[1], url) + log.error('%s, %s', self.id, msg, exc_info=True) + raise Exception(msg) + + def description(self): + return ('ns', 'location') + + +class Include(SchemaObject): + """ + Represents an (xsd) schema node + @ivar location: The (optional) location. + @type location: namespace-uri + @ivar opened: Opened and I{imported} flag. + @type opened: boolean + """ + + locations = {} + + def __init__(self, schema, root): + SchemaObject.__init__(self, schema, root) + self.location = root.get('schemaLocation') + if self.location is None: + self.location = self.locations.get(self.ns[1]) + self.opened = False + + def open(self, options): + """ + Open and include the refrenced schema. + @param options: An options dictionary. + @type options: L{options.Options} + @return: The referenced schema. + @rtype: L{Schema} + """ + if self.opened: + return + self.opened = True + log.debug('%s, including location="%s"', self.id, self.location) + result = self.download(options) + log.debug('included:\n%s', result) + return result + + def download(self, options): + """ download the schema """ + url = self.location + try: + if '://' not in url: + url = urljoin(self.schema.baseurl, url) + reader = DocumentReader(options) + d = reader.open(url) + root = d.root() + root.set('url', url) + self.__applytns(root) + return self.schema.instance(root, url, options) + except TransportError: + msg = 'include schema at (%s), failed' % url + log.error('%s, %s', self.id, msg, exc_info=True) + raise Exception(msg) + + def __applytns(self, root): + """ make sure included schema has same tns. """ + TNS = 'targetNamespace' + tns = root.get(TNS) + if tns is None: + tns = self.schema.tns[1] + root.set(TNS, tns) + else: + if self.schema.tns[1] != tns: + raise Exception, '%s mismatch' % TNS + + + def description(self): + return ('location') + + +class Attribute(TypedContent): + """ + Represents an (xsd) node + """ + + def __init__(self, schema, root): + TypedContent.__init__(self, schema, root) + self.use = root.get('use', default='') + + def childtags(self): + return ('restriction',) + + def isattr(self): + return True + + def get_default(self): + """ + Gets the attribute value. + @return: The default value for the attribute + @rtype: str + """ + return self.root.get('default', default='') + + def optional(self): + return ( self.use != 'required' ) + + def dependencies(self): + deps = [] + midx = None + if self.ref is not None: + query = AttrQuery(self.ref) + a = query.execute(self.schema) + if a is None: + log.debug(self.schema) + raise TypeNotFound(self.ref) + deps.append(a) + midx = 0 + return (midx, deps) + + def description(self): + return ('name', 'ref', 'type') + + +class Any(Content): + """ + Represents an (xsd) node + """ + + def get_child(self, name): + root = self.root.clone() + root.set('note', 'synthesized (any) child') + child = Any(self.schema, root) + return (child, []) + + def get_attribute(self, name): + root = self.root.clone() + root.set('note', 'synthesized (any) attribute') + attribute = Any(self.schema, root) + return (attribute, []) + + def any(self): + return True + + +class Factory: + """ + @cvar tags: A factory to create object objects based on tag. + @type tags: {tag:fn,} + """ + + tags =\ + { + 'import' : Import, + 'include' : Include, + 'complexType' : Complex, + 'group' : Group, + 'attributeGroup' : AttributeGroup, + 'simpleType' : Simple, + 'list' : List, + 'element' : Element, + 'attribute' : Attribute, + 'sequence' : Sequence, + 'all' : All, + 'choice' : Choice, + 'complexContent' : ComplexContent, + 'simpleContent' : SimpleContent, + 'restriction' : Restriction, + 'enumeration' : Enumeration, + 'extension' : Extension, + 'any' : Any, + } + + @classmethod + def maptag(cls, tag, fn): + """ + Map (override) tag => I{class} mapping. + @param tag: An xsd tag name. + @type tag: str + @param fn: A function or class. + @type fn: fn|class. + """ + cls.tags[tag] = fn + + @classmethod + def create(cls, root, schema): + """ + Create an object based on the root tag name. + @param root: An XML root element. + @type root: L{Element} + @param schema: A schema object. + @type schema: L{schema.Schema} + @return: The created object. + @rtype: L{SchemaObject} + """ + fn = cls.tags.get(root.name) + if fn is not None: + return fn(schema, root) + else: + return None + + @classmethod + def build(cls, root, schema, filter=('*',)): + """ + Build an xsobject representation. + @param root: An schema XML root. + @type root: L{sax.element.Element} + @param filter: A tag filter. + @type filter: [str,...] + @return: A schema object graph. + @rtype: L{sxbase.SchemaObject} + """ + children = [] + for node in root.getChildren(ns=Namespace.xsdns): + if '*' in filter or node.name in filter: + child = cls.create(node, schema) + if child is None: + continue + children.append(child) + c = cls.build(node, schema, child.childtags()) + child.rawchildren = c + return children + + @classmethod + def collate(cls, children): + imports = [] + elements = {} + attributes = {} + types = {} + groups = {} + agrps = {} + for c in children: + if isinstance(c, (Import, Include)): + imports.append(c) + continue + if isinstance(c, Attribute): + attributes[c.qname] = c + continue + if isinstance(c, Element): + elements[c.qname] = c + continue + if isinstance(c, Group): + groups[c.qname] = c + continue + if isinstance(c, AttributeGroup): + agrps[c.qname] = c + continue + types[c.qname] = c + for i in imports: + children.remove(i) + return (children, imports, attributes, elements, types, groups, agrps) + + + + +####################################################### +# Static Import Bindings :-( +####################################################### +Import.bind( + 'http://schemas.xmlsoap.org/soap/encoding/', + 'suds://schemas.xmlsoap.org/soap/encoding/') +Import.bind( + 'http://www.w3.org/XML/1998/namespace', + 'http://www.w3.org/2001/xml.xsd') +Import.bind( + 'http://www.w3.org/2001/XMLSchema', + 'http://www.w3.org/2001/XMLSchema.xsd') diff --git a/libs/suds/xsd/sxbuiltin.py b/libs/suds/xsd/sxbuiltin.py new file mode 100644 index 00000000..f8cf428b --- /dev/null +++ b/libs/suds/xsd/sxbuiltin.py @@ -0,0 +1,274 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the (LGPL) GNU Lesser General Public License as +# published by the Free Software Foundation; either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Lesser General Public License for more details at +# ( http://www.gnu.org/licenses/lgpl.html ). +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# written by: Jeff Ortel ( jortel@redhat.com ) + +""" +The I{sxbuiltin} module provides classes that represent +XSD I{builtin} schema objects. +""" + +from logging import getLogger +from suds import * +from suds.xsd import * +from suds.sax.date import * +from suds.xsd.sxbase import XBuiltin +import datetime as dt + + +log = getLogger(__name__) + + +class XString(XBuiltin): + """ + Represents an (xsd) node + """ + pass + + +class XAny(XBuiltin): + """ + Represents an (xsd) node + """ + + def __init__(self, schema, name): + XBuiltin.__init__(self, schema, name) + self.nillable = False + + def get_child(self, name): + child = XAny(self.schema, name) + return (child, []) + + def any(self): + return True + + +class XBoolean(XBuiltin): + """ + Represents an (xsd) boolean builtin type. + """ + + translation = ( + { '1':True,'true':True,'0':False,'false':False }, + { True:'true',1:'true',False:'false',0:'false' }, + ) + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring): + return XBoolean.translation[0].get(value) + else: + return None + else: + if isinstance(value, (bool,int)): + return XBoolean.translation[1].get(value) + else: + return value + + +class XInteger(XBuiltin): + """ + Represents an (xsd) xs:int builtin type. + """ + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring) and len(value): + return int(value) + else: + return None + else: + if isinstance(value, int): + return str(value) + else: + return value + +class XLong(XBuiltin): + """ + Represents an (xsd) xs:long builtin type. + """ + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring) and len(value): + return long(value) + else: + return None + else: + if isinstance(value, (int,long)): + return str(value) + else: + return value + + +class XFloat(XBuiltin): + """ + Represents an (xsd) xs:float builtin type. + """ + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring) and len(value): + return float(value) + else: + return None + else: + if isinstance(value, float): + return str(value) + else: + return value + + +class XDate(XBuiltin): + """ + Represents an (xsd) xs:date builtin type. + """ + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring) and len(value): + return Date(value).date + else: + return None + else: + if isinstance(value, dt.date): + return str(Date(value)) + else: + return value + + +class XTime(XBuiltin): + """ + Represents an (xsd) xs:time builtin type. + """ + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring) and len(value): + return Time(value).time + else: + return None + else: + if isinstance(value, dt.date): + return str(Time(value)) + else: + return value + + +class XDateTime(XBuiltin): + """ + Represents an (xsd) xs:datetime builtin type. + """ + + def translate(self, value, topython=True): + if topython: + if isinstance(value, basestring) and len(value): + return DateTime(value).datetime + else: + return None + else: + if isinstance(value, dt.date): + return str(DateTime(value)) + else: + return value + + +class Factory: + + tags =\ + { + # any + 'anyType' : XAny, + # strings + 'string' : XString, + 'normalizedString' : XString, + 'ID' : XString, + 'Name' : XString, + 'QName' : XString, + 'NCName' : XString, + 'anySimpleType' : XString, + 'anyURI' : XString, + 'NOTATION' : XString, + 'token' : XString, + 'language' : XString, + 'IDREFS' : XString, + 'ENTITIES' : XString, + 'IDREF' : XString, + 'ENTITY' : XString, + 'NMTOKEN' : XString, + 'NMTOKENS' : XString, + # binary + 'hexBinary' : XString, + 'base64Binary' : XString, + # integers + 'int' : XInteger, + 'integer' : XInteger, + 'unsignedInt' : XInteger, + 'positiveInteger' : XInteger, + 'negativeInteger' : XInteger, + 'nonPositiveInteger' : XInteger, + 'nonNegativeInteger' : XInteger, + # longs + 'long' : XLong, + 'unsignedLong' : XLong, + # shorts + 'short' : XInteger, + 'unsignedShort' : XInteger, + 'byte' : XInteger, + 'unsignedByte' : XInteger, + # floats + 'float' : XFloat, + 'double' : XFloat, + 'decimal' : XFloat, + # dates & times + 'date' : XDate, + 'time' : XTime, + 'dateTime': XDateTime, + 'duration': XString, + 'gYearMonth' : XString, + 'gYear' : XString, + 'gMonthDay' : XString, + 'gDay' : XString, + 'gMonth' : XString, + # boolean + 'boolean' : XBoolean, + } + + @classmethod + def maptag(cls, tag, fn): + """ + Map (override) tag => I{class} mapping. + @param tag: An xsd tag name. + @type tag: str + @param fn: A function or class. + @type fn: fn|class. + """ + cls.tags[tag] = fn + + @classmethod + def create(cls, schema, name): + """ + Create an object based on the root tag name. + @param schema: A schema object. + @type schema: L{schema.Schema} + @param name: The name. + @type name: str + @return: The created object. + @rtype: L{XBuiltin} + """ + fn = cls.tags.get(name) + if fn is not None: + return fn(schema, name) + else: + return XBuiltin(schema, name)