dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New metadata scanner

Browse Source
This commit is contained in:
Ruud
2012-01-27 18:54:12 +01:00
parent fe7ac58bf5
commit 228f7e9e9e
192 changed files with 193 additions and 44918 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/plugins/scanner/main.py
+23 -17
View File
@@ -5,15 +5,17 @@ from couchpotato.core.helpers.variable import getExt, getImdb, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import File
from couchpotato.environment import Env
from flask.helpers import json
from enzyme.exceptions import NoParserError
import enzyme
import logging
import os
import re
import subprocess
import sys
import time
import traceback
enzyme_logger = logging.getLogger('enzyme')
enzyme_logger.setLevel(logging.INFO)
log = CPLog(__name__)
@@ -307,11 +309,12 @@ class Scanner(Plugin):
meta = self.getMeta(cur_file)
try:
data['video'] = self.getCodec(cur_file, self.codecs['video'])
data['audio'] = meta['audio stream'][0]['compression']
data['resolution_width'] = meta['video stream'][0]['image width']
data['resolution_height'] = meta['video stream'][0]['image height']
data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video']))
data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio']))
data['resolution_width'] = meta.get('resolution_width', 720)
data['resolution_height'] = meta.get('resolution_height', 480)
except:
log.debug('Error parsing metadata: %s %s' % (cur_file, traceback.format_exc()))
pass
if data.get('audio'): break
@@ -329,17 +332,20 @@ class Scanner(Plugin):
return data
def getMeta(self, filename):
lib_dir = os.path.join(Env.get('app_dir'), 'libs')
script = os.path.join(lib_dir, 'getmeta.py')
p = subprocess.Popen([sys.executable, script, filename], stdout = subprocess.PIPE, stderr = subprocess.PIPE, cwd = lib_dir)
z = p.communicate()[0]
try:
meta = json.loads(z)
return meta
except Exception:
log.error('Couldn\'t get metadata from file: %s' % traceback.format_exc())
p = enzyme.parse(filename)
return {
'video': p.video[0].codec,
'audio': p.audio[0].codec,
'resolution_width': p.video[0].width,
'resolution_height': p.video[0].height,
}
except NoParserError:
log.debug('No parser found for %s' % filename)
return {}
def determineMovie(self, group):
imdb_id = None
libs/enzyme/core.py
+6 -6
View File
@@ -64,7 +64,7 @@ class Media(object):
_keys = MEDIACORE
table_mapping = {}
def __init__(self, hash=None):
def __init__(self, hash = None):
if hash is not None:
# create Media based on dict
for key, value in hash.items():
@@ -253,7 +253,7 @@ class Media(object):
"""
return hasattr(self, key)
def get(self, attr, default=None):
def get(self, attr, default = None):
"""
Returns the given attribute. If the attribute is not set by
the parser return 'default'.
@@ -315,7 +315,7 @@ class Tag(object):
Tag values are strings (for binary data), unicode objects, or datetime
objects for tags that represent dates or times.
"""
def __init__(self, value=None, langcode='und', binary=False):
def __init__(self, value = None, langcode = 'und', binary = False):
super(Tag, self).__init__()
self.value = value
self.langcode = langcode
@@ -363,7 +363,7 @@ class Tags(dict, Tag):
The attribute RATING has a value (PG), but it also has a child tag
COUNTRY that specifies the country code the rating belongs to.
"""
def __init__(self, value=None, langcode='und', binary=False):
def __init__(self, value = None, langcode = 'und', binary = False):
super(Tags, self).__init__()
self.value = value
self.langcode = langcode
@@ -410,7 +410,7 @@ class Chapter(Media):
"""
_keys = ['enabled', 'name', 'pos', 'id']
def __init__(self, name=None, pos=0):
def __init__(self, name = None, pos = 0):
Media.__init__(self)
self.name = name
self.pos = pos
@@ -424,7 +424,7 @@ class Subtitle(Media):
_keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title',
'id', 'codec']
def __init__(self, language=None):
def __init__(self, language = None):
Media.__init__(self)
self.language = language
libs/getmeta.py
-205
View File
@@ -1,205 +0,0 @@
from flask.helpers import json
from hachoir_core.cmd_line import unicodeFilename
from hachoir_metadata import extractMetadata
from hachoir_parser import createParser
import datetime
import re
import sys
def getMetadata(filename):
filename, realname = unicodeFilename(filename), filename
parser = createParser(filename, realname)
try:
metadata = extractMetadata(parser)
except:
return None
if metadata is not None:
metadata = metadata.exportPlaintext()
return metadata
return None
def parseMetadata(meta, jsonsafe=True):
'''
Return a dict of section headings like 'Video stream' or 'Audio stream'. Each key will have a list of dicts.
This supports multiple video/audio/subtitle/whatever streams per stream type. Each element in the list of streams
will he a dict with keys like 'Image height' and 'Compression'...anything that hachoir is able to extract.
An example output:
{'Audio stream': [{u'Channel': u'6',
u'Compression': u'A_AC3',
u'Sample rate': u'48.0 kHz'}],
u'Common': [{u'Creation date': u'2008-03-20 09:09:43',
u'Duration': u'1 hour 40 min 6 sec',
u'Endianness': u'Big endian',
u'MIME type': u'video/x-matroska',
u'Producer': u'libebml v0.7.7 + libmatroska v0.8.1'}],
'Video stream': [{u'Compression': u'V_MPEG4/ISO/AVC',
u'Image height': u'688 pixels',
u'Image width': u'1280 pixels',
u'Language': u'English'}]}
'''
if not meta:
return
sections = {}
what = []
for line in meta:
#if line doesn't start with "- " it is a section heading
if line[:2] != "- ":
section = line.strip(":").lower()
#lets collapse multiple stream headings into one...
search = re.search(r'#\d+\Z', section)
if search:
section = re.sub(search.group(), '', section).strip()
if section not in sections:
sections[section] = [dict()]
else:
sections[section].append(dict())
else:
#This isn't a section heading, so we put it in the last section heading we found.
#meta always starts out with a section heading so 'section' will always be defined
i = line.find(":")
key = line[2:i].lower()
value = _parseValue(section, key, line[i+2:])
if value is None:
value = line[i+2:]
if jsonsafe:
try:
v = json.dumps(value)
except TypeError:
value = str(value)
sections[section][-1][key] = value
return sections
def _parseValue(section, key, value, jsonsafe = True):
'''
Tediously check all the types that we know about (checked over 7k videos to find these)
and convert them to python native types.
If jsonsafe is True, we'll make json-unfriendly types like datetime into json-friendly.
'''
date_search = re.search("\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d", value)
if key == 'bit rate':
ret = _parseBitRate(value.lower())
elif key == 'bits/sample' or key == 'bits/pixel':
try:
bits = int(value.split()[0])
ret = bits
except:
ret = None
elif key == 'channel':
if value == 'stereo':
ret = 2
elif value == 'mono':
ret = 1
else:
try:
channels = int(value)
ret = channels
except:
ret = None
elif key == 'compression':
ret = _parseCompression(value)
elif key == 'compression rate':
try:
ret = float(value.split('x')[0])
except:
ret = None
elif key == 'duration':
try:
ret = _parseDuration(value)
except:
ret = None
elif key == 'sample rate':
try:
ret = float(value.split()[0]) * 1000
except:
ret = None
elif key == 'frame rate':
try:
ret = float(value.split()[0])
except:
pass
elif key == 'image height' or key == 'image width':
pixels = re.match("(?P<pixels>\d{1,4}) pixel", value)
if pixels:
ret = int(pixels.group('pixels'))
else:
ret = None
elif date_search:
try:
ret = datetime.datetime.strptime(date_search.group(), "%Y-%m-%d %H:%M:%S")
except:
ret = None
else:
#If it's something we don't know about...
ret = None
return ret
def _parseDuration(value):
t = re.search(r"((?P<hour>\d+) hour(s|))? ?((?P<min>\d+) min)? ?((?P<sec>\d+) sec)? ?((?P<ms>\d+) ms)?", value)
if t:
hour = 0 if not t.group('hour') else int(t.group('hour'))
min = 0 if not t.group('min') else int(t.group('min'))
sec = 0 if not t.group('sec') else int(t.group('sec'))
ms = 0 if not t.group('ms') else int(t.group('ms'))
return datetime.timedelta(hours = hour, minutes = min, seconds = sec, milliseconds = ms)
def _parseCompression(value):
codecs = {
'v_mpeg4/iso/avc': 'AVC',
'x264': 'AVC',
'divx': 'divx',
'xvid': 'xvid',
'v_ms/vfw/fourcc': 'vfw',
'vorbis': 'vorbis',
'xvid': 'xvid',
'mpeg layer 3': 'mp3',
'a_dts': 'DTS',
'a_aac': 'AAC',
'a_truehd': 'TRUEHD',
'microsoft mpeg': 'MPEG',
'ac3': 'AC3',
'wvc1': 'WVC1',
'pulse code modulation': 'PCM',
'pcm': 'PCM',
'windows media audio': 'WMA',
'windows media video': 'WMV',
's_text/ascii': 'ASCII',
's_text/utf8': 'UTF8',
's_text/ssa': 'SSA',
's_text/ass': 'ASS'
}
for codec in codecs:
if codec in value.lower():
return codecs[codec]
def _parseBitRate(value):
try:
bitrate = float(value.split()[0])
except:
return None
if 'kbit' in value.lower():
multi = 1000
elif 'mbit' in value.lower():
multi = 1000 * 1000
else:
return None
return bitrate * multi
print json.dumps(parseMetadata(getMetadata(sys.argv[1])))
libs/guessit/__init__.py
+4 -1
View File
@@ -18,7 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.2'
__version__ = '0.3-dev'
__all__ = [ 'Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info' ]
@@ -52,6 +52,9 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
result = []
hashers = []
if isinstance(info, basestring):
info = [ info ]
for infotype in info:
if infotype == 'filename':
m = IterativeMatcher(filename, filetype = filetype)
libs/guessit/filetype.py
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# GuessIt is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Lesser GNU General Public License for more details.
#
# You should have received a copy of the Lesser GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from guessit.patterns import subtitle_exts, video_exts, episode_rexps, find_properties, canonical_form
import os.path
import re
import logging
log = logging.getLogger("guessit.filetype")
def guess_filetype(filename, filetype = 'autodetect'):
other = {}
# look at the extension first
fileext = os.path.splitext(filename)[1][1:].lower()
if fileext in subtitle_exts:
if 'movie' in filetype:
filetype = 'moviesubtitle'
elif 'episode' in filetype:
filetype = 'episodesubtitle'
else:
filetype = 'subtitle'
other = { 'container': fileext }
elif fileext in video_exts:
if filetype == 'autodetect':
filetype = 'video'
other = { 'container': fileext }
else:
if filetype == 'autodetect':
filetype = 'unknown'
other = { 'extension': fileext }
# now look whether there are some specific hints for episode vs movie
if filetype in ('video', 'subtitle'):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, filename, re.IGNORECASE)
if match:
if filetype == 'video':
filetype = 'episode'
elif filetype == 'subtitle':
filetype = 'episodesubtitle'
break
for prop, value, start, end in find_properties(filename):
if canonical_form(value) == 'DVB':
if filetype == 'video':
filetype = 'episode'
elif filetype == 'subtitle':
filetype = 'episodesubtitle'
break
# if no episode info found, assume it's a movie
if filetype == 'video':
filetype = 'movie'
elif filetype == 'subtitle':
filetype = 'moviesubtitle'
return filetype, other
libs/guessit/fileutils.py
+16 -1
View File
@@ -18,7 +18,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import ntpath
import os.path
import zipfile
def split_path(path):
@@ -44,7 +46,7 @@ def split_path(path):
"""
result = []
while True:
head, tail = os.path.split(path)
head, tail = ntpath.split(path)
# on Unix systems, the root folder is '/'
if head == '/' and tail == '':
@@ -81,3 +83,16 @@ def file_in_same_dir(ref_file, desired_file):
"""
return os.path.join(*(split_path(ref_file)[:-1] + [ desired_file ]))
def load_file_in_same_dir(ref_file, filename):
"""Load a given file. Works even when the file is contained inside a zip."""
path = split_path(ref_file)[:-1] + [ filename ]
for i, p in enumerate(path):
if p.endswith('.zip'):
zfilename = os.path.join(*path[:i+1])
zfile = zipfile.ZipFile(zfilename)
return zfile.read('/'.join(path[i+1:]))
return open(os.path.join(*path)).read()
libs/guessit/language.py
+2 -1
View File
@@ -33,7 +33,8 @@ log = logging.getLogger('guessit.language')
# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
# an alpha-2 code (when given), an English name, and a French name of a language
# are all separated by pipe (|) characters."
language_matrix = [ l.strip().decode('utf-8').split('|') for l in open(fileutils.file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt')) ]
language_matrix = [ l.strip().decode('utf-8').split('|')
for l in fileutils.load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt').split('\n') ]
lng3 = frozenset(filter(bool, (l[0] for l in language_matrix)))
lng3term = frozenset(filter(bool, (l[1] for l in language_matrix)))
libs/guessit/matcher.py
+32 -55
View File
@@ -3,6 +3,7 @@
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2011 Ricard Marxer <ricardmp@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -22,7 +23,8 @@ from guessit import fileutils, textutils
from guessit.guess import Guess, merge_similar_guesses, merge_all, choose_int, choose_string
from guessit.date import search_date, search_year
from guessit.language import search_language
from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, properties, canonical_form
from guessit.filetype import guess_filetype
from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, find_properties, canonical_form, unlikely_series
from guessit.matchtree import get_group, find_group, leftover_valid_groups, tree_to_string
from guessit.textutils import find_first_level_groups, split_on_groups, blank_region, clean_string, to_utf8
from guessit.fileutils import split_path_components
@@ -31,6 +33,7 @@ import os.path
import re
import copy
import logging
import mimetypes
log = logging.getLogger("guessit.matcher")
@@ -148,22 +151,11 @@ def guess_groups(string, result, filetype):
# common well-defined words and regexps
clow = current.lower()
confidence = 1.0 # for all of them
for prop, values in properties.items():
for value in values:
pos = clow.find(value.lower())
if pos != -1:
end = pos + len(value)
# make sure our word is always surrounded by separators
if clow[pos-1] not in sep or clow[end] not in sep:
# note: sep is a regexp, but in this case using it as
# a sequence achieves the same goal
continue
for prop, value, pos, end in find_properties(current):
guess = guessed({ prop: value }, confidence = confidence)
current = update_found(current, guess, (pos, end))
guess = guessed({ prop: value }, confidence = confidence)
current = update_found(current, guess, (pos, end))
clow = current.lower()
# weak guesses for episode number, only run it if we don't have an estimate already
if filetype in ('episode', 'episodesubtitle'):
@@ -341,9 +333,10 @@ class IterativeMatcher(object):
resolution when they arise.
"""
if filetype not in ('autodetect', 'subtitle', 'movie', 'moviesubtitle',
if filetype not in ('autodetect', 'subtitle', 'video',
'movie', 'moviesubtitle',
'episode', 'episodesubtitle'):
raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')"
raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'video', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')"
if not isinstance(filename, unicode):
log.debug('WARNING: given filename to matcher is not unicode...')
@@ -368,43 +361,21 @@ class IterativeMatcher(object):
# 1- first split our path into dirs + basename + ext
match_tree = split_path_components(filename)
fileext = match_tree.pop(-1)[1:].lower()
if fileext in subtitle_exts:
if 'movie' in filetype:
filetype = 'moviesubtitle'
elif 'episode' in filetype:
filetype = 'episodesubtitle'
else:
filetype = 'subtitle'
extguess = guessed({ 'container': fileext }, confidence = 1.0)
elif fileext in video_exts:
extguess = guessed({ 'container': fileext }, confidence = 1.0)
else:
extguess = guessed({ 'extension': fileext}, confidence = 1.0)
# TODO: depending on the extension, we could already grab some info and maybe specialized
# guessers, eg: a lang parser for idx files, an automatic detection of the language
# for srt files, a video metadata extractor for avi, mkv, ...
# if we are on autodetect, try to do it now so we can tell the
# guess_groups function what type of info it should be looking for
if filetype in ('autodetect', 'subtitle'):
for rexp, confidence, span_adjust in episode_rexps:
match = re.search(rexp, filename, re.IGNORECASE)
if match:
if filetype == 'autodetect':
filetype = 'episode'
elif filetype == 'subtitle':
filetype = 'episodesubtitle'
break
# if no episode info found, assume it's a movie
if filetype == 'autodetect':
filetype = 'movie'
elif filetype == 'subtitle':
filetype = 'moviesubtitle'
# try to detect the file type
filetype, other = guess_filetype(filename, filetype)
guessed({ 'type': filetype }, confidence = 1.0)
extguess = guessed(other, confidence = 1.0)
# guess the mimetype of the filename
# TODO: handle other mimetypes not found on the default type_maps
# mimetypes.types_map['.srt']='text/subtitle'
mime, _ = mimetypes.guess_type(filename, strict=False)
if mime is not None:
guessed({ 'mimetype': mime }, confidence = 1.0)
# remove the extension from the match tree, as all indices relative
# the the filename groups assume the basename is the last one
fileext = match_tree.pop(-1)[1:].lower()
# 2- split each of those into explicit groups, if any
@@ -453,8 +424,14 @@ class IterativeMatcher(object):
if len(previous) == 1:
guess = guessed({ 'series': previous[0][0] }, confidence = 0.5)
leftover = update_found(leftover, previous[0][1], guess)
# reduce the confidence of unlikely series
for guess in result:
if 'series' in guess:
if guess['series'].lower() in unlikely_series:
guess.set_confidence('series', guess.confidence('series') * 0.5)
elif filetype in ('movie', 'moviesubtitle'):
leftover_all = leftover_valid_groups(match_tree)
libs/guessit/patterns.py
+34 -7
View File
@@ -3,6 +3,7 @@
#
# GuessIt - A library for guessing information from filenames
# Copyright (c) 2011 Nicolas Wack <wackou@gmail.com>
# Copyright (c) 2011 Ricard Marxer <ricardmp@gmail.com>
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -19,9 +20,9 @@
#
subtitle_exts = [ 'srt', 'idx', 'sub' ]
subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa', 'txt' ]
video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'mov', 'ogg', 'ogm', 'ogv', 'wmv' ]
video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv', 'wmv', 'divx' ]
# separator character regexp
sep = r'[][)(}{+ \._-]' # regexp art, hehe :D
@@ -34,6 +35,9 @@ episode_rexps = [ # ... Season 2 ...
(r'season (?P<season>[0-9]+)', 1.0, (0, 0)),
(r'saison (?P<season>[0-9]+)', 1.0, (0, 0)),
# ... s02-x01 ...
(r's(?P<season>[0-9]{1,2})-x(?P<bonusNumber>[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
# ... s02e13 ...
(r'[Ss](?P<season>[0-9]{1,2}).{,3}[EeXx](?P<episodeNumber>[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
@@ -41,7 +45,7 @@ episode_rexps = [ # ... Season 2 ...
(r'[^0-9](?P<season>[0-9]{1,2})[x\.](?P<episodeNumber>[0-9]{2})[^0-9]', 0.8, (1, -1)),
# ... s02 ...
(sep + r's(?P<season>[0-9]{1,2})' + sep + '?', 0.6, (0, 0)),
(sep + r's(?P<season>[0-9]{1,2})' + sep + '?', 0.6, (1, -1)),
# v2 or v3 for some mangas which have multiples rips
(sep + r'(?P<episodeNumber>[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
@@ -77,11 +81,11 @@ video_rexps = [ # cd number
websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com', 'sharethefiles.com' ]
properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'Blu-ray', 'BDRip', 'BRRip',
'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'WEBRip', 'DVDSCR', 'Screener', 'VHS',
'VIDEO_TS' ],
unlikely_series = ['series']
'container': [ 'avi', 'mkv', 'ogv', 'ogm', 'wmv', 'mp4', 'mov' ],
properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'Blu-ray', 'BDRip', 'BRRip',
'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'DVBRip', 'PDTV', 'WEBRip',
'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS' ],
'screenSize': [ '720p', '720' ],
@@ -106,10 +110,29 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
],
}
def find_properties(filename):
result = []
clow = filename.lower()
for prop, values in properties.items():
for value in values:
pos = clow.find(value.lower())
if pos != -1:
end = pos + len(value)
# make sure our word is always surrounded by separators
if ((pos > 0 and clow[pos-1] not in sep) or
(end < len(clow) and clow[end] not in sep)):
# note: sep is a regexp, but in this case using it as
# a sequence achieves the same goal
continue
result.append((prop, value, pos, end))
return result
property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
'HD-DVD': [ 'HDDVD', 'HDDVDRip' ],
'BluRay': [ 'BDRip', 'BRRip', 'Blu-ray' ],
'DVB': [ 'DVBRip', 'PDTV' ],
'Screener': [ 'DVDSCR' ],
'DivX': [ 'DVDivX' ],
'h264': [ 'x264' ],
@@ -123,6 +146,10 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
reverse_synonyms = {}
for prop, values in properties.items():
for value in values:
reverse_synonyms[value.lower()] = value
for canonical, synonyms in property_synonyms.items():
for synonym in synonyms:
reverse_synonyms[synonym.lower()] = canonical
libs/hachoir_core/__init__.py
-2
View File
@@ -1,2 +0,0 @@
from hachoir_core.version import VERSION as __version__, PACKAGE, WEBSITE, LICENSE
libs/hachoir_core/benchmark.py
-210
View File
@@ -1,210 +0,0 @@
from hachoir_core.tools import humanDurationNanosec
from hachoir_core.i18n import _
from math import floor
from time import time
class BenchmarkError(Exception):
"""
Error during benchmark, use str(err) to format it as string.
"""
def __init__(self, message):
Exception.__init__(self,
"Benchmark internal error: %s" % message)
class BenchmarkStat:
"""
Benchmark statistics. This class automatically computes minimum value,
maximum value and sum of all values.
Methods:
- append(value): append a value
- getMin(): minimum value
- getMax(): maximum value
- getSum(): sum of all values
- __len__(): get number of elements
- __nonzero__(): isn't empty?
"""
def __init__(self):
self._values = []
def append(self, value):
self._values.append(value)
try:
self._min = min(self._min, value)
self._max = max(self._max, value)
self._sum += value
except AttributeError:
self._min = value
self._max = value
self._sum = value
def __len__(self):
return len(self._values)
def __nonzero__(self):
return bool(self._values)
def getMin(self):
return self._min
def getMax(self):
return self._max
def getSum(self):
return self._sum
class Benchmark:
def __init__(self, max_time=5.0,
min_count=5, max_count=None, progress_time=1.0):
"""
Constructor:
- max_time: Maximum wanted duration of the whole benchmark
(default: 5 seconds, minimum: 1 second).
- min_count: Minimum number of function calls to get good statistics
(defaut: 5, minimum: 1).
- progress_time: Time between each "progress" message
(default: 1 second, minimum: 250 ms).
- max_count: Maximum number of function calls (default: no limit).
- verbose: Is verbose? (default: False)
- disable_gc: Disable garbage collector? (default: False)
"""
self.max_time = max(max_time, 1.0)
self.min_count = max(min_count, 1)
self.max_count = max_count
self.progress_time = max(progress_time, 0.25)
self.verbose = False
self.disable_gc = False
def formatTime(self, value):
"""
Format a time delta to string: use humanDurationNanosec()
"""
return humanDurationNanosec(value * 1000000000)
def displayStat(self, stat):
"""
Display statistics to stdout:
- best time (minimum)
- average time (arithmetic average)
- worst time (maximum)
- total time (sum)
Use arithmetic avertage instead of geometric average because
geometric fails if any value is zero (returns zero) and also
because floating point multiplication lose precision with many
values.
"""
average = stat.getSum() / len(stat)
values = (stat.getMin(), average, stat.getMax(), stat.getSum())
values = tuple(self.formatTime(value) for value in values)
print _("Benchmark: best=%s average=%s worst=%s total=%s") \
% values
def _runOnce(self, func, args, kw):
before = time()
func(*args, **kw)
after = time()
return after - before
def _run(self, func, args, kw):
"""
Call func(*args, **kw) as many times as needed to get
good statistics. Algorithm:
- call the function once
- compute needed number of calls
- and then call function N times
To compute number of calls, parameters are:
- time of first function call
- minimum number of calls (min_count attribute)
- maximum test time (max_time attribute)
Notice: The function will approximate number of calls.
"""
# First call of the benchmark
stat = BenchmarkStat()
diff = self._runOnce(func, args, kw)
best = diff
stat.append(diff)
total_time = diff
# Compute needed number of calls
count = int(floor(self.max_time / diff))
count = max(count, self.min_count)
if self.max_count:
count = min(count, self.max_count)
# Not other call? Just exit
if count == 1:
return stat
estimate = diff * count
if self.verbose:
print _("Run benchmark: %s calls (estimate: %s)") \
% (count, self.formatTime(estimate))
display_progress = self.verbose and (1.0 <= estimate)
total_count = 1
while total_count < count:
# Run benchmark and display each result
if display_progress:
print _("Result %s/%s: %s (best: %s)") % \
(total_count, count,
self.formatTime(diff), self.formatTime(best))
part = count - total_count
# Will takes more than one second?
average = total_time / total_count
if self.progress_time < part * average:
part = max( int(self.progress_time / average), 1)
for index in xrange(part):
diff = self._runOnce(func, args, kw)
stat.append(diff)
total_time += diff
best = min(diff, best)
total_count += part
if display_progress:
print _("Result %s/%s: %s (best: %s)") % \
(count, count,
self.formatTime(diff), self.formatTime(best))
return stat
def validateStat(self, stat):
"""
Check statistics and raise a BenchmarkError if they are invalid.
Example of tests: reject empty stat, reject stat with only nul values.
"""
if not stat:
raise BenchmarkError("empty statistics")
if not stat.getSum():
raise BenchmarkError("nul statistics")
def run(self, func, *args, **kw):
"""
Run function func(*args, **kw), validate statistics,
and display the result on stdout.
Disable garbage collector if asked too.
"""
# Disable garbarge collector is needed and if it does exist
# (Jython 2.2 don't have it for example)
if self.disable_gc:
try:
import gc
except ImportError:
self.disable_gc = False
if self.disable_gc:
gc_enabled = gc.isenabled()
gc.disable()
else:
gc_enabled = False
# Run the benchmark
stat = self._run(func, args, kw)
if gc_enabled:
gc.enable()
# Validate and display stats
self.validateStat(stat)
self.displayStat(stat)
libs/hachoir_core/bits.py
-277
View File
@@ -1,277 +0,0 @@
"""
Utilities to convert integers and binary strings to binary (number), binary
string, number, hexadecimal, etc.
"""
from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
from hachoir_core.compatibility import reversed
from itertools import chain, repeat
from struct import calcsize, unpack, error as struct_error
def swap16(value):
"""
Swap byte between big and little endian of a 16 bits integer.
>>> "%x" % swap16(0x1234)
'3412'
"""
return (value & 0xFF) << 8 | (value >> 8)
def swap32(value):
"""
Swap byte between big and little endian of a 32 bits integer.
>>> "%x" % swap32(0x12345678)
'78563412'
"""
value = long(value)
return ((value & 0x000000FFL) << 24) \
| ((value & 0x0000FF00L) << 8) \
| ((value & 0x00FF0000L) >> 8) \
| ((value & 0xFF000000L) >> 24)
def bin2long(text, endian):
"""
Convert binary number written in a string into an integer.
Skip characters differents than "0" and "1".
>>> bin2long("110", BIG_ENDIAN)
6
>>> bin2long("110", LITTLE_ENDIAN)
3
>>> bin2long("11 00", LITTLE_ENDIAN)
3
"""
assert endian in (LITTLE_ENDIAN, BIG_ENDIAN)
bits = [ (ord(character)-ord("0")) \
for character in text if character in "01" ]
assert len(bits) != 0
if endian is not BIG_ENDIAN:
bits = reversed(bits)
value = 0
for bit in bits:
value *= 2
value += bit
return value
def str2hex(value, prefix="", glue=u"", format="%02X"):
r"""
Convert binary string in hexadecimal (base 16).
>>> str2hex("ABC")
u'414243'
>>> str2hex("\xF0\xAF", glue=" ")
u'F0 AF'
>>> str2hex("ABC", prefix="0x")
u'0x414243'
>>> str2hex("ABC", format=r"\x%02X")
u'\\x41\\x42\\x43'
"""
if isinstance(glue, str):
glue = unicode(glue)
if 0 < len(prefix):
text = [prefix]
else:
text = []
for character in value:
text.append(format % ord(character))
return glue.join(text)
def countBits(value):
"""
Count number of bits needed to store a (positive) integer number.
>>> countBits(0)
1
>>> countBits(1000)
10
>>> countBits(44100)
16
>>> countBits(18446744073709551615)
64
"""
assert 0 <= value
count = 1
bits = 1
while (1 << bits) <= value:
count += bits
value >>= bits
bits <<= 1
while 2 <= value:
if bits != 1:
bits >>= 1
else:
bits -= 1
while (1 << bits) <= value:
count += bits
value >>= bits
return count
def byte2bin(number, classic_mode=True):
"""
Convert a byte (integer in 0..255 range) to a binary string.
If classic_mode is true (default value), reverse bits.
>>> byte2bin(10)
'00001010'
>>> byte2bin(10, False)
'01010000'
"""
text = ""
for i in range(0, 8):
if classic_mode:
mask = 1 << (7-i)
else:
mask = 1 << i
if (number & mask) == mask:
text += "1"
else:
text += "0"
return text
def long2raw(value, endian, size=None):
r"""
Convert a number (positive and not nul) to a raw string.
If size is given, add nul bytes to fill to size bytes.
>>> long2raw(0x1219, BIG_ENDIAN)
'\x12\x19'
>>> long2raw(0x1219, BIG_ENDIAN, 4) # 32 bits
'\x00\x00\x12\x19'
>>> long2raw(0x1219, LITTLE_ENDIAN, 4) # 32 bits
'\x19\x12\x00\x00'
"""
assert (not size and 0 < value) or (0 <= value)
assert endian in (LITTLE_ENDIAN, BIG_ENDIAN)
text = []
while (value != 0 or text == ""):
byte = value % 256
text.append( chr(byte) )
value >>= 8
if size:
need = max(size - len(text), 0)
else:
need = 0
if need:
if endian is BIG_ENDIAN:
text = chain(repeat("\0", need), reversed(text))
else:
text = chain(text, repeat("\0", need))
else:
if endian is BIG_ENDIAN:
text = reversed(text)
return "".join(text)
def long2bin(size, value, endian, classic_mode=False):
"""
Convert a number into bits (in a string):
- size: size in bits of the number
- value: positive (or nul) number
- endian: BIG_ENDIAN (most important bit first)
or LITTLE_ENDIAN (least important bit first)
- classic_mode (default: False): reverse each packet of 8 bits
>>> long2bin(16, 1+4 + (1+8)*256, BIG_ENDIAN)
'10100000 10010000'
>>> long2bin(16, 1+4 + (1+8)*256, BIG_ENDIAN, True)
'00000101 00001001'
>>> long2bin(16, 1+4 + (1+8)*256, LITTLE_ENDIAN)
'00001001 00000101'
>>> long2bin(16, 1+4 + (1+8)*256, LITTLE_ENDIAN, True)
'10010000 10100000'
"""
text = ""
assert endian in (LITTLE_ENDIAN, BIG_ENDIAN)
assert 0 <= value
for index in xrange(size):
if (value & 1) == 1:
text += "1"
else:
text += "0"
value >>= 1
if endian is LITTLE_ENDIAN:
text = text[::-1]
result = ""
while len(text) != 0:
if len(result) != 0:
result += " "
if classic_mode:
result += text[7::-1]
else:
result += text[:8]
text = text[8:]
return result
def str2bin(value, classic_mode=True):
r"""
Convert binary string to binary numbers.
If classic_mode is true (default value), reverse bits.
>>> str2bin("\x03\xFF")
'00000011 11111111'
>>> str2bin("\x03\xFF", False)
'11000000 11111111'
"""
text = ""
for character in value:
if text != "":
text += " "
byte = ord(character)
text += byte2bin(byte, classic_mode)
return text
def _createStructFormat():
"""
Create a dictionnary (endian, size_byte) => struct format used
by str2long() to convert raw data to positive integer.
"""
format = {
BIG_ENDIAN: {},
LITTLE_ENDIAN: {},
}
for struct_format in "BHILQ":
try:
size = calcsize(struct_format)
format[BIG_ENDIAN][size] = '>%s' % struct_format
format[LITTLE_ENDIAN][size] = '<%s' % struct_format
except struct_error:
pass
return format
_struct_format = _createStructFormat()
def str2long(data, endian):
r"""
Convert a raw data (type 'str') into a long integer.
>>> chr(str2long('*', BIG_ENDIAN))
'*'
>>> str2long("\x00\x01\x02\x03", BIG_ENDIAN) == 0x10203
True
>>> str2long("\x2a\x10", LITTLE_ENDIAN) == 0x102a
True
>>> str2long("\xff\x14\x2a\x10", BIG_ENDIAN) == 0xff142a10
True
>>> str2long("\x00\x01\x02\x03", LITTLE_ENDIAN) == 0x3020100
True
>>> str2long("\xff\x14\x2a\x10\xab\x00\xd9\x0e", BIG_ENDIAN) == 0xff142a10ab00d90e
True
>>> str2long("\xff\xff\xff\xff\xff\xff\xff\xff", BIG_ENDIAN) == (2**64-1)
True
"""
assert 1 <= len(data) <= 32 # arbitrary limit: 256 bits
try:
return unpack(_struct_format[endian][len(data)], data)[0]
except KeyError:
pass
assert endian in (BIG_ENDIAN, LITTLE_ENDIAN)
shift = 0
value = 0
if endian is BIG_ENDIAN:
data = reversed(data)
for character in data:
byte = ord(character)
value += (byte << shift)
shift += 8
return value
libs/hachoir_core/cmd_line.py
-43
View File
@@ -1,43 +0,0 @@
from optparse import OptionGroup
from hachoir_core.log import log
from hachoir_core.i18n import _, getTerminalCharset
from hachoir_core.tools import makePrintable
import hachoir_core.config as config
def getHachoirOptions(parser):
"""
Create an option group (type optparse.OptionGroup) of Hachoir
library options.
"""
def setLogFilename(*args):
log.setFilename(args[2])
common = OptionGroup(parser, _("Hachoir library"), \
"Configure Hachoir library")
common.add_option("--verbose", help=_("Verbose mode"),
default=False, action="store_true")
common.add_option("--log", help=_("Write log in a file"),
type="string", action="callback", callback=setLogFilename)
common.add_option("--quiet", help=_("Quiet mode (don't display warning)"),
default=False, action="store_true")
common.add_option("--debug", help=_("Debug mode"),
default=False, action="store_true")
return common
def configureHachoir(option):
# Configure Hachoir using "option" (value from optparse)
if option.quiet:
config.quiet = True
if option.verbose:
config.verbose = True
if option.debug:
config.debug = True
def unicodeFilename(filename, charset=None):
if not charset:
charset = getTerminalCharset()
try:
return unicode(filename, charset)
except UnicodeDecodeError:
return makePrintable(filename, charset, to_unicode=True)
libs/hachoir_core/compatibility.py
-185
View File
@@ -1,185 +0,0 @@
"""
Compatibility constants and functions. This module works on Python 1.5 to 2.5.
This module provides:
- True and False constants ;
- any() and all() function ;
- has_yield and has_slice values ;
- isinstance() with Python 2.3 behaviour ;
- reversed() and sorted() function.
True and False constants
========================
Truth constants: True is yes (one) and False is no (zero).
>>> int(True), int(False) # int value
(1, 0)
>>> int(False | True) # and binary operator
1
>>> int(True & False) # or binary operator
0
>>> int(not(True) == False) # not binary operator
1
Warning: on Python smaller than 2.3, True and False are aliases to
number 1 and 0. So "print True" will displays 1 and not True.
any() function
==============
any() returns True if at least one items is True, or False otherwise.
>>> any([False, True])
True
>>> any([True, True])
True
>>> any([False, False])
False
all() function
==============
all() returns True if all items are True, or False otherwise.
This function is just apply binary and operator (&) on all values.
>>> all([True, True])
True
>>> all([False, True])
False
>>> all([False, False])
False
has_yield boolean
=================
has_yield: boolean which indicatese if the interpreter supports yield keyword.
yield keyworkd is available since Python 2.0.
has_yield boolean
=================
has_slice: boolean which indicates if the interpreter supports slices with step
argument or not. slice with step is available since Python 2.3.
reversed() and sorted() function
================================
reversed() and sorted() function has been introduced in Python 2.4.
It's should returns a generator, but this module it may be a list.
>>> data = list("cab")
>>> list(sorted(data))
['a', 'b', 'c']
>>> list(reversed("abc"))
['c', 'b', 'a']
"""
import copy
import operator
# --- True and False constants from Python 2.0 ---
# --- Warning: for Python < 2.3, they are aliases for 1 and 0 ---
try:
True = True
False = False
except NameError:
True = 1
False = 0
# --- any() from Python 2.5 ---
try:
from __builtin__ import any
except ImportError:
def any(items):
for item in items:
if item:
return True
return False
# ---all() from Python 2.5 ---
try:
from __builtin__ import all
except ImportError:
def all(items):
return reduce(operator.__and__, items)
# --- test if interpreter supports yield keyword ---
try:
eval(compile("""
from __future__ import generators
def gen():
yield 1
yield 2
if list(gen()) != [1, 2]:
raise KeyError("42")
""", "<string>", "exec"))
except (KeyError, SyntaxError):
has_yield = False
else:
has_yield = True
# --- test if interpreter supports slices (with step argument) ---
try:
has_slice = eval('"abc"[::-1] == "cba"')
except (TypeError, SyntaxError):
has_slice = False
# --- isinstance with isinstance Python 2.3 behaviour (arg 2 is a type) ---
try:
if isinstance(1, int):
from __builtin__ import isinstance
except TypeError:
print "Redef isinstance"
def isinstance20(a, typea):
if type(typea) != type(type):
raise TypeError("TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types")
return type(typea) != typea
isinstance = isinstance20
# --- reversed() from Python 2.4 ---
try:
from __builtin__ import reversed
except ImportError:
# if hasYield() == "ok":
# code = """
#def reversed(data):
# for index in xrange(len(data)-1, -1, -1):
# yield data[index];
#reversed"""
# reversed = eval(compile(code, "<string>", "exec"))
if has_slice:
def reversed(data):
if not isinstance(data, list):
data = list(data)
return data[::-1]
else:
def reversed(data):
if not isinstance(data, list):
data = list(data)
reversed_data = []
for index in xrange(len(data)-1, -1, -1):
reversed_data.append(data[index])
return reversed_data
# --- sorted() from Python 2.4 ---
try:
from __builtin__ import sorted
except ImportError:
def sorted(data):
sorted_data = copy.copy(data)
sorted_data.sort()
return sorted
__all__ = ("True", "False",
"any", "all", "has_yield", "has_slice",
"isinstance", "reversed", "sorted")
libs/hachoir_core/config.py
-29
View File
@@ -1,29 +0,0 @@
"""
Configuration of Hachoir
"""
import os
# UI: display options
max_string_length = 40 # Max. length in characters of GenericString.display
max_byte_length = 14 # Max. length in bytes of RawBytes.display
max_bit_length = 256 # Max. length in bits of RawBits.display
unicode_stdout = True # Replace stdout and stderr with Unicode compatible objects
# Disable it for readline or ipython
# Global options
debug = False # Display many informations usefull to debug
verbose = False # Display more informations
quiet = False # Don't display warnings
# Use internationalization and localization (gettext)?
if os.name == "nt":
# TODO: Remove this hack and make i18n works on Windows :-)
use_i18n = False
else:
use_i18n = True
# Parser global options
autofix = True # Enable Autofix? see hachoir_core.field.GenericFieldSet
check_padding_pattern = True # Check padding fields pattern?
libs/hachoir_core/dict.py
-183
View File
@@ -1,183 +0,0 @@
"""
Dictionnary classes which store values order.
"""
from hachoir_core.error import HachoirError
from hachoir_core.i18n import _
class UniqKeyError(HachoirError):
"""
Error raised when a value is set whereas the key already exist in a
dictionnary.
"""
pass
class Dict(object):
"""
This class works like classic Python dict() but has an important method:
__iter__() which allow to iterate into the dictionnary _values_ (and not
keys like Python's dict does).
"""
def __init__(self, values=None):
self._index = {} # key => index
self._key_list = [] # index => key
self._value_list = [] # index => value
if values:
for key, value in values:
self.append(key,value)
def _getValues(self):
return self._value_list
values = property(_getValues)
def index(self, key):
"""
Search a value by its key and returns its index
Returns None if the key doesn't exist.
>>> d=Dict( (("two", "deux"), ("one", "un")) )
>>> d.index("two")
0
>>> d.index("one")
1
>>> d.index("three") is None
True
"""
return self._index.get(key)
def __getitem__(self, key):
"""
Get item with specified key.
To get a value by it's index, use mydict.values[index]
>>> d=Dict( (("two", "deux"), ("one", "un")) )
>>> d["one"]
'un'
"""
return self._value_list[self._index[key]]
def __setitem__(self, key, value):
self._value_list[self._index[key]] = value
def append(self, key, value):
"""
Append new value
"""
if key in self._index:
raise UniqKeyError(_("Key '%s' already exists") % key)
self._index[key] = len(self._value_list)
self._key_list.append(key)
self._value_list.append(value)
def __len__(self):
return len(self._value_list)
def __contains__(self, key):
return key in self._index
def __iter__(self):
return iter(self._value_list)
def iteritems(self):
"""
Create a generator to iterate on: (key, value).
>>> d=Dict( (("two", "deux"), ("one", "un")) )
>>> for key, value in d.iteritems():
... print "%r: %r" % (key, value)
...
'two': 'deux'
'one': 'un'
"""
for index in xrange(len(self)):
yield (self._key_list[index], self._value_list[index])
def itervalues(self):
"""
Create an iterator on values
"""
return iter(self._value_list)
def iterkeys(self):
"""
Create an iterator on keys
"""
return iter(self._key_list)
def replace(self, oldkey, newkey, new_value):
"""
Replace an existing value with another one
>>> d=Dict( (("two", "deux"), ("one", "un")) )
>>> d.replace("one", "three", 3)
>>> d
{'two': 'deux', 'three': 3}
You can also use the classic form:
>>> d['three'] = 4
>>> d
{'two': 'deux', 'three': 4}
"""
index = self._index[oldkey]
self._value_list[index] = new_value
if oldkey != newkey:
del self._index[oldkey]
self._index[newkey] = index
self._key_list[index] = newkey
def __delitem__(self, index):
"""
Delete item at position index. May raise IndexError.
>>> d=Dict( ((6, 'six'), (9, 'neuf'), (4, 'quatre')) )
>>> del d[1]
>>> d
{6: 'six', 4: 'quatre'}
"""
if index < 0:
index += len(self._value_list)
if not (0 <= index < len(self._value_list)):
raise IndexError(_("list assignment index out of range (%s/%s)")
% (index, len(self._value_list)))
del self._value_list[index]
del self._key_list[index]
# First loop which may alter self._index
for key, item_index in self._index.iteritems():
if item_index == index:
del self._index[key]
break
# Second loop update indexes
for key, item_index in self._index.iteritems():
if index < item_index:
self._index[key] -= 1
def insert(self, index, key, value):
"""
Insert an item at specified position index.
>>> d=Dict( ((6, 'six'), (9, 'neuf'), (4, 'quatre')) )
>>> d.insert(1, '40', 'quarante')
>>> d
{6: 'six', '40': 'quarante', 9: 'neuf', 4: 'quatre'}
"""
if key in self:
raise UniqKeyError(_("Insert error: key '%s' ready exists") % key)
_index = index
if index < 0:
index += len(self._value_list)
if not(0 <= index <= len(self._value_list)):
raise IndexError(_("Insert error: index '%s' is invalid") % _index)
for item_key, item_index in self._index.iteritems():
if item_index >= index:
self._index[item_key] += 1
self._index[key] = index
self._key_list.insert(index, key)
self._value_list.insert(index, value)
def __repr__(self):
items = ( "%r: %r" % (key, value) for key, value in self.iteritems() )
return "{%s}" % ", ".join(items)
libs/hachoir_core/endian.py
-15
View File
@@ -1,15 +0,0 @@
"""
Constant values about endian.
"""
from hachoir_core.i18n import _
BIG_ENDIAN = "ABCD"
LITTLE_ENDIAN = "DCBA"
NETWORK_ENDIAN = BIG_ENDIAN
endian_name = {
BIG_ENDIAN: _("Big endian"),
LITTLE_ENDIAN: _("Little endian"),
}
libs/hachoir_core/error.py
-45
View File
@@ -1,45 +0,0 @@
"""
Functions to display an error (error, warning or information) message.
"""
from hachoir_core.log import log
from hachoir_core.tools import makePrintable
import sys, traceback
def getBacktrace(empty="Empty backtrace."):
"""
Try to get backtrace as string.
Returns "Error while trying to get backtrace" on failure.
"""
try:
info = sys.exc_info()
trace = traceback.format_exception(*info)
sys.exc_clear()
if trace[0] != "None\n":
return "".join(trace)
except:
# No i18n here (imagine if i18n function calls error...)
return "Error while trying to get backtrace"
return empty
class HachoirError(Exception):
"""
Parent of all errors in Hachoir library
"""
def __init__(self, message):
message_bytes = makePrintable(message, "ASCII")
Exception.__init__(self, message_bytes)
self.text = message
def __unicode__(self):
return self.text
# Error classes which may be raised by Hachoir core
# FIXME: Add EnvironmentError (IOError or OSError) and AssertionError?
# FIXME: Remove ArithmeticError and RuntimeError?
HACHOIR_ERRORS = (HachoirError, LookupError, NameError, AttributeError,
TypeError, ValueError, ArithmeticError, RuntimeError)
info = log.info
warning = log.warning
error = log.error
libs/hachoir_core/event_handler.py
-26
View File
@@ -1,26 +0,0 @@
class EventHandler(object):
"""
Class to connect events to event handlers.
"""
def __init__(self):
self.handlers = {}
def connect(self, event_name, handler):
"""
Connect an event handler to an event. Append it to handlers list.
"""
try:
self.handlers[event_name].append(handler)
except KeyError:
self.handlers[event_name] = [handler]
def raiseEvent(self, event_name, *args):
"""
Raiser an event: call each handler for this event_name.
"""
if event_name not in self.handlers:
return
for handler in self.handlers[event_name]:
handler(*args)
libs/hachoir_core/field/__init__.py
-59
View File
@@ -1,59 +0,0 @@
# Field classes
from hachoir_core.field.field import Field, FieldError, MissingField, joinPath
from hachoir_core.field.bit_field import Bit, Bits, RawBits
from hachoir_core.field.byte_field import Bytes, RawBytes
from hachoir_core.field.sub_file import SubFile, CompressedField
from hachoir_core.field.character import Character
from hachoir_core.field.integer import (
Int8, Int16, Int24, Int32, Int64,
UInt8, UInt16, UInt24, UInt32, UInt64,
GenericInteger)
from hachoir_core.field.enum import Enum
from hachoir_core.field.string_field import (GenericString,
String, CString, UnixLine,
PascalString8, PascalString16, PascalString32)
from hachoir_core.field.padding import (PaddingBits, PaddingBytes,
NullBits, NullBytes)
# Functions
from hachoir_core.field.helper import (isString, isInteger,
createPaddingField, createNullField, createRawField,
writeIntoFile, createOrphanField)
# FieldSet classes
from hachoir_core.field.fake_array import FakeArray
from hachoir_core.field.basic_field_set import (BasicFieldSet,
ParserError, MatchError)
from hachoir_core.field.generic_field_set import GenericFieldSet
from hachoir_core.field.seekable_field_set import SeekableFieldSet, RootSeekableFieldSet
from hachoir_core.field.field_set import FieldSet
from hachoir_core.field.static_field_set import StaticFieldSet
from hachoir_core.field.parser import Parser
from hachoir_core.field.vector import GenericVector, UserVector
# Complex types
from hachoir_core.field.float import Float32, Float64, Float80
from hachoir_core.field.timestamp import (GenericTimestamp,
TimestampUnix32, TimestampUnix64, TimestampMac32, TimestampUUID60, TimestampWin64,
DateTimeMSDOS32, TimeDateMSDOS32, TimedeltaWin64)
# Special Field classes
from hachoir_core.field.link import Link, Fragment
available_types = (
Bit, Bits, RawBits,
Bytes, RawBytes,
SubFile,
Character,
Int8, Int16, Int24, Int32, Int64,
UInt8, UInt16, UInt24, UInt32, UInt64,
String, CString, UnixLine,
PascalString8, PascalString16, PascalString32,
Float32, Float64,
PaddingBits, PaddingBytes,
NullBits, NullBytes,
TimestampUnix32, TimestampMac32, TimestampWin64,
DateTimeMSDOS32, TimeDateMSDOS32,
# GenericInteger, GenericString,
)
libs/hachoir_core/field/basic_field_set.py
-147
View File
@@ -1,147 +0,0 @@
from hachoir_core.field import Field, FieldError
from hachoir_core.stream import InputStream
from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
from hachoir_core.event_handler import EventHandler
class ParserError(FieldError):
"""
Error raised by a field set.
@see: L{FieldError}
"""
pass
class MatchError(FieldError):
"""
Error raised by a field set when the stream content doesn't
match to file format.
@see: L{FieldError}
"""
pass
class BasicFieldSet(Field):
_event_handler = None
is_field_set = True
endian = None
def __init__(self, parent, name, stream, description, size):
# Sanity checks (preconditions)
assert not parent or issubclass(parent.__class__, BasicFieldSet)
assert issubclass(stream.__class__, InputStream)
# Set field set size
if size is None and self.static_size:
assert isinstance(self.static_size, (int, long))
size = self.static_size
# Set Field attributes
self._parent = parent
self._name = name
self._size = size
self._description = description
self.stream = stream
self._field_array_count = {}
# Set endian
if not self.endian:
assert parent and parent.endian
self.endian = parent.endian
if parent:
# This field set is one of the root leafs
self._address = parent.nextFieldAddress()
self.root = parent.root
assert id(self.stream) == id(parent.stream)
else:
# This field set is the root
self._address = 0
self.root = self
self._global_event_handler = None
# Sanity checks (post-conditions)
assert self.endian in (BIG_ENDIAN, LITTLE_ENDIAN)
if (self._size is not None) and (self._size <= 0):
raise ParserError("Invalid parser '%s' size: %s" % (self.path, self._size))
def reset(self):
self._field_array_count = {}
def createValue(self):
return None
def connectEvent(self, event_name, handler, local=True):
assert event_name in (
# Callback prototype: def f(field)
# Called when new value is already set
"field-value-changed",
# Callback prototype: def f(field)
# Called when field size is already set
"field-resized",
# A new field has been inserted in the field set
# Callback prototype: def f(index, new_field)
"field-inserted",
# Callback prototype: def f(old_field, new_field)
# Called when new field is already in field set
"field-replaced",
# Callback prototype: def f(field, new_value)
# Called to ask to set new value
"set-field-value"
), "Event name %r is invalid" % event_name
if local:
if self._event_handler is None:
self._event_handler = EventHandler()
self._event_handler.connect(event_name, handler)
else:
if self.root._global_event_handler is None:
self.root._global_event_handler = EventHandler()
self.root._global_event_handler.connect(event_name, handler)
def raiseEvent(self, event_name, *args):
# Transfer event to local listeners
if self._event_handler is not None:
self._event_handler.raiseEvent(event_name, *args)
# Transfer event to global listeners
if self.root._global_event_handler is not None:
self.root._global_event_handler.raiseEvent(event_name, *args)
def setUniqueFieldName(self, field):
key = field._name[:-2]
try:
self._field_array_count[key] += 1
except KeyError:
self._field_array_count[key] = 0
field._name = key + "[%u]" % self._field_array_count[key]
def readFirstFields(self, number):
"""
Read first number fields if they are not read yet.
Returns number of new added fields.
"""
number = number - self.current_length
if 0 < number:
return self.readMoreFields(number)
else:
return 0
def createFields(self):
raise NotImplementedError()
def __iter__(self):
raise NotImplementedError()
def __len__(self):
raise NotImplementedError()
def getField(self, key, const=True):
raise NotImplementedError()
def nextFieldAddress(self):
raise NotImplementedError()
def getFieldIndex(self, field):
raise NotImplementedError()
def readMoreFields(self, number):
raise NotImplementedError()
libs/hachoir_core/field/bit_field.py
-68
View File
@@ -1,68 +0,0 @@
"""
Bit sized classes:
- Bit: Single bit, value is False or True ;
- Bits: Integer with a size in bits ;
- RawBits: unknown content with a size in bits.
"""
from hachoir_core.field import Field
from hachoir_core.i18n import _
from hachoir_core import config
class RawBits(Field):
"""
Unknown content with a size in bits.
"""
static_size = staticmethod(lambda *args, **kw: args[1])
def __init__(self, parent, name, size, description=None):
"""
Constructor: see L{Field.__init__} for parameter description
"""
Field.__init__(self, parent, name, size, description)
def hasValue(self):
return True
def createValue(self):
return self._parent.stream.readBits(
self.absolute_address, self._size, self._parent.endian)
def createDisplay(self):
if self._size < config.max_bit_length:
return unicode(self.value)
else:
return _("<%s size=%u>" %
(self.__class__.__name__, self._size))
createRawDisplay = createDisplay
class Bits(RawBits):
"""
Positive integer with a size in bits
@see: L{Bit}
@see: L{RawBits}
"""
pass
class Bit(RawBits):
"""
Single bit: value can be False or True, and size is exactly one bit.
@see: L{Bits}
"""
static_size = 1
def __init__(self, parent, name, description=None):
"""
Constructor: see L{Field.__init__} for parameter description
"""
RawBits.__init__(self, parent, name, 1, description=description)
def createValue(self):
return 1 == self._parent.stream.readBits(
self.absolute_address, 1, self._parent.endian)
def createRawDisplay(self):
return unicode(int(self.value))
libs/hachoir_core/field/byte_field.py
-73
View File
@@ -1,73 +0,0 @@
"""
Very basic field: raw content with a size in byte. Use this class for
unknown content.
"""
from hachoir_core.field import Field, FieldError
from hachoir_core.tools import makePrintable
from hachoir_core.bits import str2hex
from hachoir_core import config
MAX_LENGTH = (2**64)
class RawBytes(Field):
"""
Byte vector of unknown content
@see: L{Bytes}
"""
static_size = staticmethod(lambda *args, **kw: args[1]*8)
def __init__(self, parent, name, length, description="Raw data"):
assert issubclass(parent.__class__, Field)
if not(0 < length <= MAX_LENGTH):
raise FieldError("Invalid RawBytes length (%s)!" % length)
Field.__init__(self, parent, name, length*8, description)
self._display = None
def _createDisplay(self, human):
max_bytes = config.max_byte_length
if type(self._getValue) is type(lambda: None):
display = self.value[:max_bytes]
else:
if self._display is None:
address = self.absolute_address
length = min(self._size / 8, max_bytes)
self._display = self._parent.stream.readBytes(address, length)
display = self._display
truncated = (8 * len(display) < self._size)
if human:
if truncated:
display += "(...)"
return makePrintable(display, "latin-1", quote='"', to_unicode=True)
else:
display = str2hex(display, format=r"\x%02x")
if truncated:
return '"%s(...)"' % display
else:
return '"%s"' % display
def createDisplay(self):
return self._createDisplay(True)
def createRawDisplay(self):
return self._createDisplay(False)
def hasValue(self):
return True
def createValue(self):
assert (self._size % 8) == 0
if self._display:
self._display = None
return self._parent.stream.readBytes(
self.absolute_address, self._size / 8)
class Bytes(RawBytes):
"""
Byte vector: can be used for magic number or GUID/UUID for example.
@see: L{RawBytes}
"""
pass
libs/hachoir_core/field/character.py
-27
View File
@@ -1,27 +0,0 @@
"""
Character field class: a 8-bit character
"""
from hachoir_core.field import Bits
from hachoir_core.endian import BIG_ENDIAN
from hachoir_core.tools import makePrintable
class Character(Bits):
"""
A 8-bit character using ASCII charset for display attribute.
"""
static_size = 8
def __init__(self, parent, name, description=None):
Bits.__init__(self, parent, name, 8, description=description)
def createValue(self):
return chr(self._parent.stream.readBits(
self.absolute_address, 8, BIG_ENDIAN))
def createRawDisplay(self):
return unicode(Bits.createValue(self))
def createDisplay(self):
return makePrintable(self.value, "ASCII", quote="'", to_unicode=True)
libs/hachoir_core/field/enum.py
-26
View File
@@ -1,26 +0,0 @@
def Enum(field, enum, key_func=None):
"""
Enum is an adapter to another field: it will just change its display
attribute. It uses a dictionary to associate a value to another.
key_func is an optional function with prototype "def func(key)->key"
which is called to transform key.
"""
display = field.createDisplay
if key_func:
def createDisplay():
try:
key = key_func(field.value)
return enum[key]
except LookupError:
return display()
else:
def createDisplay():
try:
return enum[field.value]
except LookupError:
return display()
field.createDisplay = createDisplay
field.getEnum = lambda: enum
return field
libs/hachoir_core/field/fake_array.py
-81
View File
@@ -1,81 +0,0 @@
import itertools
from hachoir_core.field import MissingField
class FakeArray:
"""
Simulate an array for GenericFieldSet.array(): fielset.array("item")[0] is
equivalent to fielset.array("item[0]").
It's possible to iterate over the items using::
for element in fieldset.array("item"):
...
And to get array size using len(fieldset.array("item")).
"""
def __init__(self, fieldset, name):
pos = name.rfind("/")
if pos != -1:
self.fieldset = fieldset[name[:pos]]
self.name = name[pos+1:]
else:
self.fieldset = fieldset
self.name = name
self._format = "%s[%%u]" % self.name
self._cache = {}
self._known_size = False
self._max_index = -1
def __nonzero__(self):
"Is the array empty or not?"
if self._cache:
return True
else:
return (0 in self)
def __len__(self):
"Number of fields in the array"
total = self._max_index+1
if not self._known_size:
for index in itertools.count(total):
try:
field = self[index]
total += 1
except MissingField:
break
return total
def __contains__(self, index):
try:
field = self[index]
return True
except MissingField:
return False
def __getitem__(self, index):
"""
Get a field of the array. Returns a field, or raise MissingField
exception if the field doesn't exist.
"""
try:
value = self._cache[index]
except KeyError:
try:
value = self.fieldset[self._format % index]
except MissingField:
self._known_size = True
raise
self._cache[index] = value
self._max_index = max(index, self._max_index)
return value
def __iter__(self):
"""
Iterate in the fields in their index order: field[0], field[1], ...
"""
for index in itertools.count(0):
try:
yield self[index]
except MissingField:
raise StopIteration()
libs/hachoir_core/field/field.py
-262
View File
@@ -1,262 +0,0 @@
"""
Parent of all (field) classes in Hachoir: Field.
"""
from hachoir_core.compatibility import reversed
from hachoir_core.stream import InputFieldStream
from hachoir_core.error import HachoirError, HACHOIR_ERRORS
from hachoir_core.log import Logger
from hachoir_core.i18n import _
from hachoir_core.tools import makePrintable
from weakref import ref as weakref_ref
class FieldError(HachoirError):
"""
Error raised by a L{Field}.
@see: L{HachoirError}
"""
pass
def joinPath(path, name):
if path != "/":
return "/".join((path, name))
else:
return "/%s" % name
class MissingField(KeyError, FieldError):
def __init__(self, field, key):
KeyError.__init__(self)
self.field = field
self.key = key
def __str__(self):
return 'Can\'t get field "%s" from %s' % (self.key, self.field.path)
def __unicode__(self):
return u'Can\'t get field "%s" from %s' % (self.key, self.field.path)
class Field(Logger):
# static size can have two differents value: None (no static size), an
# integer (number of bits), or a function which returns an integer.
#
# This function receives exactly the same arguments than the constructor
# except the first one (one). Example of function:
# static_size = staticmethod(lambda *args, **kw: args[1])
static_size = None
# Indicate if this field contains other fields (is a field set) or not
is_field_set = False
def __init__(self, parent, name, size=None, description=None):
"""
Set default class attributes, set right address if None address is
given.
@param parent: Parent field of this field
@type parent: L{Field}|None
@param name: Name of the field, have to be unique in parent. If it ends
with "[]", end will be replaced with "[new_id]" (eg. "raw[]"
becomes "raw[0]", next will be "raw[1]", and then "raw[2]", etc.)
@type name: str
@param size: Size of the field in bit (can be None, so it
will be computed later)
@type size: int|None
@param address: Address in bit relative to the parent absolute address
@type address: int|None
@param description: Optional string description
@type description: str|None
"""
assert issubclass(parent.__class__, Field)
assert (size is None) or (0 <= size)
self._parent = parent
if not name:
raise ValueError("empty field name")
self._name = name
self._address = parent.nextFieldAddress()
self._size = size
self._description = description
def _logger(self):
return self.path
def createDescription(self):
return ""
def _getDescription(self):
if self._description is None:
try:
self._description = self.createDescription()
if isinstance(self._description, str):
self._description = makePrintable(
self._description, "ISO-8859-1", to_unicode=True)
except HACHOIR_ERRORS, err:
self.error("Error getting description: " + unicode(err))
self._description = ""
return self._description
description = property(_getDescription,
doc="Description of the field (string)")
def __str__(self):
return self.display
def __unicode__(self):
return self.display
def __repr__(self):
return "<%s path=%r, address=%s, size=%s>" % (
self.__class__.__name__, self.path, self._address, self._size)
def hasValue(self):
return self._getValue() is not None
def createValue(self):
raise NotImplementedError()
def _getValue(self):
try:
value = self.createValue()
except HACHOIR_ERRORS, err:
self.error(_("Unable to create value: %s") % unicode(err))
value = None
self._getValue = lambda: value
return value
value = property(lambda self: self._getValue(), doc="Value of field")
def _getParent(self):
return self._parent
parent = property(_getParent, doc="Parent of this field")
def createDisplay(self):
return unicode(self.value)
def _getDisplay(self):
if not hasattr(self, "_Field__display"):
try:
self.__display = self.createDisplay()
except HACHOIR_ERRORS, err:
self.error("Unable to create display: %s" % err)
self.__display = u""
return self.__display
display = property(lambda self: self._getDisplay(),
doc="Short (unicode) string which represents field content")
def createRawDisplay(self):
value = self.value
if isinstance(value, str):
return makePrintable(value, "ASCII", to_unicode=True)
else:
return unicode(value)
def _getRawDisplay(self):
if not hasattr(self, "_Field__raw_display"):
try:
self.__raw_display = self.createRawDisplay()
except HACHOIR_ERRORS, err:
self.error("Unable to create raw display: %s" % err)
self.__raw_display = u""
return self.__raw_display
raw_display = property(lambda self: self._getRawDisplay(),
doc="(Unicode) string which represents raw field content")
def _getName(self):
return self._name
name = property(_getName,
doc="Field name (unique in its parent field set list)")
def _getIndex(self):
if not self._parent:
return None
return self._parent.getFieldIndex(self)
index = property(_getIndex)
def _getPath(self):
if not self._parent:
return '/'
names = []
field = self
while field is not None:
names.append(field._name)
field = field._parent
names[-1] = ''
return '/'.join(reversed(names))
path = property(_getPath,
doc="Full path of the field starting at root field")
def _getAddress(self):
return self._address
address = property(_getAddress,
doc="Relative address in bit to parent address")
def _getAbsoluteAddress(self):
address = self._address
current = self._parent
while current:
address += current._address
current = current._parent
return address
absolute_address = property(_getAbsoluteAddress,
doc="Absolute address (from stream beginning) in bit")
def _getSize(self):
return self._size
size = property(_getSize, doc="Content size in bit")
def _getField(self, name, const):
if name.strip("."):
return None
field = self
for index in xrange(1, len(name)):
field = field._parent
if field is None:
break
return field
def getField(self, key, const=True):
if key:
if key[0] == "/":
if self._parent:
current = self._parent.root
else:
current = self
if len(key) == 1:
return current
key = key[1:]
else:
current = self
for part in key.split("/"):
field = current._getField(part, const)
if field is None:
raise MissingField(current, part)
current = field
return current
raise KeyError("Key must not be an empty string!")
def __getitem__(self, key):
return self.getField(key, False)
def __contains__(self, key):
try:
return self.getField(key, False) is not None
except FieldError:
return False
def _createInputStream(self, **args):
assert self._parent
return InputFieldStream(self, **args)
def getSubIStream(self):
if hasattr(self, "_sub_istream"):
stream = self._sub_istream()
else:
stream = None
if stream is None:
stream = self._createInputStream()
self._sub_istream = weakref_ref(stream)
return stream
def setSubIStream(self, createInputStream):
cis = self._createInputStream
self._createInputStream = lambda **args: createInputStream(cis, **args)
def __nonzero__(self):
"""
Method called by code like "if field: (...)".
Always returns True
"""
return True
def getFieldType(self):
return self.__class__.__name__
libs/hachoir_core/field/field_set.py
-7
View File
@@ -1,7 +0,0 @@
from hachoir_core.field import BasicFieldSet, GenericFieldSet
class FieldSet(GenericFieldSet):
def __init__(self, parent, name, *args, **kw):
assert issubclass(parent.__class__, BasicFieldSet)
GenericFieldSet.__init__(self, parent, name, parent.stream, *args, **kw)
libs/hachoir_core/field/float.py
-99
View File
@@ -1,99 +0,0 @@
from hachoir_core.field import Bit, Bits, FieldSet
from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
import struct
# Make sure that we use right struct types
assert struct.calcsize("f") == 4
assert struct.calcsize("d") == 8
assert struct.unpack("<d", "\x1f\x85\xebQ\xb8\x1e\t@")[0] == 3.14
assert struct.unpack(">d", "\xc0\0\0\0\0\0\0\0")[0] == -2.0
class FloatMantissa(Bits):
def createValue(self):
value = Bits.createValue(self)
return 1 + float(value) / (2 ** self.size)
def createRawDisplay(self):
return unicode(Bits.createValue(self))
class FloatExponent(Bits):
def __init__(self, parent, name, size):
Bits.__init__(self, parent, name, size)
self.bias = 2 ** (size-1) - 1
def createValue(self):
return Bits.createValue(self) - self.bias
def createRawDisplay(self):
return unicode(self.value + self.bias)
def floatFactory(name, format, mantissa_bits, exponent_bits, doc):
size = 1 + mantissa_bits + exponent_bits
class Float(FieldSet):
static_size = size
__doc__ = doc
def __init__(self, parent, name, description=None):
assert parent.endian in (BIG_ENDIAN, LITTLE_ENDIAN)
FieldSet.__init__(self, parent, name, description, size)
if format:
if self._parent.endian == BIG_ENDIAN:
self.struct_format = ">"+format
else:
self.struct_format = "<"+format
else:
self.struct_format = None
def createValue(self):
"""
Create float value: use struct.unpack() when it's possible
(32 and 64-bit float) or compute it with :
mantissa * (2.0 ** exponent)
This computation may raise an OverflowError.
"""
if self.struct_format:
raw = self._parent.stream.readBytes(
self.absolute_address, self._size//8)
try:
return struct.unpack(self.struct_format, raw)[0]
except struct.error, err:
raise ValueError("[%s] conversion error: %s" %
(self.__class__.__name__, err))
else:
try:
value = self["mantissa"].value * (2.0 ** float(self["exponent"].value))
if self["negative"].value:
return -(value)
else:
return value
except OverflowError:
raise ValueError("[%s] floating point overflow" %
self.__class__.__name__)
def createFields(self):
yield Bit(self, "negative")
yield FloatExponent(self, "exponent", exponent_bits)
if 64 <= mantissa_bits:
yield Bit(self, "one")
yield FloatMantissa(self, "mantissa", mantissa_bits-1)
else:
yield FloatMantissa(self, "mantissa", mantissa_bits)
cls = Float
cls.__name__ = name
return cls
# 32-bit float (standard: IEEE 754/854)
Float32 = floatFactory("Float32", "f", 23, 8,
"Floating point number: format IEEE 754 int 32 bit")
# 64-bit float (standard: IEEE 754/854)
Float64 = floatFactory("Float64", "d", 52, 11,
"Floating point number: format IEEE 754 in 64 bit")
# 80-bit float (standard: IEEE 754/854)
Float80 = floatFactory("Float80", None, 64, 15,
"Floating point number: format IEEE 754 in 80 bit")
libs/hachoir_core/field/generic_field_set.py
-532
View File
@@ -1,532 +0,0 @@
from hachoir_core.field import (MissingField, BasicFieldSet, Field, ParserError,
createRawField, createNullField, createPaddingField, FakeArray)
from hachoir_core.dict import Dict, UniqKeyError
from hachoir_core.error import HACHOIR_ERRORS
from hachoir_core.tools import lowerBound
import hachoir_core.config as config
class GenericFieldSet(BasicFieldSet):
"""
Ordered list of fields. Use operator [] to access fields using their
name (field names are unique in a field set, but not in the whole
document).
Class attributes:
- endian: Bytes order (L{BIG_ENDIAN} or L{LITTLE_ENDIAN}). Optional if the
field set has a parent ;
- static_size: (optional) Size of FieldSet in bits. This attribute should
be used in parser of constant size.
Instance attributes/methods:
- _fields: Ordered dictionnary of all fields, may be incomplete
because feeded when a field is requested ;
- stream: Input stream used to feed fields' value
- root: The root of all field sets ;
- __len__(): Number of fields, may need to create field set ;
- __getitem__(): Get an field by it's name or it's path.
And attributes inherited from Field class:
- parent: Parent field (may be None if it's the root) ;
- name: Field name (unique in parent field set) ;
- value: The field set ;
- address: Field address (in bits) relative to parent ;
- description: A string describing the content (can be None) ;
- size: Size of field set in bits, may need to create field set.
Event handling:
- "connectEvent": Connect an handler to an event ;
- "raiseEvent": Raise an event.
To implement a new field set, you need to:
- create a class which inherite from FieldSet ;
- write createFields() method using lines like:
yield Class(self, "name", ...) ;
- and maybe set endian and static_size class attributes.
"""
_current_size = 0
def __init__(self, parent, name, stream, description=None, size=None):
"""
Constructor
@param parent: Parent field set, None for root parser
@param name: Name of the field, have to be unique in parent. If it ends
with "[]", end will be replaced with "[new_id]" (eg. "raw[]"
becomes "raw[0]", next will be "raw[1]", and then "raw[2]", etc.)
@type name: str
@param stream: Input stream from which data are read
@type stream: L{InputStream}
@param description: Optional string description
@type description: str|None
@param size: Size in bits. If it's None, size will be computed. You
can also set size with class attribute static_size
"""
BasicFieldSet.__init__(self, parent, name, stream, description, size)
self._fields = Dict()
self._field_generator = self.createFields()
self._array_cache = {}
self.__is_feeding = False
def array(self, key):
try:
return self._array_cache[key]
except KeyError:
array = FakeArray(self, key)
self._array_cache[key] = array
return self._array_cache[key]
def reset(self):
"""
Reset a field set:
* clear fields ;
* restart field generator ;
* set current size to zero ;
* clear field array count.
But keep: name, value, description and size.
"""
BasicFieldSet.reset(self)
self._fields = Dict()
self._field_generator = self.createFields()
self._current_size = 0
self._array_cache = {}
def __str__(self):
return '<%s path=%s, current_size=%s, current length=%s>' % \
(self.__class__.__name__, self.path, self._current_size, len(self._fields))
def __len__(self):
"""
Returns number of fields, may need to create all fields
if it's not done yet.
"""
if self._field_generator is not None:
self._feedAll()
return len(self._fields)
def _getCurrentLength(self):
return len(self._fields)
current_length = property(_getCurrentLength)
def _getSize(self):
if self._size is None:
self._feedAll()
return self._size
size = property(_getSize, doc="Size in bits, may create all fields to get size")
def _getCurrentSize(self):
assert not(self.done)
return self._current_size
current_size = property(_getCurrentSize)
eof = property(lambda self: self._checkSize(self._current_size + 1, True) < 0)
def _checkSize(self, size, strict):
field = self
while field._size is None:
if not field._parent:
assert self.stream.size is None
if not strict:
return None
if self.stream.sizeGe(size):
return 0
break
size += field._address
field = field._parent
return field._size - size
autofix = property(lambda self: self.root.autofix)
def _addField(self, field):
"""
Add a field to the field set:
* add it into _fields
* update _current_size
May raise a StopIteration() on error
"""
if not issubclass(field.__class__, Field):
raise ParserError("Field type (%s) is not a subclass of 'Field'!"
% field.__class__.__name__)
assert isinstance(field._name, str)
if field._name.endswith("[]"):
self.setUniqueFieldName(field)
if config.debug:
self.info("[+] DBG: _addField(%s)" % field.name)
# required for the msoffice parser
if field._address != self._current_size:
self.warning("Fix address of %s to %s (was %s)" %
(field.path, self._current_size, field._address))
field._address = self._current_size
ask_stop = False
# Compute field size and check that there is enough place for it
self.__is_feeding = True
try:
field_size = field.size
except HACHOIR_ERRORS, err:
if field.is_field_set and field.current_length and field.eof:
self.warning("Error when getting size of '%s': %s" % (field.name, err))
field._stopFeeding()
ask_stop = True
else:
self.warning("Error when getting size of '%s': delete it" % field.name)
self.__is_feeding = False
raise
self.__is_feeding = False
# No more place?
dsize = self._checkSize(field._address + field.size, False)
if (dsize is not None and dsize < 0) or (field.is_field_set and field.size <= 0):
if self.autofix and self._current_size:
self._fixFieldSize(field, field.size + dsize)
else:
raise ParserError("Field %s is too large!" % field.path)
self._current_size += field.size
try:
self._fields.append(field._name, field)
except UniqKeyError, err:
self.warning("Duplicate field name " + unicode(err))
field._name += "[]"
self.setUniqueFieldName(field)
self._fields.append(field._name, field)
if ask_stop:
raise StopIteration()
def _fixFieldSize(self, field, new_size):
if new_size > 0:
if field.is_field_set and 0 < field.size:
field._truncate(new_size)
return
# Don't add the field <=> delete item
if self._size is None:
self._size = self._current_size + new_size
self.warning("[Autofix] Delete '%s' (too large)" % field.path)
raise StopIteration()
def _getField(self, name, const):
field = Field._getField(self, name, const)
if field is None:
if name in self._fields:
field = self._fields[name]
elif self._field_generator is not None and not const:
field = self._feedUntil(name)
return field
def getField(self, key, const=True):
if isinstance(key, (int, long)):
if key < 0:
raise KeyError("Key must be positive!")
if not const:
self.readFirstFields(key+1)
if len(self._fields.values) <= key:
raise MissingField(self, key)
return self._fields.values[key]
return Field.getField(self, key, const)
def _truncate(self, size):
assert size > 0
if size < self._current_size:
self._size = size
while True:
field = self._fields.values[-1]
if field._address < size:
break
del self._fields[-1]
self._current_size = field._address
size -= field._address
if size < field._size:
if field.is_field_set:
field._truncate(size)
else:
del self._fields[-1]
field = createRawField(self, size, "raw[]")
self._fields.append(field._name, field)
self._current_size = self._size
else:
assert size < self._size or self._size is None
self._size = size
if self._size == self._current_size:
self._field_generator = None
def _deleteField(self, index):
field = self._fields.values[index]
size = field.size
self._current_size -= size
del self._fields[index]
return field
def _fixLastField(self):
"""
Try to fix last field when we know current field set size.
Returns new added field if any, or None.
"""
assert self._size is not None
# Stop parser
message = ["stop parser"]
self._field_generator = None
# If last field is too big, delete it
while self._size < self._current_size:
field = self._deleteField(len(self._fields)-1)
message.append("delete field %s" % field.path)
assert self._current_size <= self._size
# If field size current is smaller: add a raw field
size = self._size - self._current_size
if size:
field = createRawField(self, size, "raw[]")
message.append("add padding")
self._current_size += field.size
self._fields.append(field._name, field)
else:
field = None
message = ", ".join(message)
self.warning("[Autofix] Fix parser error: " + message)
assert self._current_size == self._size
return field
def _stopFeeding(self):
new_field = None
if self._size is None:
if self._parent:
self._size = self._current_size
elif self._size != self._current_size:
if self.autofix:
new_field = self._fixLastField()
else:
raise ParserError("Invalid parser \"%s\" size!" % self.path)
self._field_generator = None
return new_field
def _fixFeedError(self, exception):
"""
Try to fix a feeding error. Returns False if error can't be fixed,
otherwise returns new field if any, or None.
"""
if self._size is None or not self.autofix:
return False
self.warning(unicode(exception))
return self._fixLastField()
def _feedUntil(self, field_name):
"""
Return the field if it was found, None else
"""
if self.__is_feeding \
or (self._field_generator and self._field_generator.gi_running):
self.warning("Unable to get %s (and generator is already running)"
% field_name)
return None
try:
while True:
field = self._field_generator.next()
self._addField(field)
if field.name == field_name:
return field
except HACHOIR_ERRORS, err:
if self._fixFeedError(err) is False:
raise
except StopIteration:
self._stopFeeding()
return None
def readMoreFields(self, number):
"""
Read more number fields, or do nothing if parsing is done.
Returns number of new added fields.
"""
if self._field_generator is None:
return 0
oldlen = len(self._fields)
try:
for index in xrange(number):
self._addField( self._field_generator.next() )
except HACHOIR_ERRORS, err:
if self._fixFeedError(err) is False:
raise
except StopIteration:
self._stopFeeding()
return len(self._fields) - oldlen
def _feedAll(self):
if self._field_generator is None:
return
try:
while True:
field = self._field_generator.next()
self._addField(field)
except HACHOIR_ERRORS, err:
if self._fixFeedError(err) is False:
raise
except StopIteration:
self._stopFeeding()
def __iter__(self):
"""
Create a generator to iterate on each field, may create new
fields when needed
"""
try:
done = 0
while True:
if done == len(self._fields):
if self._field_generator is None:
break
self._addField( self._field_generator.next() )
for field in self._fields.values[done:]:
yield field
done += 1
except HACHOIR_ERRORS, err:
field = self._fixFeedError(err)
if isinstance(field, Field):
yield field
elif hasattr(field, '__iter__'):
for f in field:
yield f
elif field is False:
raise
except StopIteration:
field = self._stopFeeding()
if isinstance(field, Field):
yield field
elif hasattr(field, '__iter__'):
for f in field:
yield f
def _isDone(self):
return (self._field_generator is None)
done = property(_isDone, doc="Boolean to know if parsing is done or not")
#
# FieldSet_SeekUtility
#
def seekBit(self, address, name="padding[]",
description=None, relative=True, null=False):
"""
Create a field to seek to specified address,
or None if it's not needed.
May raise an (ParserError) exception if address is invalid.
"""
if relative:
nbits = address - self._current_size
else:
nbits = address - (self.absolute_address + self._current_size)
if nbits < 0:
raise ParserError("Seek error, unable to go back!")
if 0 < nbits:
if null:
return createNullField(self, nbits, name, description)
else:
return createPaddingField(self, nbits, name, description)
else:
return None
def seekByte(self, address, name="padding[]", description=None, relative=True, null=False):
"""
Same as seekBit(), but with address in byte.
"""
return self.seekBit(address * 8, name, description, relative, null=null)
#
# RandomAccessFieldSet
#
def replaceField(self, name, new_fields):
# TODO: Check in self and not self.field
# Problem is that "generator is already executing"
if name not in self._fields:
raise ParserError("Unable to replace %s: field doesn't exist!" % name)
assert 1 <= len(new_fields)
old_field = self[name]
total_size = sum( (field.size for field in new_fields) )
if old_field.size != total_size:
raise ParserError("Unable to replace %s: "
"new field(s) hasn't same size (%u bits instead of %u bits)!"
% (name, total_size, old_field.size))
field = new_fields[0]
if field._name.endswith("[]"):
self.setUniqueFieldName(field)
field._address = old_field.address
if field.name != name and field.name in self._fields:
raise ParserError(
"Unable to replace %s: name \"%s\" is already used!"
% (name, field.name))
self._fields.replace(name, field.name, field)
self.raiseEvent("field-replaced", old_field, field)
if 1 < len(new_fields):
index = self._fields.index(new_fields[0].name)+1
address = field.address + field.size
for field in new_fields[1:]:
if field._name.endswith("[]"):
self.setUniqueFieldName(field)
field._address = address
if field.name in self._fields:
raise ParserError(
"Unable to replace %s: name \"%s\" is already used!"
% (name, field.name))
self._fields.insert(index, field.name, field)
self.raiseEvent("field-inserted", index, field)
index += 1
address += field.size
def getFieldByAddress(self, address, feed=True):
"""
Only search in existing fields
"""
if feed and self._field_generator is not None:
self._feedAll()
if address < self._current_size:
i = lowerBound(self._fields.values, lambda x: x.address + x.size <= address)
if i is not None:
return self._fields.values[i]
return None
def writeFieldsIn(self, old_field, address, new_fields):
"""
Can only write in existing fields (address < self._current_size)
"""
# Check size
total_size = sum( field.size for field in new_fields )
if old_field.size < total_size:
raise ParserError( \
"Unable to write fields at address %s " \
"(too big)!" % (address))
# Need padding before?
replace = []
size = address - old_field.address
assert 0 <= size
if 0 < size:
padding = createPaddingField(self, size)
padding._address = old_field.address
replace.append(padding)
# Set fields address
for field in new_fields:
field._address = address
address += field.size
replace.append(field)
# Need padding after?
size = (old_field.address + old_field.size) - address
assert 0 <= size
if 0 < size:
padding = createPaddingField(self, size)
padding._address = address
replace.append(padding)
self.replaceField(old_field.name, replace)
def nextFieldAddress(self):
return self._current_size
def getFieldIndex(self, field):
return self._fields.index(field._name)
libs/hachoir_core/field/helper.py
-57
View File
@@ -1,57 +0,0 @@
from hachoir_core.field import (FieldError,
RawBits, RawBytes,
PaddingBits, PaddingBytes,
NullBits, NullBytes,
GenericString, GenericInteger)
from hachoir_core.stream import FileOutputStream
def createRawField(parent, size, name="raw[]", description=None):
if size <= 0:
raise FieldError("Unable to create raw field of %s bits" % size)
if (size % 8) == 0:
return RawBytes(parent, name, size/8, description)
else:
return RawBits(parent, name, size, description)
def createPaddingField(parent, nbits, name="padding[]", description=None):
if nbits <= 0:
raise FieldError("Unable to create padding of %s bits" % nbits)
if (nbits % 8) == 0:
return PaddingBytes(parent, name, nbits/8, description)
else:
return PaddingBits(parent, name, nbits, description)
def createNullField(parent, nbits, name="padding[]", description=None):
if nbits <= 0:
raise FieldError("Unable to create null padding of %s bits" % nbits)
if (nbits % 8) == 0:
return NullBytes(parent, name, nbits/8, description)
else:
return NullBits(parent, name, nbits, description)
def isString(field):
return issubclass(field.__class__, GenericString)
def isInteger(field):
return issubclass(field.__class__, GenericInteger)
def writeIntoFile(fieldset, filename):
output = FileOutputStream(filename)
fieldset.writeInto(output)
def createOrphanField(fieldset, address, field_cls, *args, **kw):
"""
Create an orphan field at specified address:
field_cls(fieldset, *args, **kw)
The field uses the fieldset properties but it isn't added to the
field set.
"""
save_size = fieldset._current_size
try:
fieldset._current_size = address
field = field_cls(fieldset, *args, **kw)
finally:
fieldset._current_size = save_size
return field
libs/hachoir_core/field/integer.py
-44
View File
@@ -1,44 +0,0 @@
"""
Integer field classes:
- UInt8, UInt16, UInt24, UInt32, UInt64: unsigned integer of 8, 16, 32, 64 bits ;
- Int8, Int16, Int24, Int32, Int64: signed integer of 8, 16, 32, 64 bits.
"""
from hachoir_core.field import Bits, FieldError
class GenericInteger(Bits):
"""
Generic integer class used to generate other classes.
"""
def __init__(self, parent, name, signed, size, description=None):
if not (8 <= size <= 256):
raise FieldError("Invalid integer size (%s): have to be in 8..256" % size)
Bits.__init__(self, parent, name, size, description)
self.signed = signed
def createValue(self):
return self._parent.stream.readInteger(
self.absolute_address, self.signed, self._size, self._parent.endian)
def integerFactory(name, is_signed, size, doc):
class Integer(GenericInteger):
__doc__ = doc
static_size = size
def __init__(self, parent, name, description=None):
GenericInteger.__init__(self, parent, name, is_signed, size, description)
cls = Integer
cls.__name__ = name
return cls
UInt8 = integerFactory("UInt8", False, 8, "Unsigned integer of 8 bits")
UInt16 = integerFactory("UInt16", False, 16, "Unsigned integer of 16 bits")
UInt24 = integerFactory("UInt24", False, 24, "Unsigned integer of 24 bits")
UInt32 = integerFactory("UInt32", False, 32, "Unsigned integer of 32 bits")
UInt64 = integerFactory("UInt64", False, 64, "Unsigned integer of 64 bits")
Int8 = integerFactory("Int8", True, 8, "Signed integer of 8 bits")
Int16 = integerFactory("Int16", True, 16, "Signed integer of 16 bits")
Int24 = integerFactory("Int24", True, 24, "Signed integer of 24 bits")
Int32 = integerFactory("Int32", True, 32, "Signed integer of 32 bits")
Int64 = integerFactory("Int64", True, 64, "Signed integer of 64 bits")
libs/hachoir_core/field/link.py
-109
View File
@@ -1,109 +0,0 @@
from hachoir_core.field import Field, FieldSet, ParserError, Bytes, MissingField
from hachoir_core.stream import FragmentedStream
class Link(Field):
def __init__(self, parent, name, *args, **kw):
Field.__init__(self, parent, name, 0, *args, **kw)
def hasValue(self):
return True
def createValue(self):
return self._parent[self.display]
def createDisplay(self):
value = self.value
if value is None:
return "<%s>" % MissingField.__name__
return value.path
def _getField(self, name, const):
target = self.value
assert self != target
return target._getField(name, const)
class Fragments:
def __init__(self, first):
self.first = first
def __iter__(self):
fragment = self.first
while fragment is not None:
data = fragment.getData()
yield data and data.size
fragment = fragment.next
class Fragment(FieldSet):
_first = None
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._field_generator = self._createFields(self._field_generator)
if self.__class__.createFields == Fragment.createFields:
self._getData = lambda: self
def getData(self):
try:
return self._getData()
except MissingField, e:
self.error(str(e))
return None
def setLinks(self, first, next=None):
self._first = first or self
self._next = next
self._feedLinks = lambda: self
return self
def _feedLinks(self):
while self._first is None and self.readMoreFields(1):
pass
if self._first is None:
raise ParserError("first is None")
return self
first = property(lambda self: self._feedLinks()._first)
def _getNext(self):
next = self._feedLinks()._next
if callable(next):
self._next = next = next()
return next
next = property(_getNext)
def _createInputStream(self, **args):
first = self.first
if first is self and hasattr(first, "_getData"):
return FragmentedStream(first, packets=Fragments(first), **args)
return FieldSet._createInputStream(self, **args)
def _createFields(self, field_generator):
if self._first is None:
for field in field_generator:
if self._first is not None:
break
yield field
else:
raise ParserError("Fragment.setLinks not called")
else:
field = None
if self._first is not self:
link = Link(self, "first", None)
link._getValue = lambda: self._first
yield link
if self._next:
link = Link(self, "next", None)
link.createValue = self._getNext
yield link
if field:
yield field
for field in field_generator:
yield field
def createFields(self):
if self._size is None:
self._size = self._getSize()
yield Bytes(self, "data", self._size/8)
libs/hachoir_core/field/new_seekable_field_set.py
-82
View File
@@ -1,82 +0,0 @@
from hachoir_core.field import BasicFieldSet, GenericFieldSet, ParserError, createRawField
from hachoir_core.error import HACHOIR_ERRORS
# getgaps(int, int, [listof (int, int)]) -> generator of (int, int)
# Gets all the gaps not covered by a block in `blocks` from `start` for `length` units.
def getgaps(start, length, blocks):
'''
Example:
>>> list(getgaps(0, 20, [(15,3), (6,2), (6,2), (1,2), (2,3), (11,2), (9,5)]))
[(0, 1), (5, 1), (8, 1), (14, 1), (18, 2)]
'''
# done this way to avoid mutating the original
blocks = sorted(blocks, key=lambda b: b[0])
end = start+length
for s, l in blocks:
if s > start:
yield (start, s-start)
start = s
if s+l > start:
start = s+l
if start < end:
yield (start, end-start)
class NewRootSeekableFieldSet(GenericFieldSet):
def seekBit(self, address, relative=True):
if not relative:
address -= self.absolute_address
if address < 0:
raise ParserError("Seek below field set start (%s.%s)" % divmod(address, 8))
self._current_size = address
return None
def seekByte(self, address, relative=True):
return self.seekBit(address*8, relative)
def _fixLastField(self):
"""
Try to fix last field when we know current field set size.
Returns new added field if any, or None.
"""
assert self._size is not None
# Stop parser
message = ["stop parser"]
self._field_generator = None
# If last field is too big, delete it
while self._size < self._current_size:
field = self._deleteField(len(self._fields)-1)
message.append("delete field %s" % field.path)
assert self._current_size <= self._size
blocks = [(x.absolute_address, x.size) for x in self._fields]
fields = []
for start, length in getgaps(self.absolute_address, self._size, blocks):
self.seekBit(start, relative=False)
field = createRawField(self, length, "unparsed[]")
self.setUniqueFieldName(field)
self._fields.append(field.name, field)
fields.append(field)
message.append("found unparsed segment: start %s, length %s" % (start, length))
self.seekBit(self._size, relative=False)
message = ", ".join(message)
if fields:
self.warning("[Autofix] Fix parser error: " + message)
return fields
def _stopFeeding(self):
new_field = None
if self._size is None:
if self._parent:
self._size = self._current_size
new_field = self._fixLastField()
self._field_generator = None
return new_field
class NewSeekableFieldSet(NewRootSeekableFieldSet):
def __init__(self, parent, name, description=None, size=None):
assert issubclass(parent.__class__, BasicFieldSet)
NewRootSeekableFieldSet.__init__(self, parent, name, parent.stream, description, size)
libs/hachoir_core/field/padding.py
-138
View File
@@ -1,138 +0,0 @@
from hachoir_core.field import Bits, Bytes
from hachoir_core.tools import makePrintable, humanFilesize
from hachoir_core import config
class PaddingBits(Bits):
"""
Padding bits used, for example, to align address (of next field).
See also NullBits and PaddingBytes types.
Arguments:
* nbits: Size of the field in bits
Optional arguments:
* pattern (int): Content pattern, eg. 0 if all bits are set to 0
"""
static_size = staticmethod(lambda *args, **kw: args[1])
MAX_SIZE = 128
def __init__(self, parent, name, nbits, description="Padding", pattern=None):
Bits.__init__(self, parent, name, nbits, description)
self.pattern = pattern
self._display_pattern = self.checkPattern()
def checkPattern(self):
if not(config.check_padding_pattern):
return False
if self.pattern != 0:
return False
if self.MAX_SIZE < self._size:
value = self._parent.stream.readBits(
self.absolute_address, self.MAX_SIZE, self._parent.endian)
else:
value = self.value
if value != 0:
self.warning("padding contents doesn't look normal (invalid pattern)")
return False
if self.MAX_SIZE < self._size:
self.info("only check first %u bits" % self.MAX_SIZE)
return True
def createDisplay(self):
if self._display_pattern:
return u"<padding pattern=%s>" % self.pattern
else:
return Bits.createDisplay(self)
class PaddingBytes(Bytes):
"""
Padding bytes used, for example, to align address (of next field).
See also NullBytes and PaddingBits types.
Arguments:
* nbytes: Size of the field in bytes
Optional arguments:
* pattern (str): Content pattern, eg. "\0" for nul bytes
"""
static_size = staticmethod(lambda *args, **kw: args[1]*8)
MAX_SIZE = 4096
def __init__(self, parent, name, nbytes,
description="Padding", pattern=None):
""" pattern is None or repeated string """
assert (pattern is None) or (isinstance(pattern, str))
Bytes.__init__(self, parent, name, nbytes, description)
self.pattern = pattern
self._display_pattern = self.checkPattern()
def checkPattern(self):
if not(config.check_padding_pattern):
return False
if self.pattern is None:
return False
if self.MAX_SIZE < self._size/8:
self.info("only check first %s of padding" % humanFilesize(self.MAX_SIZE))
content = self._parent.stream.readBytes(
self.absolute_address, self.MAX_SIZE)
else:
content = self.value
index = 0
pattern_len = len(self.pattern)
while index < len(content):
if content[index:index+pattern_len] != self.pattern:
self.warning(
"padding contents doesn't look normal"
" (invalid pattern at byte %u)!"
% index)
return False
index += pattern_len
return True
def createDisplay(self):
if self._display_pattern:
return u"<padding pattern=%s>" % makePrintable(self.pattern, "ASCII", quote="'")
else:
return Bytes.createDisplay(self)
def createRawDisplay(self):
return Bytes.createDisplay(self)
class NullBits(PaddingBits):
"""
Null padding bits used, for example, to align address (of next field).
See also PaddingBits and NullBytes types.
Arguments:
* nbits: Size of the field in bits
"""
def __init__(self, parent, name, nbits, description=None):
PaddingBits.__init__(self, parent, name, nbits, description, pattern=0)
def createDisplay(self):
if self._display_pattern:
return "<null>"
else:
return Bits.createDisplay(self)
class NullBytes(PaddingBytes):
"""
Null padding bytes used, for example, to align address (of next field).
See also PaddingBytes and NullBits types.
Arguments:
* nbytes: Size of the field in bytes
"""
def __init__(self, parent, name, nbytes, description=None):
PaddingBytes.__init__(self, parent, name, nbytes, description, pattern="\0")
def createDisplay(self):
if self._display_pattern:
return "<null>"
else:
return Bytes.createDisplay(self)
libs/hachoir_core/field/parser.py
-40
View File
@@ -1,40 +0,0 @@
from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
from hachoir_core.field import GenericFieldSet
from hachoir_core.log import Logger
import hachoir_core.config as config
class Parser(GenericFieldSet):
"""
A parser is the root of all other fields. It create first level of fields
and have special attributes and methods:
- endian: Byte order (L{BIG_ENDIAN} or L{LITTLE_ENDIAN}) of input data ;
- stream: Data input stream (set in L{__init__()}) ;
- size: Field set size will be size of input stream.
"""
def __init__(self, stream, description=None):
"""
Parser constructor
@param stream: Data input stream (see L{InputStream})
@param description: (optional) String description
"""
# Check arguments
assert hasattr(self, "endian") \
and self.endian in (BIG_ENDIAN, LITTLE_ENDIAN)
# Call parent constructor
GenericFieldSet.__init__(self, None, "root", stream, description, stream.askSize(self))
def _logger(self):
return Logger._logger(self)
def _setSize(self, size):
self._truncate(size)
self.raiseEvent("field-resized", self)
size = property(lambda self: self._size, doc="Size in bits")
path = property(lambda self: "/")
# dummy definition to prevent hachoir-core from depending on hachoir-parser
autofix = property(lambda self: config.autofix)
libs/hachoir_core/field/seekable_field_set.py
-182
View File
@@ -1,182 +0,0 @@
from hachoir_core.field import Field, BasicFieldSet, FakeArray, MissingField, ParserError
from hachoir_core.tools import makeUnicode
from hachoir_core.error import HACHOIR_ERRORS
from itertools import repeat
import hachoir_core.config as config
class RootSeekableFieldSet(BasicFieldSet):
def __init__(self, parent, name, stream, description, size):
BasicFieldSet.__init__(self, parent, name, stream, description, size)
self._generator = self.createFields()
self._offset = 0
self._current_size = 0
if size:
self._current_max_size = size
else:
self._current_max_size = 0
self._field_dict = {}
self._field_array = []
def _feedOne(self):
assert self._generator
field = self._generator.next()
self._addField(field)
return field
def array(self, key):
return FakeArray(self, key)
def getFieldByAddress(self, address, feed=True):
for field in self._field_array:
if field.address <= address < field.address + field.size:
return field
for field in self._readFields():
if field.address <= address < field.address + field.size:
return field
return None
def _stopFeed(self):
self._size = self._current_max_size
self._generator = None
done = property(lambda self: not bool(self._generator))
def _getSize(self):
if self._size is None:
self._feedAll()
return self._size
size = property(_getSize)
def _getField(self, key, const):
field = Field._getField(self, key, const)
if field is not None:
return field
if key in self._field_dict:
return self._field_dict[key]
if self._generator and not const:
try:
while True:
field = self._feedOne()
if field.name == key:
return field
except StopIteration:
self._stopFeed()
except HACHOIR_ERRORS, err:
self.error("Error: %s" % makeUnicode(err))
self._stopFeed()
return None
def getField(self, key, const=True):
if isinstance(key, (int, long)):
if key < 0:
raise KeyError("Key must be positive!")
if not const:
self.readFirstFields(key+1)
if len(self._field_array) <= key:
raise MissingField(self, key)
return self._field_array[key]
return Field.getField(self, key, const)
def _addField(self, field):
if field._name.endswith("[]"):
self.setUniqueFieldName(field)
if config.debug:
self.info("[+] DBG: _addField(%s)" % field.name)
if field._address != self._offset:
self.warning("Set field %s address to %s (was %s)" % (
field.path, self._offset//8, field._address//8))
field._address = self._offset
assert field.name not in self._field_dict
self._checkFieldSize(field)
self._field_dict[field.name] = field
self._field_array.append(field)
self._current_size += field.size
self._offset += field.size
self._current_max_size = max(self._current_max_size, field.address + field.size)
def _checkAddress(self, address):
if self._size is not None:
max_addr = self._size
else:
# FIXME: Use parent size
max_addr = self.stream.size
return address < max_addr
def _checkFieldSize(self, field):
size = field.size
addr = field.address
if not self._checkAddress(addr+size-1):
raise ParserError("Unable to add %s: field is too large" % field.name)
def seekBit(self, address, relative=True):
if not relative:
address -= self.absolute_address
if address < 0:
raise ParserError("Seek below field set start (%s.%s)" % divmod(address, 8))
if not self._checkAddress(address):
raise ParserError("Seek above field set end (%s.%s)" % divmod(address, 8))
self._offset = address
return None
def seekByte(self, address, relative=True):
return self.seekBit(address*8, relative)
def readMoreFields(self, number):
return self._readMoreFields(xrange(number))
def _feedAll(self):
return self._readMoreFields(repeat(1))
def _readFields(self):
while True:
added = self._readMoreFields(xrange(1))
if not added:
break
yield self._field_array[-1]
def _readMoreFields(self, index_generator):
added = 0
if self._generator:
try:
for index in index_generator:
self._feedOne()
added += 1
except StopIteration:
self._stopFeed()
except HACHOIR_ERRORS, err:
self.error("Error: %s" % makeUnicode(err))
self._stopFeed()
return added
current_length = property(lambda self: len(self._field_array))
current_size = property(lambda self: self._offset)
def __iter__(self):
for field in self._field_array:
yield field
if self._generator:
try:
while True:
yield self._feedOne()
except StopIteration:
self._stopFeed()
raise StopIteration
def __len__(self):
if self._generator:
self._feedAll()
return len(self._field_array)
def nextFieldAddress(self):
return self._offset
def getFieldIndex(self, field):
return self._field_array.index(field)
class SeekableFieldSet(RootSeekableFieldSet):
def __init__(self, parent, name, description=None, size=None):
assert issubclass(parent.__class__, BasicFieldSet)
RootSeekableFieldSet.__init__(self, parent, name, parent.stream, description, size)
libs/hachoir_core/field/static_field_set.py
-54
View File
@@ -1,54 +0,0 @@
from hachoir_core.field import FieldSet, ParserError
class StaticFieldSet(FieldSet):
"""
Static field set: format class attribute is a tuple of all fields
in syntax like:
format = (
(TYPE1, ARG1, ARG2, ...),
(TYPE2, ARG1, ARG2, ..., {KEY1=VALUE1, ...}),
...
)
Types with dynamic size are forbidden, eg. CString, PascalString8, etc.
"""
format = None # You have to redefine this class variable
_class = None
def __new__(cls, *args, **kw):
assert cls.format is not None, "Class attribute 'format' is not set"
if cls._class is not cls.__name__:
cls._class = cls.__name__
cls.static_size = cls._computeStaticSize()
return object.__new__(cls, *args, **kw)
@staticmethod
def _computeItemSize(item):
item_class = item[0]
if item_class.static_size is None:
raise ParserError("Unable to get static size of field type: %s"
% item_class.__name__)
if callable(item_class.static_size):
if isinstance(item[-1], dict):
return item_class.static_size(*item[1:-1], **item[-1])
else:
return item_class.static_size(*item[1:])
else:
assert isinstance(item_class.static_size, (int, long))
return item_class.static_size
def createFields(self):
for item in self.format:
if isinstance(item[-1], dict):
yield item[0](self, *item[1:-1], **item[-1])
else:
yield item[0](self, *item[1:])
@classmethod
def _computeStaticSize(cls, *args):
return sum(cls._computeItemSize(item) for item in cls.format)
# Initial value of static_size, it changes when first instance
# is created (see __new__)
static_size = _computeStaticSize
libs/hachoir_core/field/string_field.py
-402
View File
@@ -1,402 +0,0 @@
"""
String field classes:
- String: Fixed length string (no prefix/no suffix) ;
- CString: String which ends with nul byte ("\0") ;
- UnixLine: Unix line of text, string which ends with "\n" ;
- PascalString8, PascalString16, PascalString32: String prefixed with
length written in a 8, 16, 32-bit integer (use parent endian).
Constructor has optional arguments:
- strip: value can be a string or True ;
- charset: if set, convert string to unicode using this charset (in "replace"
mode which replace all buggy characters with ".").
Note: For PascalStringXX, prefixed value is the number of bytes and not
of characters!
"""
from hachoir_core.field import FieldError, Bytes
from hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
from hachoir_core.tools import alignValue, makePrintable
from hachoir_core.i18n import guessBytesCharset, _
from hachoir_core import config
from codecs import BOM_UTF16_LE, BOM_UTF16_BE, BOM_UTF32_LE, BOM_UTF32_BE
# Default charset used to convert byte string to Unicode
# This charset is used if no charset is specified or on conversion error
FALLBACK_CHARSET = "ISO-8859-1"
class GenericString(Bytes):
"""
Generic string class.
charset have to be in CHARSET_8BIT or in UTF_CHARSET.
"""
VALID_FORMATS = ("C", "UnixLine",
"fixed", "Pascal8", "Pascal16", "Pascal32")
# 8-bit charsets
CHARSET_8BIT = set((
"ASCII", # ANSI X3.4-1968
"MacRoman",
"CP037", # EBCDIC 037
"CP874", # Thai
"WINDOWS-1250", # Central Europe
"WINDOWS-1251", # Cyrillic
"WINDOWS-1252", # Latin I
"WINDOWS-1253", # Greek
"WINDOWS-1254", # Turkish
"WINDOWS-1255", # Hebrew
"WINDOWS-1256", # Arabic
"WINDOWS-1257", # Baltic
"WINDOWS-1258", # Vietnam
"ISO-8859-1", # Latin-1
"ISO-8859-2", # Latin-2
"ISO-8859-3", # Latin-3
"ISO-8859-4", # Latin-4
"ISO-8859-5",
"ISO-8859-6",
"ISO-8859-7",
"ISO-8859-8",
"ISO-8859-9", # Latin-5
"ISO-8859-10", # Latin-6
"ISO-8859-11", # Thai
"ISO-8859-13", # Latin-7
"ISO-8859-14", # Latin-8
"ISO-8859-15", # Latin-9 or ("Latin-0")
"ISO-8859-16", # Latin-10
))
# UTF-xx charset familly
UTF_CHARSET = {
"UTF-8": (8, None),
"UTF-16-LE": (16, LITTLE_ENDIAN),
"UTF-32LE": (32, LITTLE_ENDIAN),
"UTF-16-BE": (16, BIG_ENDIAN),
"UTF-32BE": (32, BIG_ENDIAN),
"UTF-16": (16, "BOM"),
"UTF-32": (32, "BOM"),
}
# UTF-xx BOM => charset with endian
UTF_BOM = {
16: {BOM_UTF16_LE: "UTF-16-LE", BOM_UTF16_BE: "UTF-16-BE"},
32: {BOM_UTF32_LE: "UTF-32LE", BOM_UTF32_BE: "UTF-32BE"},
}
# Suffix format: value is suffix (string)
SUFFIX_FORMAT = {
"C": {
8: {LITTLE_ENDIAN: "\0", BIG_ENDIAN: "\0"},
16: {LITTLE_ENDIAN: "\0\0", BIG_ENDIAN: "\0\0"},
32: {LITTLE_ENDIAN: "\0\0\0\0", BIG_ENDIAN: "\0\0\0\0"},
},
"UnixLine": {
8: {LITTLE_ENDIAN: "\n", BIG_ENDIAN: "\n"},
16: {LITTLE_ENDIAN: "\n\0", BIG_ENDIAN: "\0\n"},
32: {LITTLE_ENDIAN: "\n\0\0\0", BIG_ENDIAN: "\0\0\0\n"},
},
}
# Pascal format: value is the size of the prefix in bits
PASCAL_FORMATS = {
"Pascal8": 1,
"Pascal16": 2,
"Pascal32": 4
}
# Raw value: with prefix and suffix, not stripped,
# and not converted to Unicode
_raw_value = None
def __init__(self, parent, name, format, description=None,
strip=None, charset=None, nbytes=None, truncate=None):
Bytes.__init__(self, parent, name, 1, description)
# Is format valid?
assert format in self.VALID_FORMATS
# Store options
self._format = format
self._strip = strip
self._truncate = truncate
# Check charset and compute character size in bytes
# (or None when it's not possible to guess character size)
if not charset or charset in self.CHARSET_8BIT:
self._character_size = 1 # one byte per character
elif charset in self.UTF_CHARSET:
self._character_size = None
else:
raise FieldError("Invalid charset for %s: \"%s\"" %
(self.path, charset))
self._charset = charset
# It is a fixed string?
if nbytes is not None:
assert self._format == "fixed"
# Arbitrary limits, just to catch some bugs...
if not (1 <= nbytes <= 0xffff):
raise FieldError("Invalid string size for %s: %s" %
(self.path, nbytes))
self._content_size = nbytes # content length in bytes
self._size = nbytes * 8
self._content_offset = 0
else:
# Format with a suffix: Find the end of the string
if self._format in self.SUFFIX_FORMAT:
self._content_offset = 0
# Choose the suffix
suffix = self.suffix_str
# Find the suffix
length = self._parent.stream.searchBytesLength(
suffix, False, self.absolute_address)
if length is None:
raise FieldError("Unable to find end of string %s (format %s)!"
% (self.path, self._format))
if 1 < len(suffix):
# Fix length for little endian bug with UTF-xx charset:
# u"abc" -> "a\0b\0c\0\0\0" (UTF-16-LE)
# search returns length=5, whereas real lenght is 6
length = alignValue(length, len(suffix))
# Compute sizes
self._content_size = length # in bytes
self._size = (length + len(suffix)) * 8
# Format with a prefix: Read prefixed length in bytes
else:
assert self._format in self.PASCAL_FORMATS
# Get the prefix size
prefix_size = self.PASCAL_FORMATS[self._format]
self._content_offset = prefix_size
# Read the prefix and compute sizes
value = self._parent.stream.readBits(
self.absolute_address, prefix_size*8, self._parent.endian)
self._content_size = value # in bytes
self._size = (prefix_size + value) * 8
# For UTF-16 and UTF-32, choose the right charset using BOM
if self._charset in self.UTF_CHARSET:
# Charset requires a BOM?
bomsize, endian = self.UTF_CHARSET[self._charset]
if endian == "BOM":
# Read the BOM value
nbytes = bomsize // 8
bom = self._parent.stream.readBytes(self.absolute_address, nbytes)
# Choose right charset using the BOM
bom_endian = self.UTF_BOM[bomsize]
if bom not in bom_endian:
raise FieldError("String %s has invalid BOM (%s)!"
% (self.path, repr(bom)))
self._charset = bom_endian[bom]
self._content_size -= nbytes
self._content_offset += nbytes
# Compute length in character if possible
if self._character_size:
self._length = self._content_size // self._character_size
else:
self._length = None
@staticmethod
def staticSuffixStr(format, charset, endian):
if format not in GenericString.SUFFIX_FORMAT:
return ''
suffix = GenericString.SUFFIX_FORMAT[format]
if charset in GenericString.UTF_CHARSET:
suffix_size = GenericString.UTF_CHARSET[charset][0]
suffix = suffix[suffix_size]
else:
suffix = suffix[8]
return suffix[endian]
def _getSuffixStr(self):
return self.staticSuffixStr(
self._format, self._charset, self._parent.endian)
suffix_str = property(_getSuffixStr)
def _convertText(self, text):
if not self._charset:
# charset is still unknown: guess the charset
self._charset = guessBytesCharset(text, default=FALLBACK_CHARSET)
# Try to convert to Unicode
try:
return unicode(text, self._charset, "strict")
except UnicodeDecodeError, err:
pass
#--- Conversion error ---
# Fix truncated UTF-16 string like 'B\0e' (3 bytes)
# => Add missing nul byte: 'B\0e\0' (4 bytes)
if err.reason == "truncated data" \
and err.end == len(text) \
and self._charset == "UTF-16-LE":
try:
text = unicode(text+"\0", self._charset, "strict")
self.warning("Fix truncated %s string: add missing nul byte" % self._charset)
return text
except UnicodeDecodeError, err:
pass
# On error, use FALLBACK_CHARSET
self.warning(u"Unable to convert string to Unicode: %s" % err)
return unicode(text, FALLBACK_CHARSET, "strict")
def _guessCharset(self):
addr = self.absolute_address + self._content_offset * 8
bytes = self._parent.stream.readBytes(addr, self._content_size)
return guessBytesCharset(bytes, default=FALLBACK_CHARSET)
def createValue(self, human=True):
# Compress data address (in bits) and size (in bytes)
if human:
addr = self.absolute_address + self._content_offset * 8
size = self._content_size
else:
addr = self.absolute_address
size = self._size // 8
if size == 0:
# Empty string
return u""
# Read bytes in data stream
text = self._parent.stream.readBytes(addr, size)
# Don't transform data?
if not human:
return text
# Convert text to Unicode
text = self._convertText(text)
# Truncate
if self._truncate:
pos = text.find(self._truncate)
if 0 <= pos:
text = text[:pos]
# Strip string if needed
if self._strip:
if isinstance(self._strip, (str, unicode)):
text = text.strip(self._strip)
else:
text = text.strip()
assert isinstance(text, unicode)
return text
def createDisplay(self, human=True):
if not human:
if self._raw_value is None:
self._raw_value = GenericString.createValue(self, False)
value = makePrintable(self._raw_value, "ASCII", to_unicode=True)
elif self._charset:
value = makePrintable(self.value, "ISO-8859-1", to_unicode=True)
else:
value = self.value
if config.max_string_length < len(value):
# Truncate string if needed
value = "%s(...)" % value[:config.max_string_length]
if not self._charset or not human:
return makePrintable(value, "ASCII", quote='"', to_unicode=True)
else:
if value:
return '"%s"' % value.replace('"', '\\"')
else:
return _("(empty)")
def createRawDisplay(self):
return GenericString.createDisplay(self, human=False)
def _getLength(self):
if self._length is None:
self._length = len(self.value)
return self._length
length = property(_getLength, doc="String length in characters")
def _getFormat(self):
return self._format
format = property(_getFormat, doc="String format (eg. 'C')")
def _getCharset(self):
if not self._charset:
self._charset = self._guessCharset()
return self._charset
charset = property(_getCharset, doc="String charset (eg. 'ISO-8859-1')")
def _getContentSize(self):
return self._content_size
content_size = property(_getContentSize, doc="Content size in bytes")
def _getContentOffset(self):
return self._content_offset
content_offset = property(_getContentOffset, doc="Content offset in bytes")
def getFieldType(self):
info = self.charset
if self._strip:
if isinstance(self._strip, (str, unicode)):
info += ",strip=%s" % makePrintable(self._strip, "ASCII", quote="'")
else:
info += ",strip=True"
return "%s<%s>" % (Bytes.getFieldType(self), info)
def stringFactory(name, format, doc):
class NewString(GenericString):
__doc__ = doc
def __init__(self, parent, name, description=None,
strip=None, charset=None, truncate=None):
GenericString.__init__(self, parent, name, format, description,
strip=strip, charset=charset, truncate=truncate)
cls = NewString
cls.__name__ = name
return cls
# String which ends with nul byte ("\0")
CString = stringFactory("CString", "C",
r"""C string: string ending with nul byte.
See GenericString to get more information.""")
# Unix line of text: string which ends with "\n" (ASCII 0x0A)
UnixLine = stringFactory("UnixLine", "UnixLine",
r"""Unix line: string ending with "\n" (ASCII code 10).
See GenericString to get more information.""")
# String prefixed with length written in a 8-bit integer
PascalString8 = stringFactory("PascalString8", "Pascal8",
r"""Pascal string: string prefixed with 8-bit integer containing its length (endian depends on parent endian).
See GenericString to get more information.""")
# String prefixed with length written in a 16-bit integer (use parent endian)
PascalString16 = stringFactory("PascalString16", "Pascal16",
r"""Pascal string: string prefixed with 16-bit integer containing its length (endian depends on parent endian).
See GenericString to get more information.""")
# String prefixed with length written in a 32-bit integer (use parent endian)
PascalString32 = stringFactory("PascalString32", "Pascal32",
r"""Pascal string: string prefixed with 32-bit integer containing its length (endian depends on parent endian).
See GenericString to get more information.""")
class String(GenericString):
"""
String with fixed size (size in bytes).
See GenericString to get more information.
"""
static_size = staticmethod(lambda *args, **kw: args[1]*8)
def __init__(self, parent, name, nbytes, description=None,
strip=None, charset=None, truncate=None):
GenericString.__init__(self, parent, name, "fixed", description,
strip=strip, charset=charset, nbytes=nbytes, truncate=truncate)
String.__name__ = "FixedString"
libs/hachoir_core/field/sub_file.py
-72
View File
@@ -1,72 +0,0 @@
from hachoir_core.field import Bytes
from hachoir_core.tools import makePrintable, humanFilesize
from hachoir_core.stream import InputIOStream
class SubFile(Bytes):
"""
File stored in another file
"""
def __init__(self, parent, name, length, description=None,
parser=None, filename=None, mime_type=None, parser_class=None):
if filename:
if not isinstance(filename, unicode):
filename = makePrintable(filename, "ISO-8859-1")
if not description:
description = 'File "%s" (%s)' % (filename, humanFilesize(length))
Bytes.__init__(self, parent, name, length, description)
def createInputStream(cis, **args):
tags = args.setdefault("tags",[])
if parser_class:
tags.append(( "class", parser_class ))
if parser is not None:
tags.append(( "id", parser.PARSER_TAGS["id"] ))
if mime_type:
tags.append(( "mime", mime_type ))
if filename:
tags.append(( "filename", filename ))
return cis(**args)
self.setSubIStream(createInputStream)
class CompressedStream:
offset = 0
def __init__(self, stream, decompressor):
self.stream = stream
self.decompressor = decompressor(stream)
self._buffer = ''
def read(self, size):
d = self._buffer
data = [ d[:size] ]
size -= len(d)
if size > 0:
d = self.decompressor(size)
data.append(d[:size])
size -= len(d)
while size > 0:
n = 4096
if self.stream.size:
n = min(self.stream.size - self.offset, n)
if not n:
break
d = self.stream.read(self.offset, n)[1]
self.offset += 8 * len(d)
d = self.decompressor(size, d)
data.append(d[:size])
size -= len(d)
self._buffer = d[size+len(d):]
return ''.join(data)
def CompressedField(field, decompressor):
def createInputStream(cis, source=None, **args):
if field._parent:
stream = cis(source=source)
args.setdefault("tags", []).extend(stream.tags)
else:
stream = field.stream
input = CompressedStream(stream, decompressor)
if source is None:
source = "Compressed source: '%s' (offset=%s)" % (stream.source, field.absolute_address)
return InputIOStream(input, source=source, **args)
field.setSubIStream(createInputStream)
return field
libs/hachoir_core/field/timestamp.py
-86
View File
@@ -1,86 +0,0 @@
from hachoir_core.tools import (humanDatetime, humanDuration,
timestampUNIX, timestampMac32, timestampUUID60,
timestampWin64, durationWin64)
from hachoir_core.field import Bits, FieldSet
from datetime import datetime
class GenericTimestamp(Bits):
def __init__(self, parent, name, size, description=None):
Bits.__init__(self, parent, name, size, description)
def createDisplay(self):
return humanDatetime(self.value)
def createRawDisplay(self):
value = Bits.createValue(self)
return unicode(value)
def __nonzero__(self):
return Bits.createValue(self) != 0
def timestampFactory(cls_name, handler, size):
class Timestamp(GenericTimestamp):
def __init__(self, parent, name, description=None):
GenericTimestamp.__init__(self, parent, name, size, description)
def createValue(self):
value = Bits.createValue(self)
return handler(value)
cls = Timestamp
cls.__name__ = cls_name
return cls
TimestampUnix32 = timestampFactory("TimestampUnix32", timestampUNIX, 32)
TimestampUnix64 = timestampFactory("TimestampUnix64", timestampUNIX, 64)
TimestampMac32 = timestampFactory("TimestampUnix32", timestampMac32, 32)
TimestampUUID60 = timestampFactory("TimestampUUID60", timestampUUID60, 60)
TimestampWin64 = timestampFactory("TimestampWin64", timestampWin64, 64)
class TimeDateMSDOS32(FieldSet):
"""
32-bit MS-DOS timestamp (16-bit time, 16-bit date)
"""
static_size = 32
def createFields(self):
# TODO: Create type "MSDOS_Second" : value*2
yield Bits(self, "second", 5, "Second/2")
yield Bits(self, "minute", 6)
yield Bits(self, "hour", 5)
yield Bits(self, "day", 5)
yield Bits(self, "month", 4)
# TODO: Create type "MSDOS_Year" : value+1980
yield Bits(self, "year", 7, "Number of year after 1980")
def createValue(self):
return datetime(
1980+self["year"].value, self["month"].value, self["day"].value,
self["hour"].value, self["minute"].value, 2*self["second"].value)
def createDisplay(self):
return humanDatetime(self.value)
class DateTimeMSDOS32(TimeDateMSDOS32):
"""
32-bit MS-DOS timestamp (16-bit date, 16-bit time)
"""
def createFields(self):
yield Bits(self, "day", 5)
yield Bits(self, "month", 4)
yield Bits(self, "year", 7, "Number of year after 1980")
yield Bits(self, "second", 5, "Second/2")
yield Bits(self, "minute", 6)
yield Bits(self, "hour", 5)
class TimedeltaWin64(GenericTimestamp):
def __init__(self, parent, name, description=None):
GenericTimestamp.__init__(self, parent, name, 64, description)
def createDisplay(self):
return humanDuration(self.value)
def createValue(self):
value = Bits.createValue(self)
return durationWin64(value)
libs/hachoir_core/field/vector.py
-38
View File
@@ -1,38 +0,0 @@
from hachoir_core.field import Field, FieldSet, ParserError
class GenericVector(FieldSet):
def __init__(self, parent, name, nb_items, item_class, item_name="item", description=None):
# Sanity checks
assert issubclass(item_class, Field)
assert isinstance(item_class.static_size, (int, long))
if not(0 < nb_items):
raise ParserError('Unable to create empty vector "%s" in %s' \
% (name, parent.path))
size = nb_items * item_class.static_size
self.__nb_items = nb_items
self._item_class = item_class
self._item_name = item_name
FieldSet.__init__(self, parent, name, description, size=size)
def __len__(self):
return self.__nb_items
def createFields(self):
name = self._item_name + "[]"
parser = self._item_class
for index in xrange(len(self)):
yield parser(self, name)
class UserVector(GenericVector):
"""
To implement:
- item_name: name of a field without [] (eg. "color" becomes "color[0]"),
default value is "item"
- item_class: class of an item
"""
item_class = None
item_name = "item"
def __init__(self, parent, name, nb_items, description=None):
GenericVector.__init__(self, parent, name, nb_items, self.item_class, self.item_name, description)
libs/hachoir_core/i18n.py
-214
View File
@@ -1,214 +0,0 @@
# -*- coding: UTF-8 -*-
"""
Functions to manage internationalisation (i18n):
- initLocale(): setup locales and install Unicode compatible stdout and
stderr ;
- getTerminalCharset(): guess terminal charset ;
- gettext(text) translate a string to current language. The function always
returns Unicode string. You can also use the alias: _() ;
- ngettext(singular, plural, count): translate a sentence with singular and
plural form. The function always returns Unicode string.
WARNING: Loading this module indirectly calls initLocale() which sets
locale LC_ALL to ''. This is needed to get user preferred locale
settings.
"""
import hachoir_core.config as config
import hachoir_core
import locale
from os import path
import sys
from codecs import BOM_UTF8, BOM_UTF16_LE, BOM_UTF16_BE
def _getTerminalCharset():
"""
Function used by getTerminalCharset() to get terminal charset.
@see getTerminalCharset()
"""
# (1) Try locale.getpreferredencoding()
try:
charset = locale.getpreferredencoding()
if charset:
return charset
except (locale.Error, AttributeError):
pass
# (2) Try locale.nl_langinfo(CODESET)
try:
charset = locale.nl_langinfo(locale.CODESET)
if charset:
return charset
except (locale.Error, AttributeError):
pass
# (3) Try sys.stdout.encoding
if hasattr(sys.stdout, "encoding") and sys.stdout.encoding:
return sys.stdout.encoding
# (4) Otherwise, returns "ASCII"
return "ASCII"
def getTerminalCharset():
"""
Guess terminal charset using differents tests:
1. Try locale.getpreferredencoding()
2. Try locale.nl_langinfo(CODESET)
3. Try sys.stdout.encoding
4. Otherwise, returns "ASCII"
WARNING: Call initLocale() before calling this function.
"""
try:
return getTerminalCharset.value
except AttributeError:
getTerminalCharset.value = _getTerminalCharset()
return getTerminalCharset.value
class UnicodeStdout(object):
def __init__(self, old_device, charset):
self.device = old_device
self.charset = charset
def flush(self):
self.device.flush()
def write(self, text):
if isinstance(text, unicode):
text = text.encode(self.charset, 'replace')
self.device.write(text)
def writelines(self, lines):
for text in lines:
self.write(text)
def initLocale():
# Only initialize locale once
if initLocale.is_done:
return getTerminalCharset()
initLocale.is_done = True
# Setup locales
try:
locale.setlocale(locale.LC_ALL, "")
except (locale.Error, IOError):
pass
# Get the terminal charset
charset = getTerminalCharset()
# UnicodeStdout conflicts with the readline module
if config.unicode_stdout and ('readline' not in sys.modules):
# Replace stdout and stderr by unicode objet supporting unicode string
sys.stdout = UnicodeStdout(sys.stdout, charset)
sys.stderr = UnicodeStdout(sys.stderr, charset)
return charset
initLocale.is_done = False
def _dummy_gettext(text):
return unicode(text)
def _dummy_ngettext(singular, plural, count):
if 1 < abs(count) or not count:
return unicode(plural)
else:
return unicode(singular)
def _initGettext():
charset = initLocale()
# Try to load gettext module
if config.use_i18n:
try:
import gettext
ok = True
except ImportError:
ok = False
else:
ok = False
# gettext is not available or not needed: use dummy gettext functions
if not ok:
return (_dummy_gettext, _dummy_ngettext)
# Gettext variables
package = hachoir_core.PACKAGE
locale_dir = path.join(path.dirname(__file__), "..", "locale")
# Initialize gettext module
gettext.bindtextdomain(package, locale_dir)
gettext.textdomain(package)
translate = gettext.gettext
ngettext = gettext.ngettext
# TODO: translate_unicode lambda function really sucks!
# => find native function to do that
unicode_gettext = lambda text: \
unicode(translate(text), charset)
unicode_ngettext = lambda singular, plural, count: \
unicode(ngettext(singular, plural, count), charset)
return (unicode_gettext, unicode_ngettext)
UTF_BOMS = (
(BOM_UTF8, "UTF-8"),
(BOM_UTF16_LE, "UTF-16-LE"),
(BOM_UTF16_BE, "UTF-16-BE"),
)
# Set of valid characters for specific charset
CHARSET_CHARACTERS = (
# U+00E0: LATIN SMALL LETTER A WITH GRAVE
(set(u"©®éêè\xE0ç".encode("ISO-8859-1")), "ISO-8859-1"),
(set(u"©®éêè\xE0ç€".encode("ISO-8859-15")), "ISO-8859-15"),
(set(u"©®".encode("MacRoman")), "MacRoman"),
(set(u"εδηιθκμοΡσςυΈί".encode("ISO-8859-7")), "ISO-8859-7"),
)
def guessBytesCharset(bytes, default=None):
r"""
>>> guessBytesCharset("abc")
'ASCII'
>>> guessBytesCharset("\xEF\xBB\xBFabc")
'UTF-8'
>>> guessBytesCharset("abc\xC3\xA9")
'UTF-8'
>>> guessBytesCharset("File written by Adobe Photoshop\xA8 4.0\0")
'MacRoman'
>>> guessBytesCharset("\xE9l\xE9phant")
'ISO-8859-1'
>>> guessBytesCharset("100 \xA4")
'ISO-8859-15'
>>> guessBytesCharset('Word \xb8\xea\xe4\xef\xf3\xe7 - Microsoft Outlook 97 - \xd1\xf5\xe8\xec\xdf\xf3\xe5\xe9\xf2 e-mail')
'ISO-8859-7'
"""
# Check for UTF BOM
for bom_bytes, charset in UTF_BOMS:
if bytes.startswith(bom_bytes):
return charset
# Pure ASCII?
try:
text = unicode(bytes, 'ASCII', 'strict')
return 'ASCII'
except UnicodeDecodeError:
pass
# Valid UTF-8?
try:
text = unicode(bytes, 'UTF-8', 'strict')
return 'UTF-8'
except UnicodeDecodeError:
pass
# Create a set of non-ASCII characters
non_ascii_set = set( byte for byte in bytes if ord(byte) >= 128 )
for characters, charset in CHARSET_CHARACTERS:
if characters.issuperset(non_ascii_set):
return charset
return default
# Initialize _(), gettext() and ngettext() functions
gettext, ngettext = _initGettext()
_ = gettext
libs/hachoir_core/iso639.py
-558
View File
@@ -1,558 +0,0 @@
# -*- coding: utf-8 -*-
"""
ISO639-2 standart: the module only contains the dictionary ISO639_2
which maps a language code in three letters (eg. "fre") to a language
name in english (eg. "French").
"""
# ISO-639, the list comes from:
# http://www.loc.gov/standards/iso639-2/php/English_list.php
_ISO639 = (
(u"Abkhazian", "abk", "ab"),
(u"Achinese", "ace", None),
(u"Acoli", "ach", None),
(u"Adangme", "ada", None),
(u"Adygei", "ady", None),
(u"Adyghe", "ady", None),
(u"Afar", "aar", "aa"),
(u"Afrihili", "afh", None),
(u"Afrikaans", "afr", "af"),
(u"Afro-Asiatic (Other)", "afa", None),
(u"Ainu", "ain", None),
(u"Akan", "aka", "ak"),
(u"Akkadian", "akk", None),
(u"Albanian", "alb/sqi", "sq"),
(u"Alemani", "gsw", None),
(u"Aleut", "ale", None),
(u"Algonquian languages", "alg", None),
(u"Altaic (Other)", "tut", None),
(u"Amharic", "amh", "am"),
(u"Angika", "anp", None),
(u"Apache languages", "apa", None),
(u"Arabic", "ara", "ar"),
(u"Aragonese", "arg", "an"),
(u"Aramaic", "arc", None),
(u"Arapaho", "arp", None),
(u"Araucanian", "arn", None),
(u"Arawak", "arw", None),
(u"Armenian", "arm/hye", "hy"),
(u"Aromanian", "rup", None),
(u"Artificial (Other)", "art", None),
(u"Arumanian", "rup", None),
(u"Assamese", "asm", "as"),
(u"Asturian", "ast", None),
(u"Athapascan languages", "ath", None),
(u"Australian languages", "aus", None),
(u"Austronesian (Other)", "map", None),
(u"Avaric", "ava", "av"),
(u"Avestan", "ave", "ae"),
(u"Awadhi", "awa", None),
(u"Aymara", "aym", "ay"),
(u"Azerbaijani", "aze", "az"),
(u"Bable", "ast", None),
(u"Balinese", "ban", None),
(u"Baltic (Other)", "bat", None),
(u"Baluchi", "bal", None),
(u"Bambara", "bam", "bm"),
(u"Bamileke languages", "bai", None),
(u"Banda", "bad", None),
(u"Bantu (Other)", "bnt", None),
(u"Basa", "bas", None),
(u"Bashkir", "bak", "ba"),
(u"Basque", "baq/eus", "eu"),
(u"Batak (Indonesia)", "btk", None),
(u"Beja", "bej", None),
(u"Belarusian", "bel", "be"),
(u"Bemba", "bem", None),
(u"Bengali", "ben", "bn"),
(u"Berber (Other)", "ber", None),
(u"Bhojpuri", "bho", None),
(u"Bihari", "bih", "bh"),
(u"Bikol", "bik", None),
(u"Bilin", "byn", None),
(u"Bini", "bin", None),
(u"Bislama", "bis", "bi"),
(u"Blin", "byn", None),
(u"Bokmål, Norwegian", "nob", "nb"),
(u"Bosnian", "bos", "bs"),
(u"Braj", "bra", None),
(u"Breton", "bre", "br"),
(u"Buginese", "bug", None),
(u"Bulgarian", "bul", "bg"),
(u"Buriat", "bua", None),
(u"Burmese", "bur/mya", "my"),
(u"Caddo", "cad", None),
(u"Carib", "car", None),
(u"Castilian", "spa", "es"),
(u"Catalan", "cat", "ca"),
(u"Caucasian (Other)", "cau", None),
(u"Cebuano", "ceb", None),
(u"Celtic (Other)", "cel", None),
(u"Central American Indian (Other)", "cai", None),
(u"Chagatai", "chg", None),
(u"Chamic languages", "cmc", None),
(u"Chamorro", "cha", "ch"),
(u"Chechen", "che", "ce"),
(u"Cherokee", "chr", None),
(u"Chewa", "nya", "ny"),
(u"Cheyenne", "chy", None),
(u"Chibcha", "chb", None),
(u"Chichewa", "nya", "ny"),
(u"Chinese", "chi/zho", "zh"),
(u"Chinook jargon", "chn", None),
(u"Chipewyan", "chp", None),
(u"Choctaw", "cho", None),
(u"Chuang", "zha", "za"),
(u"Church Slavic", "chu", "cu"),
(u"Church Slavonic", "chu", "cu"),
(u"Chuukese", "chk", None),
(u"Chuvash", "chv", "cv"),
(u"Classical Nepal Bhasa", "nwc", None),
(u"Classical Newari", "nwc", None),
(u"Coptic", "cop", None),
(u"Cornish", "cor", "kw"),
(u"Corsican", "cos", "co"),
(u"Cree", "cre", "cr"),
(u"Creek", "mus", None),
(u"Creoles and pidgins (Other)", "crp", None),
(u"Creoles and pidgins, English based (Other)", "cpe", None),
(u"Creoles and pidgins, French-based (Other)", "cpf", None),
(u"Creoles and pidgins, Portuguese-based (Other)", "cpp", None),
(u"Crimean Tatar", "crh", None),
(u"Crimean Turkish", "crh", None),
(u"Croatian", "scr/hrv", "hr"),
(u"Cushitic (Other)", "cus", None),
(u"Czech", "cze/ces", "cs"),
(u"Dakota", "dak", None),
(u"Danish", "dan", "da"),
(u"Dargwa", "dar", None),
(u"Dayak", "day", None),
(u"Delaware", "del", None),
(u"Dhivehi", "div", "dv"),
(u"Dimili", "zza", None),
(u"Dimli", "zza", None),
(u"Dinka", "din", None),
(u"Divehi", "div", "dv"),
(u"Dogri", "doi", None),
(u"Dogrib", "dgr", None),
(u"Dravidian (Other)", "dra", None),
(u"Duala", "dua", None),
(u"Dutch", "dut/nld", "nl"),
(u"Dutch, Middle (ca.1050-1350)", "dum", None),
(u"Dyula", "dyu", None),
(u"Dzongkha", "dzo", "dz"),
(u"Eastern Frisian", "frs", None),
(u"Efik", "efi", None),
(u"Egyptian (Ancient)", "egy", None),
(u"Ekajuk", "eka", None),
(u"Elamite", "elx", None),
(u"English", "eng", "en"),
(u"English, Middle (1100-1500)", "enm", None),
(u"English, Old (ca.450-1100)", "ang", None),
(u"Erzya", "myv", None),
(u"Esperanto", "epo", "eo"),
(u"Estonian", "est", "et"),
(u"Ewe", "ewe", "ee"),
(u"Ewondo", "ewo", None),
(u"Fang", "fan", None),
(u"Fanti", "fat", None),
(u"Faroese", "fao", "fo"),
(u"Fijian", "fij", "fj"),
(u"Filipino", "fil", None),
(u"Finnish", "fin", "fi"),
(u"Finno-Ugrian (Other)", "fiu", None),
(u"Flemish", "dut/nld", "nl"),
(u"Fon", "fon", None),
(u"French", "fre/fra", "fr"),
(u"French, Middle (ca.1400-1600)", "frm", None),
(u"French, Old (842-ca.1400)", "fro", None),
(u"Friulian", "fur", None),
(u"Fulah", "ful", "ff"),
(u"Ga", "gaa", None),
(u"Gaelic", "gla", "gd"),
(u"Galician", "glg", "gl"),
(u"Ganda", "lug", "lg"),
(u"Gayo", "gay", None),
(u"Gbaya", "gba", None),
(u"Geez", "gez", None),
(u"Georgian", "geo/kat", "ka"),
(u"German", "ger/deu", "de"),
(u"German, Low", "nds", None),
(u"German, Middle High (ca.1050-1500)", "gmh", None),
(u"German, Old High (ca.750-1050)", "goh", None),
(u"Germanic (Other)", "gem", None),
(u"Gikuyu", "kik", "ki"),
(u"Gilbertese", "gil", None),
(u"Gondi", "gon", None),
(u"Gorontalo", "gor", None),
(u"Gothic", "got", None),
(u"Grebo", "grb", None),
(u"Greek, Ancient (to 1453)", "grc", None),
(u"Greek, Modern (1453-)", "gre/ell", "el"),
(u"Greenlandic", "kal", "kl"),
(u"Guarani", "grn", "gn"),
(u"Gujarati", "guj", "gu"),
(u"Gwich´in", "gwi", None),
(u"Haida", "hai", None),
(u"Haitian", "hat", "ht"),
(u"Haitian Creole", "hat", "ht"),
(u"Hausa", "hau", "ha"),
(u"Hawaiian", "haw", None),
(u"Hebrew", "heb", "he"),
(u"Herero", "her", "hz"),
(u"Hiligaynon", "hil", None),
(u"Himachali", "him", None),
(u"Hindi", "hin", "hi"),
(u"Hiri Motu", "hmo", "ho"),
(u"Hittite", "hit", None),
(u"Hmong", "hmn", None),
(u"Hungarian", "hun", "hu"),
(u"Hupa", "hup", None),
(u"Iban", "iba", None),
(u"Icelandic", "ice/isl", "is"),
(u"Ido", "ido", "io"),
(u"Igbo", "ibo", "ig"),
(u"Ijo", "ijo", None),
(u"Iloko", "ilo", None),
(u"Inari Sami", "smn", None),
(u"Indic (Other)", "inc", None),
(u"Indo-European (Other)", "ine", None),
(u"Indonesian", "ind", "id"),
(u"Ingush", "inh", None),
(u"Interlingua", "ina", "ia"),
(u"Interlingue", "ile", "ie"),
(u"Inuktitut", "iku", "iu"),
(u"Inupiaq", "ipk", "ik"),
(u"Iranian (Other)", "ira", None),
(u"Irish", "gle", "ga"),
(u"Irish, Middle (900-1200)", "mga", None),
(u"Irish, Old (to 900)", "sga", None),
(u"Iroquoian languages", "iro", None),
(u"Italian", "ita", "it"),
(u"Japanese", "jpn", "ja"),
(u"Javanese", "jav", "jv"),
(u"Judeo-Arabic", "jrb", None),
(u"Judeo-Persian", "jpr", None),
(u"Kabardian", "kbd", None),
(u"Kabyle", "kab", None),
(u"Kachin", "kac", None),
(u"Kalaallisut", "kal", "kl"),
(u"Kalmyk", "xal", None),
(u"Kamba", "kam", None),
(u"Kannada", "kan", "kn"),
(u"Kanuri", "kau", "kr"),
(u"Kara-Kalpak", "kaa", None),
(u"Karachay-Balkar", "krc", None),
(u"Karelian", "krl", None),
(u"Karen", "kar", None),
(u"Kashmiri", "kas", "ks"),
(u"Kashubian", "csb", None),
(u"Kawi", "kaw", None),
(u"Kazakh", "kaz", "kk"),
(u"Khasi", "kha", None),
(u"Khmer", "khm", "km"),
(u"Khoisan (Other)", "khi", None),
(u"Khotanese", "kho", None),
(u"Kikuyu", "kik", "ki"),
(u"Kimbundu", "kmb", None),
(u"Kinyarwanda", "kin", "rw"),
(u"Kirdki", "zza", None),
(u"Kirghiz", "kir", "ky"),
(u"Kirmanjki", "zza", None),
(u"Klingon", "tlh", None),
(u"Komi", "kom", "kv"),
(u"Kongo", "kon", "kg"),
(u"Konkani", "kok", None),
(u"Korean", "kor", "ko"),
(u"Kosraean", "kos", None),
(u"Kpelle", "kpe", None),
(u"Kru", "kro", None),
(u"Kuanyama", "kua", "kj"),
(u"Kumyk", "kum", None),
(u"Kurdish", "kur", "ku"),
(u"Kurukh", "kru", None),
(u"Kutenai", "kut", None),
(u"Kwanyama", "kua", "kj"),
(u"Ladino", "lad", None),
(u"Lahnda", "lah", None),
(u"Lamba", "lam", None),
(u"Lao", "lao", "lo"),
(u"Latin", "lat", "la"),
(u"Latvian", "lav", "lv"),
(u"Letzeburgesch", "ltz", "lb"),
(u"Lezghian", "lez", None),
(u"Limburgan", "lim", "li"),
(u"Limburger", "lim", "li"),
(u"Limburgish", "lim", "li"),
(u"Lingala", "lin", "ln"),
(u"Lithuanian", "lit", "lt"),
(u"Lojban", "jbo", None),
(u"Low German", "nds", None),
(u"Low Saxon", "nds", None),
(u"Lower Sorbian", "dsb", None),
(u"Lozi", "loz", None),
(u"Luba-Katanga", "lub", "lu"),
(u"Luba-Lulua", "lua", None),
(u"Luiseno", "lui", None),
(u"Lule Sami", "smj", None),
(u"Lunda", "lun", None),
(u"Luo (Kenya and Tanzania)", "luo", None),
(u"Lushai", "lus", None),
(u"Luxembourgish", "ltz", "lb"),
(u"Macedo-Romanian", "rup", None),
(u"Macedonian", "mac/mkd", "mk"),
(u"Madurese", "mad", None),
(u"Magahi", "mag", None),
(u"Maithili", "mai", None),
(u"Makasar", "mak", None),
(u"Malagasy", "mlg", "mg"),
(u"Malay", "may/msa", "ms"),
(u"Malayalam", "mal", "ml"),
(u"Maldivian", "div", "dv"),
(u"Maltese", "mlt", "mt"),
(u"Manchu", "mnc", None),
(u"Mandar", "mdr", None),
(u"Mandingo", "man", None),
(u"Manipuri", "mni", None),
(u"Manobo languages", "mno", None),
(u"Manx", "glv", "gv"),
(u"Maori", "mao/mri", "mi"),
(u"Marathi", "mar", "mr"),
(u"Mari", "chm", None),
(u"Marshallese", "mah", "mh"),
(u"Marwari", "mwr", None),
(u"Masai", "mas", None),
(u"Mayan languages", "myn", None),
(u"Mende", "men", None),
(u"Mi'kmaq", "mic", None),
(u"Micmac", "mic", None),
(u"Minangkabau", "min", None),
(u"Mirandese", "mwl", None),
(u"Miscellaneous languages", "mis", None),
(u"Mohawk", "moh", None),
(u"Moksha", "mdf", None),
(u"Moldavian", "mol", "mo"),
(u"Mon-Khmer (Other)", "mkh", None),
(u"Mongo", "lol", None),
(u"Mongolian", "mon", "mn"),
(u"Mossi", "mos", None),
(u"Multiple languages", "mul", None),
(u"Munda languages", "mun", None),
(u"N'Ko", "nqo", None),
(u"Nahuatl", "nah", None),
(u"Nauru", "nau", "na"),
(u"Navaho", "nav", "nv"),
(u"Navajo", "nav", "nv"),
(u"Ndebele, North", "nde", "nd"),
(u"Ndebele, South", "nbl", "nr"),
(u"Ndonga", "ndo", "ng"),
(u"Neapolitan", "nap", None),
(u"Nepal Bhasa", "new", None),
(u"Nepali", "nep", "ne"),
(u"Newari", "new", None),
(u"Nias", "nia", None),
(u"Niger-Kordofanian (Other)", "nic", None),
(u"Nilo-Saharan (Other)", "ssa", None),
(u"Niuean", "niu", None),
(u"No linguistic content", "zxx", None),
(u"Nogai", "nog", None),
(u"Norse, Old", "non", None),
(u"North American Indian", "nai", None),
(u"North Ndebele", "nde", "nd"),
(u"Northern Frisian", "frr", None),
(u"Northern Sami", "sme", "se"),
(u"Northern Sotho", "nso", None),
(u"Norwegian", "nor", "no"),
(u"Norwegian Bokmål", "nob", "nb"),
(u"Norwegian Nynorsk", "nno", "nn"),
(u"Nubian languages", "nub", None),
(u"Nyamwezi", "nym", None),
(u"Nyanja", "nya", "ny"),
(u"Nyankole", "nyn", None),
(u"Nynorsk, Norwegian", "nno", "nn"),
(u"Nyoro", "nyo", None),
(u"Nzima", "nzi", None),
(u"Occitan (post 1500)", "oci", "oc"),
(u"Oirat", "xal", None),
(u"Ojibwa", "oji", "oj"),
(u"Old Bulgarian", "chu", "cu"),
(u"Old Church Slavonic", "chu", "cu"),
(u"Old Newari", "nwc", None),
(u"Old Slavonic", "chu", "cu"),
(u"Oriya", "ori", "or"),
(u"Oromo", "orm", "om"),
(u"Osage", "osa", None),
(u"Ossetian", "oss", "os"),
(u"Ossetic", "oss", "os"),
(u"Otomian languages", "oto", None),
(u"Pahlavi", "pal", None),
(u"Palauan", "pau", None),
(u"Pali", "pli", "pi"),
(u"Pampanga", "pam", None),
(u"Pangasinan", "pag", None),
(u"Panjabi", "pan", "pa"),
(u"Papiamento", "pap", None),
(u"Papuan (Other)", "paa", None),
(u"Pedi", "nso", None),
(u"Persian", "per/fas", "fa"),
(u"Persian, Old (ca.600-400 B.C.)", "peo", None),
(u"Philippine (Other)", "phi", None),
(u"Phoenician", "phn", None),
(u"Pilipino", "fil", None),
(u"Pohnpeian", "pon", None),
(u"Polish", "pol", "pl"),
(u"Portuguese", "por", "pt"),
(u"Prakrit languages", "pra", None),
(u"Provençal", "oci", "oc"),
(u"Provençal, Old (to 1500)", "pro", None),
(u"Punjabi", "pan", "pa"),
(u"Pushto", "pus", "ps"),
(u"Quechua", "que", "qu"),
(u"Raeto-Romance", "roh", "rm"),
(u"Rajasthani", "raj", None),
(u"Rapanui", "rap", None),
(u"Rarotongan", "rar", None),
(u"Reserved for local use", "qaa/qtz", None),
(u"Romance (Other)", "roa", None),
(u"Romanian", "rum/ron", "ro"),
(u"Romany", "rom", None),
(u"Rundi", "run", "rn"),
(u"Russian", "rus", "ru"),
(u"Salishan languages", "sal", None),
(u"Samaritan Aramaic", "sam", None),
(u"Sami languages (Other)", "smi", None),
(u"Samoan", "smo", "sm"),
(u"Sandawe", "sad", None),
(u"Sango", "sag", "sg"),
(u"Sanskrit", "san", "sa"),
(u"Santali", "sat", None),
(u"Sardinian", "srd", "sc"),
(u"Sasak", "sas", None),
(u"Saxon, Low", "nds", None),
(u"Scots", "sco", None),
(u"Scottish Gaelic", "gla", "gd"),
(u"Selkup", "sel", None),
(u"Semitic (Other)", "sem", None),
(u"Sepedi", "nso", None),
(u"Serbian", "scc/srp", "sr"),
(u"Serer", "srr", None),
(u"Shan", "shn", None),
(u"Shona", "sna", "sn"),
(u"Sichuan Yi", "iii", "ii"),
(u"Sicilian", "scn", None),
(u"Sidamo", "sid", None),
(u"Sign Languages", "sgn", None),
(u"Siksika", "bla", None),
(u"Sindhi", "snd", "sd"),
(u"Sinhala", "sin", "si"),
(u"Sinhalese", "sin", "si"),
(u"Sino-Tibetan (Other)", "sit", None),
(u"Siouan languages", "sio", None),
(u"Skolt Sami", "sms", None),
(u"Slave (Athapascan)", "den", None),
(u"Slavic (Other)", "sla", None),
(u"Slovak", "slo/slk", "sk"),
(u"Slovenian", "slv", "sl"),
(u"Sogdian", "sog", None),
(u"Somali", "som", "so"),
(u"Songhai", "son", None),
(u"Soninke", "snk", None),
(u"Sorbian languages", "wen", None),
(u"Sotho, Northern", "nso", None),
(u"Sotho, Southern", "sot", "st"),
(u"South American Indian (Other)", "sai", None),
(u"South Ndebele", "nbl", "nr"),
(u"Southern Altai", "alt", None),
(u"Southern Sami", "sma", None),
(u"Spanish", "spa", "es"),
(u"Sranan Togo", "srn", None),
(u"Sukuma", "suk", None),
(u"Sumerian", "sux", None),
(u"Sundanese", "sun", "su"),
(u"Susu", "sus", None),
(u"Swahili", "swa", "sw"),
(u"Swati", "ssw", "ss"),
(u"Swedish", "swe", "sv"),
(u"Swiss German", "gsw", None),
(u"Syriac", "syr", None),
(u"Tagalog", "tgl", "tl"),
(u"Tahitian", "tah", "ty"),
(u"Tai (Other)", "tai", None),
(u"Tajik", "tgk", "tg"),
(u"Tamashek", "tmh", None),
(u"Tamil", "tam", "ta"),
(u"Tatar", "tat", "tt"),
(u"Telugu", "tel", "te"),
(u"Tereno", "ter", None),
(u"Tetum", "tet", None),
(u"Thai", "tha", "th"),
(u"Tibetan", "tib/bod", "bo"),
(u"Tigre", "tig", None),
(u"Tigrinya", "tir", "ti"),
(u"Timne", "tem", None),
(u"Tiv", "tiv", None),
(u"tlhIngan-Hol", "tlh", None),
(u"Tlingit", "tli", None),
(u"Tok Pisin", "tpi", None),
(u"Tokelau", "tkl", None),
(u"Tonga (Nyasa)", "tog", None),
(u"Tonga (Tonga Islands)", "ton", "to"),
(u"Tsimshian", "tsi", None),
(u"Tsonga", "tso", "ts"),
(u"Tswana", "tsn", "tn"),
(u"Tumbuka", "tum", None),
(u"Tupi languages", "tup", None),
(u"Turkish", "tur", "tr"),
(u"Turkish, Ottoman (1500-1928)", "ota", None),
(u"Turkmen", "tuk", "tk"),
(u"Tuvalu", "tvl", None),
(u"Tuvinian", "tyv", None),
(u"Twi", "twi", "tw"),
(u"Udmurt", "udm", None),
(u"Ugaritic", "uga", None),
(u"Uighur", "uig", "ug"),
(u"Ukrainian", "ukr", "uk"),
(u"Umbundu", "umb", None),
(u"Undetermined", "und", None),
(u"Upper Sorbian", "hsb", None),
(u"Urdu", "urd", "ur"),
(u"Uyghur", "uig", "ug"),
(u"Uzbek", "uzb", "uz"),
(u"Vai", "vai", None),
(u"Valencian", "cat", "ca"),
(u"Venda", "ven", "ve"),
(u"Vietnamese", "vie", "vi"),
(u"Volapük", "vol", "vo"),
(u"Votic", "vot", None),
(u"Wakashan languages", "wak", None),
(u"Walamo", "wal", None),
(u"Walloon", "wln", "wa"),
(u"Waray", "war", None),
(u"Washo", "was", None),
(u"Welsh", "wel/cym", "cy"),
(u"Western Frisian", "fry", "fy"),
(u"Wolof", "wol", "wo"),
(u"Xhosa", "xho", "xh"),
(u"Yakut", "sah", None),
(u"Yao", "yao", None),
(u"Yapese", "yap", None),
(u"Yiddish", "yid", "yi"),
(u"Yoruba", "yor", "yo"),
(u"Yupik languages", "ypk", None),
(u"Zande", "znd", None),
(u"Zapotec", "zap", None),
(u"Zaza", "zza", None),
(u"Zazaki", "zza", None),
(u"Zenaga", "zen", None),
(u"Zhuang", "zha", "za"),
(u"Zulu", "zul", "zu"),
(u"Zuni", "zun", None),
)
# Bibliographic ISO-639-2 form (eg. "fre" => "French")
ISO639_2 = {}
for line in _ISO639:
for key in line[1].split("/"):
ISO639_2[key] = line[0]
del _ISO639
libs/hachoir_core/language.py
-23
View File
@@ -1,23 +0,0 @@
from hachoir_core.iso639 import ISO639_2
class Language:
def __init__(self, code):
code = str(code)
if code not in ISO639_2:
raise ValueError("Invalid language code: %r" % code)
self.code = code
def __cmp__(self, other):
if other.__class__ != Language:
return 1
return cmp(self.code, other.code)
def __unicode__(self):
return ISO639_2[self.code]
def __str__(self):
return self.__unicode__()
def __repr__(self):
return "<Language '%s', code=%r>" % (unicode(self), self.code)
libs/hachoir_core/log.py
-144
View File
@@ -1,144 +0,0 @@
import os, sys, time
import hachoir_core.config as config
from hachoir_core.i18n import _
class Log:
LOG_INFO = 0
LOG_WARN = 1
LOG_ERROR = 2
level_name = {
LOG_WARN: "[warn]",
LOG_ERROR: "[err!]",
LOG_INFO: "[info]"
}
def __init__(self):
self.__buffer = {}
self.__file = None
self.use_print = True
self.use_buffer = False
self.on_new_message = None # Prototype: def func(level, prefix, text, context)
def shutdown(self):
if self.__file:
self._writeIntoFile(_("Stop Hachoir"))
def setFilename(self, filename, append=True):
"""
Use a file to store all messages. The
UTF-8 encoding will be used. Write an informative
message if the file can't be created.
@param filename: C{L{string}}
"""
# Look if file already exists or not
filename = os.path.expanduser(filename)
filename = os.path.realpath(filename)
append = os.access(filename, os.F_OK)
# Create log file (or open it in append mode, if it already exists)
try:
import codecs
if append:
self.__file = codecs.open(filename, "a", "utf-8")
else:
self.__file = codecs.open(filename, "w", "utf-8")
self._writeIntoFile(_("Starting Hachoir"))
except IOError, err:
if err.errno == 2:
self.__file = None
self.info(_("[Log] setFilename(%s) fails: no such file") % filename)
else:
raise
def _writeIntoFile(self, message):
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
self.__file.write(u"%s - %s\n" % (timestamp, message))
self.__file.flush()
def newMessage(self, level, text, ctxt=None):
"""
Write a new message : append it in the buffer,
display it to the screen (if needed), and write
it in the log file (if needed).
@param level: Message level.
@type level: C{int}
@param text: Message content.
@type text: C{str}
@param ctxt: The caller instance.
"""
if level < self.LOG_ERROR and config.quiet or \
level <= self.LOG_INFO and not config.verbose:
return
if config.debug:
from hachoir_core.error import getBacktrace
backtrace = getBacktrace(None)
if backtrace:
text += "\n\n" + backtrace
_text = text
if hasattr(ctxt, "_logger"):
_ctxt = ctxt._logger()
if _ctxt is not None:
text = "[%s] %s" % (_ctxt, text)
# Add message to log buffer
if self.use_buffer:
if not self.__buffer.has_key(level):
self.__buffer[level] = [text]
else:
self.__buffer[level].append(text)
# Add prefix
prefix = self.level_name.get(level, "[info]")
# Display on stdout (if used)
if self.use_print:
sys.stdout.flush()
sys.stderr.write("%s %s\n" % (prefix, text))
sys.stderr.flush()
# Write into outfile (if used)
if self.__file:
self._writeIntoFile("%s %s" % (prefix, text))
# Use callback (if used)
if self.on_new_message:
self.on_new_message (level, prefix, _text, ctxt)
def info(self, text):
"""
New informative message.
@type text: C{str}
"""
self.newMessage(Log.LOG_INFO, text)
def warning(self, text):
"""
New warning message.
@type text: C{str}
"""
self.newMessage(Log.LOG_WARN, text)
def error(self, text):
"""
New error message.
@type text: C{str}
"""
self.newMessage(Log.LOG_ERROR, text)
log = Log()
class Logger(object):
def _logger(self):
return "<%s>" % self.__class__.__name__
def info(self, text):
log.newMessage(Log.LOG_INFO, text, self)
def warning(self, text):
log.newMessage(Log.LOG_WARN, text, self)
def error(self, text):
log.newMessage(Log.LOG_ERROR, text, self)
libs/hachoir_core/memory.py
-99
View File
@@ -1,99 +0,0 @@
import gc
#---- Default implementation when resource is missing ----------------------
PAGE_SIZE = 4096
def getMemoryLimit():
"""
Get current memory limit in bytes.
Return None on error.
"""
return None
def setMemoryLimit(max_mem):
"""
Set memory limit in bytes.
Use value 'None' to disable memory limit.
Return True if limit is set, False on error.
"""
return False
def getMemorySize():
"""
Read currenet process memory size: size of available virtual memory.
This value is NOT the real memory usage.
This function only works on Linux (use /proc/self/statm file).
"""
try:
statm = open('/proc/self/statm').readline().split()
except IOError:
return None
return int(statm[0]) * PAGE_SIZE
def clearCaches():
"""
Try to clear all caches: call gc.collect() (Python garbage collector).
"""
gc.collect()
#import re; re.purge()
try:
#---- 'resource' implementation ---------------------------------------------
from resource import getpagesize, getrlimit, setrlimit, RLIMIT_AS
PAGE_SIZE = getpagesize()
def getMemoryLimit():
try:
limit = getrlimit(RLIMIT_AS)[0]
if 0 < limit:
limit *= PAGE_SIZE
return limit
except ValueError:
return None
def setMemoryLimit(max_mem):
if max_mem is None:
max_mem = -1
try:
setrlimit(RLIMIT_AS, (max_mem, -1))
return True
except ValueError:
return False
except ImportError:
pass
def limitedMemory(limit, func, *args, **kw):
"""
Limit memory grow when calling func(*args, **kw):
restrict memory grow to 'limit' bytes.
Use try/except MemoryError to catch the error.
"""
# First step: clear cache to gain memory
clearCaches()
# Get total program size
max_rss = getMemorySize()
if max_rss is not None:
# Get old limit and then set our new memory limit
old_limit = getMemoryLimit()
limit = max_rss + limit
limited = setMemoryLimit(limit)
else:
limited = False
try:
# Call function
return func(*args, **kw)
finally:
# and unset our memory limit
if limited:
setMemoryLimit(old_limit)
# After calling the function: clear all caches
clearCaches()
libs/hachoir_core/profiler.py
-31
View File
@@ -1,31 +0,0 @@
from hotshot import Profile
from hotshot.stats import load as loadStats
from os import unlink
def runProfiler(func, args=tuple(), kw={}, verbose=True, nb_func=25, sort_by=('cumulative', 'calls')):
profile_filename = "/tmp/profiler"
prof = Profile(profile_filename)
try:
if verbose:
print "[+] Run profiler"
result = prof.runcall(func, *args, **kw)
prof.close()
if verbose:
print "[+] Stop profiler"
print "[+] Process data..."
stat = loadStats(profile_filename)
if verbose:
print "[+] Strip..."
stat.strip_dirs()
if verbose:
print "[+] Sort data..."
stat.sort_stats(*sort_by)
if verbose:
print
print "[+] Display statistics"
print
stat.print_stats(nb_func)
return result
finally:
unlink(profile_filename)
libs/hachoir_core/stream/__init__.py
-11
View File
@@ -1,11 +0,0 @@
from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
from hachoir_core.stream.stream import StreamError
from hachoir_core.stream.input import (
InputStreamError,
InputStream, InputIOStream, StringInputStream,
InputSubStream, InputFieldStream,
FragmentedStream, ConcatStream)
from hachoir_core.stream.input_helper import FileInputStream, guessStreamCharset
from hachoir_core.stream.output import (OutputStreamError,
FileOutputStream, StringOutputStream, OutputStream)
libs/hachoir_core/stream/input.py
-563
View File
@@ -1,563 +0,0 @@
from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
from hachoir_core.error import info
from hachoir_core.log import Logger
from hachoir_core.bits import str2long
from hachoir_core.i18n import getTerminalCharset
from hachoir_core.tools import lowerBound
from hachoir_core.i18n import _
from errno import ESPIPE
from weakref import ref as weakref_ref
from hachoir_core.stream import StreamError
class InputStreamError(StreamError):
pass
class ReadStreamError(InputStreamError):
def __init__(self, size, address, got=None):
self.size = size
self.address = address
self.got = got
if self.got is not None:
msg = _("Can't read %u bits at address %u (got %u bits)") % (self.size, self.address, self.got)
else:
msg = _("Can't read %u bits at address %u") % (self.size, self.address)
InputStreamError.__init__(self, msg)
class NullStreamError(InputStreamError):
def __init__(self, source):
self.source = source
msg = _("Input size is nul (source='%s')!") % self.source
InputStreamError.__init__(self, msg)
class FileFromInputStream:
_offset = 0
_from_end = False
def __init__(self, stream):
self.stream = stream
self._setSize(stream.askSize(self))
def _setSize(self, size):
if size is None:
self._size = size
elif size % 8:
raise InputStreamError("Invalid size")
else:
self._size = size // 8
def tell(self):
if self._from_end:
while self._size is None:
self.stream._feed(max(self.stream._current_size << 1, 1 << 16))
self._from_end = False
self._offset += self._size
return self._offset
def seek(self, pos, whence=0):
if whence == 0:
self._from_end = False
self._offset = pos
elif whence == 1:
self._offset += pos
elif whence == 2:
self._from_end = True
self._offset = pos
else:
raise ValueError("seek() second argument must be 0, 1 or 2")
def read(self, size=None):
def read(address, size):
shift, data, missing = self.stream.read(8 * address, 8 * size)
if shift:
raise InputStreamError("TODO: handle non-byte-aligned data")
return data
if self._size or size is not None and not self._from_end:
# We don't want self.tell() to read anything
# and the size must be known if we read until the end.
pos = self.tell()
if size is None or None < self._size < pos + size:
size = self._size - pos
if size <= 0:
return ''
data = read(pos, size)
self._offset += len(data)
return data
elif self._from_end:
# TODO: not tested
max_size = - self._offset
if size is None or max_size < size:
size = max_size
if size <= 0:
return ''
data = '', ''
self._offset = max(0, self.stream._current_size // 8 + self._offset)
self._from_end = False
bs = max(max_size, 1 << 16)
while True:
d = read(self._offset, bs)
data = data[1], d
self._offset += len(d)
if self._size:
bs = self._size - self._offset
if not bs:
data = data[0] + data[1]
d = len(data) - max_size
return data[d:d+size]
else:
# TODO: not tested
data = [ ]
size = 1 << 16
while True:
d = read(self._offset, size)
data.append(d)
self._offset += len(d)
if self._size:
size = self._size - self._offset
if not size:
return ''.join(data)
class InputStream(Logger):
_set_size = None
_current_size = 0
def __init__(self, source=None, size=None, packets=None, **args):
self.source = source
self._size = size # in bits
if size == 0:
raise NullStreamError(source)
self.tags = tuple(args.get("tags", tuple()))
self.packets = packets
def askSize(self, client):
if self._size != self._current_size:
if self._set_size is None:
self._set_size = []
self._set_size.append(weakref_ref(client))
return self._size
def _setSize(self, size=None):
assert self._size is None or self._current_size <= self._size
if self._size != self._current_size:
self._size = self._current_size
if not self._size:
raise NullStreamError(self.source)
if self._set_size:
for client in self._set_size:
client = client()
if client:
client._setSize(self._size)
del self._set_size
size = property(lambda self: self._size, doc="Size of the stream in bits")
checked = property(lambda self: self._size == self._current_size)
def sizeGe(self, size, const=False):
return self._current_size >= size or \
not (None < self._size < size or const or self._feed(size))
def _feed(self, size):
return self.read(size-1,1)[2]
def read(self, address, size):
"""
Read 'size' bits at position 'address' (in bits)
from the beginning of the stream.
"""
raise NotImplementedError
def readBits(self, address, nbits, endian):
assert endian in (BIG_ENDIAN, LITTLE_ENDIAN)
shift, data, missing = self.read(address, nbits)
if missing:
raise ReadStreamError(nbits, address)
value = str2long(data, endian)
if endian is BIG_ENDIAN:
value >>= len(data) * 8 - shift - nbits
else:
value >>= shift
return value & (1 << nbits) - 1
def readInteger(self, address, signed, nbits, endian):
""" Read an integer number """
value = self.readBits(address, nbits, endian)
# Signe number. Example with nbits=8:
# if 128 <= value: value -= 256
if signed and (1 << (nbits-1)) <= value:
value -= (1 << nbits)
return value
def readBytes(self, address, nb_bytes):
shift, data, missing = self.read(address, 8 * nb_bytes)
if shift:
raise InputStreamError("TODO: handle non-byte-aligned data")
if missing:
raise ReadStreamError(8 * nb_bytes, address)
return data
def searchBytesLength(self, needle, include_needle,
start_address=0, end_address=None):
"""
If include_needle is True, add its length to the result.
Returns None is needle can't be found.
"""
pos = self.searchBytes(needle, start_address, end_address)
if pos is None:
return None
length = (pos - start_address) // 8
if include_needle:
length += len(needle)
return length
def searchBytes(self, needle, start_address=0, end_address=None):
"""
Search some bytes in [start_address;end_address[. Addresses must
be aligned to byte. Returns the address of the bytes if found,
None else.
"""
if start_address % 8:
raise InputStreamError("Unable to search bytes with address with bit granularity")
length = len(needle)
size = max(3 * length, 4096)
buffer = ''
if self._size and (end_address is None or self._size < end_address):
end_address = self._size
while True:
if end_address is not None:
todo = (end_address - start_address) >> 3
if todo < size:
if todo <= 0:
return None
size = todo
data = self.readBytes(start_address, size)
if end_address is None and self._size:
end_address = self._size
size = (end_address - start_address) >> 3
assert size > 0
data = data[:size]
start_address += 8 * size
buffer = buffer[len(buffer) - length + 1:] + data
found = buffer.find(needle)
if found >= 0:
return start_address + (found - len(buffer)) * 8
def file(self):
return FileFromInputStream(self)
class InputPipe(object):
"""
InputPipe makes input streams seekable by caching a certain
amount of data. The memory usage may be unlimited in worst cases.
A function (set_size) is called when the size of the stream is known.
InputPipe sees the input stream as an array of blocks of
size = (2 ^ self.buffer_size) and self.buffers maps to this array.
It also maintains a circular ordered list of non-discarded blocks,
sorted by access time.
Each element of self.buffers is an array of 3 elements:
* self.buffers[i][0] is the data.
len(self.buffers[i][0]) == 1 << self.buffer_size
(except at the end: the length may be smaller)
* self.buffers[i][1] is the index of a more recently used block
* self.buffers[i][2] is the opposite of self.buffers[1],
in order to have a double-linked list.
For any discarded block, self.buffers[i] = None
self.last is the index of the most recently accessed block.
self.first is the first (= smallest index) non-discarded block.
How InputPipe discards blocks:
* Just before returning from the read method.
* Only if there are more than self.buffer_nb_min blocks in memory.
* While self.buffers[self.first] is that least recently used block.
Property: There is no hole in self.buffers, except at the beginning.
"""
buffer_nb_min = 256
buffer_size = 16
last = None
size = None
def __init__(self, input, set_size=None):
self._input = input
self.first = self.address = 0
self.buffers = []
self.set_size = set_size
current_size = property(lambda self: len(self.buffers) << self.buffer_size)
def _append(self, data):
if self.last is None:
self.last = next = prev = 0
else:
prev = self.last
last = self.buffers[prev]
next = last[1]
self.last = self.buffers[next][2] = last[1] = len(self.buffers)
self.buffers.append([ data, next, prev ])
def _get(self, index):
if index >= len(self.buffers):
return ''
buf = self.buffers[index]
if buf is None:
raise InputStreamError(_("Error: Buffers too small. Can't seek backward."))
if self.last != index:
next = buf[1]
prev = buf[2]
self.buffers[next][2] = prev
self.buffers[prev][1] = next
first = self.buffers[self.last][1]
buf[1] = first
buf[2] = self.last
self.buffers[first][2] = index
self.buffers[self.last][1] = index
self.last = index
return buf[0]
def _flush(self):
lim = len(self.buffers) - self.buffer_nb_min
while self.first < lim:
buf = self.buffers[self.first]
if buf[2] != self.last:
break
info("Discarding buffer %u." % self.first)
self.buffers[self.last][1] = buf[1]
self.buffers[buf[1]][2] = self.last
self.buffers[self.first] = None
self.first += 1
def seek(self, address):
assert 0 <= address
self.address = address
def read(self, size):
end = self.address + size
for i in xrange(len(self.buffers), (end >> self.buffer_size) + 1):
data = self._input.read(1 << self.buffer_size)
if len(data) < 1 << self.buffer_size:
self.size = (len(self.buffers) << self.buffer_size) + len(data)
if self.set_size:
self.set_size(self.size)
if data:
self._append(data)
break
self._append(data)
block, offset = divmod(self.address, 1 << self.buffer_size)
data = ''.join(self._get(index)
for index in xrange(block, (end - 1 >> self.buffer_size) + 1)
)[offset:offset+size]
self._flush()
self.address += len(data)
return data
class InputIOStream(InputStream):
def __init__(self, input, size=None, **args):
if not hasattr(input, "seek"):
if size is None:
input = InputPipe(input, self._setSize)
else:
input = InputPipe(input)
elif size is None:
try:
input.seek(0, 2)
size = input.tell() * 8
except IOError, err:
if err.errno == ESPIPE:
input = InputPipe(input, self._setSize)
else:
charset = getTerminalCharset()
errmsg = unicode(str(err), charset)
source = args.get("source", "<inputio:%r>" % input)
raise InputStreamError(_("Unable to get size of %s: %s") % (source, errmsg))
self._input = input
InputStream.__init__(self, size=size, **args)
def __current_size(self):
if self._size:
return self._size
if self._input.size:
return 8 * self._input.size
return 8 * self._input.current_size
_current_size = property(__current_size)
def read(self, address, size):
assert size > 0
_size = self._size
address, shift = divmod(address, 8)
self._input.seek(address)
size = (size + shift + 7) >> 3
data = self._input.read(size)
got = len(data)
missing = size != got
if missing and _size == self._size:
raise ReadStreamError(8 * size, 8 * address, 8 * got)
return shift, data, missing
def file(self):
if hasattr(self._input, "fileno"):
from os import dup, fdopen
new_fd = dup(self._input.fileno())
new_file = fdopen(new_fd, "r")
new_file.seek(0)
return new_file
return InputStream.file(self)
class StringInputStream(InputStream):
def __init__(self, data, source="<string>", **args):
self.data = data
InputStream.__init__(self, source=source, size=8*len(data), **args)
self._current_size = self._size
def read(self, address, size):
address, shift = divmod(address, 8)
size = (size + shift + 7) >> 3
data = self.data[address:address+size]
got = len(data)
if got != size:
raise ReadStreamError(8 * size, 8 * address, 8 * got)
return shift, data, False
class InputSubStream(InputStream):
def __init__(self, stream, offset, size=None, source=None, **args):
if offset is None:
offset = 0
if size is None and stream.size is not None:
size = stream.size - offset
if None < size <= 0:
raise ValueError("InputSubStream: offset is outside input stream")
self.stream = stream
self._offset = offset
if source is None:
source = "<substream input=%s offset=%s size=%s>" % (stream.source, offset, size)
InputStream.__init__(self, source=source, size=size, **args)
self.stream.askSize(self)
_current_size = property(lambda self: min(self._size, max(0, self.stream._current_size - self._offset)))
def read(self, address, size):
return self.stream.read(self._offset + address, size)
def InputFieldStream(field, **args):
if not field.parent:
return field.stream
stream = field.parent.stream
args["size"] = field.size
args.setdefault("source", stream.source + field.path)
return InputSubStream(stream, field.absolute_address, **args)
class FragmentedStream(InputStream):
def __init__(self, field, **args):
self.stream = field.parent.stream
data = field.getData()
self.fragments = [ (0, data.absolute_address, data.size) ]
self.next = field.next
args.setdefault("source", "%s%s" % (self.stream.source, field.path))
InputStream.__init__(self, **args)
if not self.next:
self._current_size = data.size
self._setSize()
def _feed(self, end):
if self._current_size < end:
if self.checked:
raise ReadStreamError(end - self._size, self._size)
a, fa, fs = self.fragments[-1]
while self.stream.sizeGe(fa + min(fs, end - a)):
a += fs
f = self.next
if a >= end:
self._current_size = end
if a == end and not f:
self._setSize()
return False
if f:
self.next = f.next
f = f.getData()
if not f:
self._current_size = a
self._setSize()
return True
fa = f.absolute_address
fs = f.size
self.fragments += [ (a, fa, fs) ]
self._current_size = a + max(0, self.stream.size - fa)
self._setSize()
return True
return False
def read(self, address, size):
assert size > 0
missing = self._feed(address + size)
if missing:
size = self._size - address
if size <= 0:
return 0, '', True
d = []
i = lowerBound(self.fragments, lambda x: x[0] <= address)
a, fa, fs = self.fragments[i-1]
a -= address
fa -= a
fs += a
s = None
while True:
n = min(fs, size)
u, v, w = self.stream.read(fa, n)
assert not w
if s is None:
s = u
else:
assert not u
d += [ v ]
size -= n
if not size:
return s, ''.join(d), missing
a, fa, fs = self.fragments[i]
i += 1
class ConcatStream(InputStream):
# TODO: concatene any number of any type of stream
def __init__(self, streams, **args):
if len(streams) > 2 or not streams[0].checked:
raise NotImplementedError
self.__size0 = streams[0].size
size1 = streams[1].askSize(self)
if size1 is not None:
args["size"] = self.__size0 + size1
self.__streams = streams
InputStream.__init__(self, **args)
_current_size = property(lambda self: self.__size0 + self.__streams[1]._current_size)
def read(self, address, size):
_size = self._size
s = self.__size0 - address
shift, data, missing = None, '', False
if s > 0:
s = min(size, s)
shift, data, w = self.__streams[0].read(address, s)
assert not w
a, s = 0, size - s
else:
a, s = -s, size
if s:
u, v, missing = self.__streams[1].read(a, s)
if missing and _size == self._size:
raise ReadStreamError(s, a)
if shift is None:
shift = u
else:
assert not u
data += v
return shift, data, missing
libs/hachoir_core/stream/input_helper.py
-38
View File
@@ -1,38 +0,0 @@
from hachoir_core.i18n import getTerminalCharset, guessBytesCharset, _
from hachoir_core.stream import InputIOStream, InputSubStream, InputStreamError
def FileInputStream(filename, real_filename=None, **args):
"""
Create an input stream of a file. filename must be unicode.
real_filename is an optional argument used to specify the real filename,
its type can be 'str' or 'unicode'. Use real_filename when you are
not able to convert filename to real unicode string (ie. you have to
use unicode(name, 'replace') or unicode(name, 'ignore')).
"""
assert isinstance(filename, unicode)
if not real_filename:
real_filename = filename
try:
inputio = open(real_filename, 'rb')
except IOError, err:
charset = getTerminalCharset()
errmsg = unicode(str(err), charset)
raise InputStreamError(_("Unable to open file %s: %s") % (filename, errmsg))
source = "file:" + filename
offset = args.pop("offset", 0)
size = args.pop("size", None)
if offset or size:
if size:
size = 8 * size
stream = InputIOStream(inputio, source=source, **args)
return InputSubStream(stream, 8 * offset, size, **args)
else:
args.setdefault("tags",[]).append(("filename", filename))
return InputIOStream(inputio, source=source, **args)
def guessStreamCharset(stream, address, size, default=None):
size = min(size, 1024*8)
bytes = stream.readBytes(address, size//8)
return guessBytesCharset(bytes, default)
libs/hachoir_core/stream/output.py
-173
View File
@@ -1,173 +0,0 @@
from cStringIO import StringIO
from hachoir_core.endian import BIG_ENDIAN
from hachoir_core.bits import long2raw
from hachoir_core.stream import StreamError
from errno import EBADF
MAX_READ_NBYTES = 2 ** 16
class OutputStreamError(StreamError):
pass
class OutputStream(object):
def __init__(self, output, filename=None):
self._output = output
self._filename = filename
self._bit_pos = 0
self._byte = 0
def _getFilename(self):
return self._filename
filename = property(_getFilename)
def writeBit(self, state, endian):
if self._bit_pos == 7:
self._bit_pos = 0
if state:
if endian is BIG_ENDIAN:
self._byte |= 1
else:
self._byte |= 128
self._output.write(chr(self._byte))
self._byte = 0
else:
if state:
if endian is BIG_ENDIAN:
self._byte |= (1 << self._bit_pos)
else:
self._byte |= (1 << (7-self._bit_pos))
self._bit_pos += 1
def writeBits(self, count, value, endian):
assert 0 <= value < 2**count
# Feed bits to align to byte address
if self._bit_pos != 0:
n = 8 - self._bit_pos
if n <= count:
count -= n
if endian is BIG_ENDIAN:
self._byte |= (value >> count)
value &= ((1 << count) - 1)
else:
self._byte |= (value & ((1 << n)-1)) << self._bit_pos
value >>= n
self._output.write(chr(self._byte))
self._bit_pos = 0
self._byte = 0
else:
if endian is BIG_ENDIAN:
self._byte |= (value << (8-self._bit_pos-count))
else:
self._byte |= (value << self._bit_pos)
self._bit_pos += count
return
# Write byte per byte
while 8 <= count:
count -= 8
if endian is BIG_ENDIAN:
byte = (value >> count)
value &= ((1 << count) - 1)
else:
byte = (value & 0xFF)
value >>= 8
self._output.write(chr(byte))
# Keep last bits
assert 0 <= count < 8
self._bit_pos = count
if 0 < count:
assert 0 <= value < 2**count
if endian is BIG_ENDIAN:
self._byte = value << (8-count)
else:
self._byte = value
else:
assert value == 0
self._byte = 0
def writeInteger(self, value, signed, size_byte, endian):
if signed:
value += 1 << (size_byte*8 - 1)
raw = long2raw(value, endian, size_byte)
self.writeBytes(raw)
def copyBitsFrom(self, input, address, nb_bits, endian):
if (nb_bits % 8) == 0:
self.copyBytesFrom(input, address, nb_bits/8)
else:
# Arbitrary limit (because we should use a buffer, like copyBytesFrom(),
# but with endianess problem
assert nb_bits <= 128
data = input.readBits(address, nb_bits, endian)
self.writeBits(nb_bits, data, endian)
def copyBytesFrom(self, input, address, nb_bytes):
if (address % 8):
raise OutputStreamError("Unable to copy bytes with address with bit granularity")
buffer_size = 1 << 12 # 8192 (8 KB)
while 0 < nb_bytes:
# Compute buffer size
if nb_bytes < buffer_size:
buffer_size = nb_bytes
# Read
data = input.readBytes(address, buffer_size)
# Write
self.writeBytes(data)
# Move address
address += buffer_size*8
nb_bytes -= buffer_size
def writeBytes(self, bytes):
if self._bit_pos != 0:
raise NotImplementedError()
self._output.write(bytes)
def readBytes(self, address, nbytes):
"""
Read bytes from the stream at specified address (in bits).
Address have to be a multiple of 8.
nbytes have to in 1..MAX_READ_NBYTES (64 KB).
This method is only supported for StringOuputStream (not on
FileOutputStream).
Return read bytes as byte string.
"""
assert (address % 8) == 0
assert (1 <= nbytes <= MAX_READ_NBYTES)
self._output.flush()
oldpos = self._output.tell()
try:
self._output.seek(0)
try:
return self._output.read(nbytes)
except IOError, err:
if err[0] == EBADF:
raise OutputStreamError("Stream doesn't support read() operation")
finally:
self._output.seek(oldpos)
def StringOutputStream():
"""
Create an output stream into a string.
"""
data = StringIO()
return OutputStream(data)
def FileOutputStream(filename, real_filename=None):
"""
Create an output stream into file with given name.
Filename have to be unicode, whereas (optional) real_filename can be str.
"""
assert isinstance(filename, unicode)
if not real_filename:
real_filename = filename
output = open(real_filename, 'wb')
return OutputStream(output, filename=filename)
libs/hachoir_core/stream/stream.py
-5
View File
@@ -1,5 +0,0 @@
from hachoir_core.error import HachoirError
class StreamError(HachoirError):
pass
libs/hachoir_core/text_handler.py
-60
View File
@@ -1,60 +0,0 @@
"""
Utilities used to convert a field to human classic reprentation of data.
"""
from hachoir_core.tools import (
humanDuration, humanFilesize, alignValue,
durationWin64 as doDurationWin64,
deprecated)
from types import FunctionType, MethodType
from hachoir_core.field import Field
def textHandler(field, handler):
assert isinstance(handler, (FunctionType, MethodType))
assert issubclass(field.__class__, Field)
field.createDisplay = lambda: handler(field)
return field
def displayHandler(field, handler):
assert isinstance(handler, (FunctionType, MethodType))
assert issubclass(field.__class__, Field)
field.createDisplay = lambda: handler(field.value)
return field
@deprecated("Use TimedeltaWin64 field type")
def durationWin64(field):
"""
Convert Windows 64-bit duration to string. The timestamp format is
a 64-bit number: number of 100ns. See also timestampWin64().
>>> durationWin64(type("", (), dict(value=2146280000, size=64)))
u'3 min 34 sec 628 ms'
>>> durationWin64(type("", (), dict(value=(1 << 64)-1, size=64)))
u'58494 years 88 days 5 hours'
"""
assert hasattr(field, "value") and hasattr(field, "size")
assert field.size == 64
delta = doDurationWin64(field.value)
return humanDuration(delta)
def filesizeHandler(field):
"""
Format field value using humanFilesize()
"""
return displayHandler(field, humanFilesize)
def hexadecimal(field):
"""
Convert an integer to hexadecimal in lower case. Returns unicode string.
>>> hexadecimal(type("", (), dict(value=412, size=16)))
u'0x019c'
>>> hexadecimal(type("", (), dict(value=0, size=32)))
u'0x00000000'
"""
assert hasattr(field, "value") and hasattr(field, "size")
size = field.size
padding = alignValue(size, 4) // 4
pattern = u"0x%%0%ux" % padding
return pattern % field.value
libs/hachoir_core/timeout.py
-76
View File
@@ -1,76 +0,0 @@
"""
limitedTime(): set a timeout in seconds when calling a function,
raise a Timeout error if time exceed.
"""
from math import ceil
IMPLEMENTATION = None
class Timeout(RuntimeError):
"""
Timeout error, inherits from RuntimeError
"""
pass
def signalHandler(signum, frame):
"""
Signal handler to catch timeout signal: raise Timeout exception.
"""
raise Timeout("Timeout exceed!")
def limitedTime(second, func, *args, **kw):
"""
Call func(*args, **kw) with a timeout of second seconds.
"""
return func(*args, **kw)
def fixTimeout(second):
"""
Fix timeout value: convert to integer with a minimum of 1 second
"""
if isinstance(second, float):
second = int(ceil(second))
assert isinstance(second, (int, long))
return max(second, 1)
if not IMPLEMENTATION:
try:
from signal import signal, alarm, SIGALRM
# signal.alarm() implementation
def limitedTime(second, func, *args, **kw):
second = fixTimeout(second)
old_alarm = signal(SIGALRM, signalHandler)
try:
alarm(second)
return func(*args, **kw)
finally:
alarm(0)
signal(SIGALRM, old_alarm)
IMPLEMENTATION = "signal.alarm()"
except ImportError:
pass
if not IMPLEMENTATION:
try:
from signal import signal, SIGXCPU
from resource import getrlimit, setrlimit, RLIMIT_CPU
# resource.setrlimit(RLIMIT_CPU) implementation
# "Bug": timeout is 'CPU' time so sleep() are not part of the timeout
def limitedTime(second, func, *args, **kw):
second = fixTimeout(second)
old_alarm = signal(SIGXCPU, signalHandler)
current = getrlimit(RLIMIT_CPU)
try:
setrlimit(RLIMIT_CPU, (second, current[1]))
return func(*args, **kw)
finally:
setrlimit(RLIMIT_CPU, current)
signal(SIGXCPU, old_alarm)
IMPLEMENTATION = "resource.setrlimit(RLIMIT_CPU)"
except ImportError:
pass
libs/hachoir_core/tools.py
-582
View File
@@ -1,582 +0,0 @@
# -*- coding: utf-8 -*-
"""
Various utilities.
"""
from hachoir_core.i18n import _, ngettext
import re
import stat
from datetime import datetime, timedelta, MAXYEAR
from warnings import warn
def deprecated(comment=None):
"""
This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emmitted
when the function is used.
Examples: ::
@deprecated
def oldfunc(): ...
@deprecated("use newfunc()!")
def oldfunc2(): ...
Code from: http://code.activestate.com/recipes/391367/
"""
def _deprecated(func):
def newFunc(*args, **kwargs):
message = "Call to deprecated function %s" % func.__name__
if comment:
message += ": " + comment
warn(message, category=DeprecationWarning, stacklevel=2)
return func(*args, **kwargs)
newFunc.__name__ = func.__name__
newFunc.__doc__ = func.__doc__
newFunc.__dict__.update(func.__dict__)
return newFunc
return _deprecated
def paddingSize(value, align):
"""
Compute size of a padding field.
>>> paddingSize(31, 4)
1
>>> paddingSize(32, 4)
0
>>> paddingSize(33, 4)
3
Note: (value + paddingSize(value, align)) == alignValue(value, align)
"""
if value % align != 0:
return align - (value % align)
else:
return 0
def alignValue(value, align):
"""
Align a value to next 'align' multiple.
>>> alignValue(31, 4)
32
>>> alignValue(32, 4)
32
>>> alignValue(33, 4)
36
Note: alignValue(value, align) == (value + paddingSize(value, align))
"""
if value % align != 0:
return value + align - (value % align)
else:
return value
def timedelta2seconds(delta):
"""
Convert a datetime.timedelta() objet to a number of second
(floatting point number).
>>> timedelta2seconds(timedelta(seconds=2, microseconds=40000))
2.04
>>> timedelta2seconds(timedelta(minutes=1, milliseconds=250))
60.25
"""
return delta.microseconds / 1000000.0 \
+ delta.seconds + delta.days * 60*60*24
def humanDurationNanosec(nsec):
"""
Convert a duration in nanosecond to human natural representation.
Returns an unicode string.
>>> humanDurationNanosec(60417893)
u'60.42 ms'
"""
# Nano second
if nsec < 1000:
return u"%u nsec" % nsec
# Micro seconds
usec, nsec = divmod(nsec, 1000)
if usec < 1000:
return u"%.2f usec" % (usec+float(nsec)/1000)
# Milli seconds
msec, usec = divmod(usec, 1000)
if msec < 1000:
return u"%.2f ms" % (msec + float(usec)/1000)
return humanDuration(msec)
def humanDuration(delta):
"""
Convert a duration in millisecond to human natural representation.
Returns an unicode string.
>>> humanDuration(0)
u'0 ms'
>>> humanDuration(213)
u'213 ms'
>>> humanDuration(4213)
u'4 sec 213 ms'
>>> humanDuration(6402309)
u'1 hour 46 min 42 sec'
"""
if not isinstance(delta, timedelta):
delta = timedelta(microseconds=delta*1000)
# Milliseconds
text = []
if 1000 <= delta.microseconds:
text.append(u"%u ms" % (delta.microseconds//1000))
# Seconds
minutes, seconds = divmod(delta.seconds, 60)
hours, minutes = divmod(minutes, 60)
if seconds:
text.append(u"%u sec" % seconds)
if minutes:
text.append(u"%u min" % minutes)
if hours:
text.append(ngettext("%u hour", "%u hours", hours) % hours)
# Days
years, days = divmod(delta.days, 365)
if days:
text.append(ngettext("%u day", "%u days", days) % days)
if years:
text.append(ngettext("%u year", "%u years", years) % years)
if 3 < len(text):
text = text[-3:]
elif not text:
return u"0 ms"
return u" ".join(reversed(text))
def humanFilesize(size):
"""
Convert a file size in byte to human natural representation.
It uses the values: 1 KB is 1024 bytes, 1 MB is 1024 KB, etc.
The result is an unicode string.
>>> humanFilesize(1)
u'1 byte'
>>> humanFilesize(790)
u'790 bytes'
>>> humanFilesize(256960)
u'250.9 KB'
"""
if size < 10000:
return ngettext("%u byte", "%u bytes", size) % size
units = [_("KB"), _("MB"), _("GB"), _("TB")]
size = float(size)
divisor = 1024
for unit in units:
size = size / divisor
if size < divisor:
return "%.1f %s" % (size, unit)
return "%u %s" % (size, unit)
def humanBitSize(size):
"""
Convert a size in bit to human classic representation.
It uses the values: 1 Kbit is 1000 bits, 1 Mbit is 1000 Kbit, etc.
The result is an unicode string.
>>> humanBitSize(1)
u'1 bit'
>>> humanBitSize(790)
u'790 bits'
>>> humanBitSize(256960)
u'257.0 Kbit'
"""
divisor = 1000
if size < divisor:
return ngettext("%u bit", "%u bits", size) % size
units = [u"Kbit", u"Mbit", u"Gbit", u"Tbit"]
size = float(size)
for unit in units:
size = size / divisor
if size < divisor:
return "%.1f %s" % (size, unit)
return u"%u %s" % (size, unit)
def humanBitRate(size):
"""
Convert a bit rate to human classic representation. It uses humanBitSize()
to convert size into human reprensation. The result is an unicode string.
>>> humanBitRate(790)
u'790 bits/sec'
>>> humanBitRate(256960)
u'257.0 Kbit/sec'
"""
return "".join((humanBitSize(size), "/sec"))
def humanFrequency(hertz):
"""
Convert a frequency in hertz to human classic representation.
It uses the values: 1 KHz is 1000 Hz, 1 MHz is 1000 KMhz, etc.
The result is an unicode string.
>>> humanFrequency(790)
u'790 Hz'
>>> humanFrequency(629469)
u'629.5 kHz'
"""
divisor = 1000
if hertz < divisor:
return u"%u Hz" % hertz
units = [u"kHz", u"MHz", u"GHz", u"THz"]
hertz = float(hertz)
for unit in units:
hertz = hertz / divisor
if hertz < divisor:
return u"%.1f %s" % (hertz, unit)
return u"%s %s" % (hertz, unit)
regex_control_code = re.compile(r"([\x00-\x1f\x7f])")
controlchars = tuple({
# Don't use "\0", because "\0"+"0"+"1" = "\001" = "\1" (1 character)
# Same rease to not use octal syntax ("\1")
ord("\n"): r"\n",
ord("\r"): r"\r",
ord("\t"): r"\t",
ord("\a"): r"\a",
ord("\b"): r"\b",
}.get(code, '\\x%02x' % code)
for code in xrange(128)
)
def makePrintable(data, charset, quote=None, to_unicode=False, smart=True):
r"""
Prepare a string to make it printable in the specified charset.
It escapes control characters. Characters with code bigger than 127
are escaped if data type is 'str' or if charset is "ASCII".
Examples with Unicode:
>>> aged = unicode("âgé", "UTF-8")
>>> repr(aged) # text type is 'unicode'
"u'\\xe2g\\xe9'"
>>> makePrintable("abc\0", "UTF-8")
'abc\\0'
>>> makePrintable(aged, "latin1")
'\xe2g\xe9'
>>> makePrintable(aged, "latin1", quote='"')
'"\xe2g\xe9"'
Examples with string encoded in latin1:
>>> aged_latin = unicode("âgé", "UTF-8").encode("latin1")
>>> repr(aged_latin) # text type is 'str'
"'\\xe2g\\xe9'"
>>> makePrintable(aged_latin, "latin1")
'\\xe2g\\xe9'
>>> makePrintable("", "latin1")
''
>>> makePrintable("a", "latin1", quote='"')
'"a"'
>>> makePrintable("", "latin1", quote='"')
'(empty)'
>>> makePrintable("abc", "latin1", quote="'")
"'abc'"
Control codes:
>>> makePrintable("\0\x03\x0a\x10 \x7f", "latin1")
'\\0\\3\\n\\x10 \\x7f'
Quote character may also be escaped (only ' and "):
>>> print makePrintable("a\"b", "latin-1", quote='"')
"a\"b"
>>> print makePrintable("a\"b", "latin-1", quote="'")
'a"b'
>>> print makePrintable("a'b", "latin-1", quote="'")
'a\'b'
"""
if data:
if not isinstance(data, unicode):
data = unicode(data, "ISO-8859-1")
charset = "ASCII"
data = regex_control_code.sub(
lambda regs: controlchars[ord(regs.group(1))], data)
if quote:
if quote in "\"'":
data = data.replace(quote, '\\' + quote)
data = ''.join((quote, data, quote))
elif quote:
data = "(empty)"
data = data.encode(charset, "backslashreplace")
if smart:
# Replace \x00\x01 by \0\1
data = re.sub(r"\\x0([0-7])(?=[^0-7]|$)", r"\\\1", data)
if to_unicode:
data = unicode(data, charset)
return data
def makeUnicode(text):
r"""
Convert text to printable Unicode string. For byte string (type 'str'),
use charset ISO-8859-1 for the conversion to Unicode
>>> makeUnicode(u'abc\0d')
u'abc\\0d'
>>> makeUnicode('a\xe9')
u'a\xe9'
"""
if isinstance(text, str):
text = unicode(text, "ISO-8859-1")
elif not isinstance(text, unicode):
text = unicode(text)
text = regex_control_code.sub(
lambda regs: controlchars[ord(regs.group(1))], text)
text = re.sub(r"\\x0([0-7])(?=[^0-7]|$)", r"\\\1", text)
return text
def binarySearch(seq, cmp_func):
"""
Search a value in a sequence using binary search. Returns index of the
value, or None if the value doesn't exist.
'seq' have to be sorted in ascending order according to the
comparaison function ;
'cmp_func', prototype func(x), is the compare function:
- Return strictly positive value if we have to search forward ;
- Return strictly negative value if we have to search backward ;
- Otherwise (zero) we got the value.
>>> # Search number 5 (search forward)
... binarySearch([0, 4, 5, 10], lambda x: 5-x)
2
>>> # Backward search
... binarySearch([10, 5, 4, 0], lambda x: x-5)
1
"""
lower = 0
upper = len(seq)
while lower < upper:
index = (lower + upper) >> 1
diff = cmp_func(seq[index])
if diff < 0:
upper = index
elif diff > 0:
lower = index + 1
else:
return index
return None
def lowerBound(seq, cmp_func):
f = 0
l = len(seq)
while l > 0:
h = l >> 1
m = f + h
if cmp_func(seq[m]):
f = m
f += 1
l -= h + 1
else:
l = h
return f
def humanUnixAttributes(mode):
"""
Convert a Unix file attributes (or "file mode") to an unicode string.
Original source code:
http://cvs.savannah.gnu.org/viewcvs/coreutils/lib/filemode.c?root=coreutils
>>> humanUnixAttributes(0644)
u'-rw-r--r-- (644)'
>>> humanUnixAttributes(02755)
u'-rwxr-sr-x (2755)'
"""
def ftypelet(mode):
if stat.S_ISREG (mode) or not stat.S_IFMT(mode):
return '-'
if stat.S_ISBLK (mode): return 'b'
if stat.S_ISCHR (mode): return 'c'
if stat.S_ISDIR (mode): return 'd'
if stat.S_ISFIFO(mode): return 'p'
if stat.S_ISLNK (mode): return 'l'
if stat.S_ISSOCK(mode): return 's'
return '?'
chars = [ ftypelet(mode), 'r', 'w', 'x', 'r', 'w', 'x', 'r', 'w', 'x' ]
for i in xrange(1, 10):
if not mode & 1 << 9 - i:
chars[i] = '-'
if mode & stat.S_ISUID:
if chars[3] != 'x':
chars[3] = 'S'
else:
chars[3] = 's'
if mode & stat.S_ISGID:
if chars[6] != 'x':
chars[6] = 'S'
else:
chars[6] = 's'
if mode & stat.S_ISVTX:
if chars[9] != 'x':
chars[9] = 'T'
else:
chars[9] = 't'
return u"%s (%o)" % (''.join(chars), mode)
def createDict(data, index):
"""
Create a new dictionnay from dictionnary key=>values:
just keep value number 'index' from all values.
>>> data={10: ("dix", 100, "a"), 20: ("vingt", 200, "b")}
>>> createDict(data, 0)
{10: 'dix', 20: 'vingt'}
>>> createDict(data, 2)
{10: 'a', 20: 'b'}
"""
return dict( (key,values[index]) for key, values in data.iteritems() )
# Start of UNIX timestamp (Epoch): 1st January 1970 at 00:00
UNIX_TIMESTAMP_T0 = datetime(1970, 1, 1)
def timestampUNIX(value):
"""
Convert an UNIX (32-bit) timestamp to datetime object. Timestamp value
is the number of seconds since the 1st January 1970 at 00:00. Maximum
value is 2147483647: 19 january 2038 at 03:14:07.
May raise ValueError for invalid value: value have to be in 0..2147483647.
>>> timestampUNIX(0)
datetime.datetime(1970, 1, 1, 0, 0)
>>> timestampUNIX(1154175644)
datetime.datetime(2006, 7, 29, 12, 20, 44)
>>> timestampUNIX(1154175644.37)
datetime.datetime(2006, 7, 29, 12, 20, 44, 370000)
>>> timestampUNIX(2147483647)
datetime.datetime(2038, 1, 19, 3, 14, 7)
"""
if not isinstance(value, (float, int, long)):
raise TypeError("timestampUNIX(): an integer or float is required")
if not(0 <= value <= 2147483647):
raise ValueError("timestampUNIX(): value have to be in 0..2147483647")
return UNIX_TIMESTAMP_T0 + timedelta(seconds=value)
# Start of Macintosh timestamp: 1st January 1904 at 00:00
MAC_TIMESTAMP_T0 = datetime(1904, 1, 1)
def timestampMac32(value):
"""
Convert an Mac (32-bit) timestamp to string. The format is the number
of seconds since the 1st January 1904 (to 2040). Returns unicode string.
>>> timestampMac32(0)
datetime.datetime(1904, 1, 1, 0, 0)
>>> timestampMac32(2843043290)
datetime.datetime(1994, 2, 2, 14, 14, 50)
"""
if not isinstance(value, (float, int, long)):
raise TypeError("an integer or float is required")
if not(0 <= value <= 4294967295):
return _("invalid Mac timestamp (%s)") % value
return MAC_TIMESTAMP_T0 + timedelta(seconds=value)
def durationWin64(value):
"""
Convert Windows 64-bit duration to string. The timestamp format is
a 64-bit number: number of 100ns. See also timestampWin64().
>>> str(durationWin64(1072580000))
'0:01:47.258000'
>>> str(durationWin64(2146280000))
'0:03:34.628000'
"""
if not isinstance(value, (float, int, long)):
raise TypeError("an integer or float is required")
if value < 0:
raise ValueError("value have to be a positive or nul integer")
return timedelta(microseconds=value/10)
# Start of 64-bit Windows timestamp: 1st January 1600 at 00:00
WIN64_TIMESTAMP_T0 = datetime(1601, 1, 1, 0, 0, 0)
def timestampWin64(value):
"""
Convert Windows 64-bit timestamp to string. The timestamp format is
a 64-bit number which represents number of 100ns since the
1st January 1601 at 00:00. Result is an unicode string.
See also durationWin64(). Maximum date is 28 may 60056.
>>> timestampWin64(0)
datetime.datetime(1601, 1, 1, 0, 0)
>>> timestampWin64(127840491566710000)
datetime.datetime(2006, 2, 10, 12, 45, 56, 671000)
"""
try:
return WIN64_TIMESTAMP_T0 + durationWin64(value)
except OverflowError:
raise ValueError(_("date newer than year %s (value=%s)") % (MAXYEAR, value))
# Start of 60-bit UUID timestamp: 15 October 1582 at 00:00
UUID60_TIMESTAMP_T0 = datetime(1582, 10, 15, 0, 0, 0)
def timestampUUID60(value):
"""
Convert UUID 60-bit timestamp to string. The timestamp format is
a 60-bit number which represents number of 100ns since the
the 15 October 1582 at 00:00. Result is an unicode string.
>>> timestampUUID60(0)
datetime.datetime(1582, 10, 15, 0, 0)
>>> timestampUUID60(130435676263032368)
datetime.datetime(1996, 2, 14, 5, 13, 46, 303236)
"""
if not isinstance(value, (float, int, long)):
raise TypeError("an integer or float is required")
if value < 0:
raise ValueError("value have to be a positive or nul integer")
try:
return UUID60_TIMESTAMP_T0 + timedelta(microseconds=value/10)
except OverflowError:
raise ValueError(_("timestampUUID60() overflow (value=%s)") % value)
def humanDatetime(value, strip_microsecond=True):
"""
Convert a timestamp to Unicode string: use ISO format with space separator.
>>> humanDatetime( datetime(2006, 7, 29, 12, 20, 44) )
u'2006-07-29 12:20:44'
>>> humanDatetime( datetime(2003, 6, 30, 16, 0, 5, 370000) )
u'2003-06-30 16:00:05'
>>> humanDatetime( datetime(2003, 6, 30, 16, 0, 5, 370000), False )
u'2003-06-30 16:00:05.370000'
"""
text = unicode(value.isoformat())
text = text.replace('T', ' ')
if strip_microsecond and "." in text:
text = text.split(".")[0]
return text
NEWLINES_REGEX = re.compile("\n+")
def normalizeNewline(text):
r"""
Replace Windows and Mac newlines with Unix newlines.
Replace multiple consecutive newlines with one newline.
>>> normalizeNewline('a\r\nb')
'a\nb'
>>> normalizeNewline('a\r\rb')
'a\nb'
>>> normalizeNewline('a\n\nb')
'a\nb'
"""
text = text.replace("\r\n", "\n")
text = text.replace("\r", "\n")
return NEWLINES_REGEX.sub("\n", text)
libs/hachoir_core/version.py
-5
View File
@@ -1,5 +0,0 @@
PACKAGE = "hachoir-core"
VERSION = "1.3.4"
WEBSITE = 'http://bitbucket.org/haypo/hachoir/wiki/hachoir-core'
LICENSE = 'GNU GPL v2'
libs/hachoir_metadata/__init__.py
-15
View File
@@ -1,15 +0,0 @@
from hachoir_metadata.version import VERSION as __version__
from hachoir_metadata.metadata import extractMetadata
# Just import the module,
# each module use registerExtractor() method
import hachoir_metadata.archive
import hachoir_metadata.audio
import hachoir_metadata.file_system
import hachoir_metadata.image
import hachoir_metadata.jpeg
import hachoir_metadata.misc
import hachoir_metadata.program
import hachoir_metadata.riff
import hachoir_metadata.video
libs/hachoir_metadata/archive.py
-166
View File
@@ -1,166 +0,0 @@
from hachoir_metadata.metadata_item import QUALITY_BEST, QUALITY_FASTEST
from hachoir_metadata.safe import fault_tolerant, getValue
from hachoir_metadata.metadata import (
RootMetadata, Metadata, MultipleMetadata, registerExtractor)
from hachoir_parser.archive import (Bzip2Parser, CabFile, GzipParser,
TarFile, ZipFile, MarFile)
from hachoir_core.tools import humanUnixAttributes
from hachoir_core.i18n import _
def maxNbFile(meta):
if meta.quality <= QUALITY_FASTEST:
return 0
if QUALITY_BEST <= meta.quality:
return None
return 1 + int(10 * meta.quality)
def computeCompressionRate(meta):
"""
Compute compression rate, sizes have to be in byte.
"""
if not meta.has("file_size") \
or not meta.get("compr_size", 0):
return
file_size = meta.get("file_size")
if not file_size:
return
meta.compr_rate = float(file_size) / meta.get("compr_size")
class Bzip2Metadata(RootMetadata):
def extract(self, zip):
if "file" in zip:
self.compr_size = zip["file"].size/8
class GzipMetadata(RootMetadata):
def extract(self, gzip):
self.useHeader(gzip)
computeCompressionRate(self)
@fault_tolerant
def useHeader(self, gzip):
self.compression = gzip["compression"].display
if gzip["mtime"]:
self.last_modification = gzip["mtime"].value
self.os = gzip["os"].display
if gzip["has_filename"].value:
self.filename = getValue(gzip, "filename")
if gzip["has_comment"].value:
self.comment = getValue(gzip, "comment")
self.compr_size = gzip["file"].size/8
self.file_size = gzip["size"].value
class ZipMetadata(MultipleMetadata):
def extract(self, zip):
max_nb = maxNbFile(self)
for index, field in enumerate(zip.array("file")):
if max_nb is not None and max_nb <= index:
self.warning("ZIP archive contains many files, but only first %s files are processed" % max_nb)
break
self.processFile(field)
@fault_tolerant
def processFile(self, field):
meta = Metadata(self)
meta.filename = field["filename"].value
meta.creation_date = field["last_mod"].value
meta.compression = field["compression"].display
if "data_desc" in field:
meta.file_size = field["data_desc/file_uncompressed_size"].value
if field["data_desc/file_compressed_size"].value:
meta.compr_size = field["data_desc/file_compressed_size"].value
else:
meta.file_size = field["uncompressed_size"].value
if field["compressed_size"].value:
meta.compr_size = field["compressed_size"].value
computeCompressionRate(meta)
self.addGroup(field.name, meta, "File \"%s\"" % meta.get('filename'))
class TarMetadata(MultipleMetadata):
def extract(self, tar):
max_nb = maxNbFile(self)
for index, field in enumerate(tar.array("file")):
if max_nb is not None and max_nb <= index:
self.warning("TAR archive contains many files, but only first %s files are processed" % max_nb)
break
meta = Metadata(self)
self.extractFile(field, meta)
if meta.has("filename"):
title = _('File "%s"') % meta.getText('filename')
else:
title = _("File")
self.addGroup(field.name, meta, title)
@fault_tolerant
def extractFile(self, field, meta):
meta.filename = field["name"].value
meta.file_attr = humanUnixAttributes(field.getOctal("mode"))
meta.file_size = field.getOctal("size")
try:
if field.getOctal("mtime"):
meta.last_modification = field.getDatetime()
except ValueError:
pass
meta.file_type = field["type"].display
meta.author = "%s (uid=%s), group %s (gid=%s)" %\
(field["uname"].value, field.getOctal("uid"),
field["gname"].value, field.getOctal("gid"))
class CabMetadata(MultipleMetadata):
def extract(self, cab):
if "folder[0]" in cab:
self.useFolder(cab["folder[0]"])
self.format_version = "Microsoft Cabinet version %s" % cab["cab_version"].display
self.comment = "%s folders, %s files" % (
cab["nb_folder"].value, cab["nb_files"].value)
max_nb = maxNbFile(self)
for index, field in enumerate(cab.array("file")):
if max_nb is not None and max_nb <= index:
self.warning("CAB archive contains many files, but only first %s files are processed" % max_nb)
break
self.useFile(field)
@fault_tolerant
def useFolder(self, folder):
compr = folder["compr_method"].display
if folder["compr_method"].value != 0:
compr += " (level %u)" % folder["compr_level"].value
self.compression = compr
@fault_tolerant
def useFile(self, field):
meta = Metadata(self)
meta.filename = field["filename"].value
meta.file_size = field["filesize"].value
meta.creation_date = field["timestamp"].value
attr = field["attributes"].value
if attr != "(none)":
meta.file_attr = attr
if meta.has("filename"):
title = _("File \"%s\"") % meta.getText('filename')
else:
title = _("File")
self.addGroup(field.name, meta, title)
class MarMetadata(MultipleMetadata):
def extract(self, mar):
self.comment = "Contains %s files" % mar["nb_file"].value
self.format_version = "Microsoft Archive version %s" % mar["version"].value
max_nb = maxNbFile(self)
for index, field in enumerate(mar.array("file")):
if max_nb is not None and max_nb <= index:
self.warning("MAR archive contains many files, but only first %s files are processed" % max_nb)
break
meta = Metadata(self)
meta.filename = field["filename"].value
meta.compression = "None"
meta.file_size = field["filesize"].value
self.addGroup(field.name, meta, "File \"%s\"" % meta.getText('filename'))
registerExtractor(CabFile, CabMetadata)
registerExtractor(GzipParser, GzipMetadata)
registerExtractor(Bzip2Parser, Bzip2Metadata)
registerExtractor(TarFile, TarMetadata)
registerExtractor(ZipFile, ZipMetadata)
registerExtractor(MarFile, MarMetadata)
libs/hachoir_metadata/audio.py
-406
View File
@@ -1,406 +0,0 @@
from hachoir_metadata.metadata import (registerExtractor,
Metadata, RootMetadata, MultipleMetadata)
from hachoir_parser.audio import AuFile, MpegAudioFile, RealAudioFile, AiffFile, FlacParser
from hachoir_parser.container import OggFile, RealMediaFile
from hachoir_core.i18n import _
from hachoir_core.tools import makePrintable, timedelta2seconds, humanBitRate
from datetime import timedelta
from hachoir_metadata.metadata_item import QUALITY_FAST, QUALITY_NORMAL, QUALITY_BEST
from hachoir_metadata.safe import fault_tolerant, getValue
def computeComprRate(meta, size):
if not meta.has("duration") \
or not meta.has("sample_rate") \
or not meta.has("bits_per_sample") \
or not meta.has("nb_channel") \
or not size:
return
orig_size = timedelta2seconds(meta.get("duration")) * meta.get('sample_rate') * meta.get('bits_per_sample') * meta.get('nb_channel')
meta.compr_rate = float(orig_size) / size
def computeBitRate(meta):
if not meta.has("bits_per_sample") \
or not meta.has("nb_channel") \
or not meta.has("sample_rate"):
return
meta.bit_rate = meta.get('bits_per_sample') * meta.get('nb_channel') * meta.get('sample_rate')
VORBIS_KEY_TO_ATTR = {
"ARTIST": "artist",
"ALBUM": "album",
"TRACKNUMBER": "track_number",
"TRACKTOTAL": "track_total",
"ENCODER": "producer",
"TITLE": "title",
"LOCATION": "location",
"DATE": "creation_date",
"ORGANIZATION": "organization",
"GENRE": "music_genre",
"": "comment",
"COMPOSER": "music_composer",
"DESCRIPTION": "comment",
"COMMENT": "comment",
"WWW": "url",
"WOAF": "url",
"LICENSE": "copyright",
}
@fault_tolerant
def readVorbisComment(metadata, comment):
metadata.producer = getValue(comment, "vendor")
for item in comment.array("metadata"):
if "=" in item.value:
key, value = item.value.split("=", 1)
key = key.upper()
if key in VORBIS_KEY_TO_ATTR:
key = VORBIS_KEY_TO_ATTR[key]
setattr(metadata, key, value)
elif value:
metadata.warning("Skip Vorbis comment %s: %s" % (key, value))
class OggMetadata(MultipleMetadata):
def extract(self, ogg):
granule_quotient = None
for index, page in enumerate(ogg.array("page")):
if "segments" not in page:
continue
page = page["segments"]
if "vorbis_hdr" in page:
meta = Metadata(self)
self.vorbisHeader(page["vorbis_hdr"], meta)
self.addGroup("audio[]", meta, "Audio")
if not granule_quotient and meta.has("sample_rate"):
granule_quotient = meta.get('sample_rate')
if "theora_hdr" in page:
meta = Metadata(self)
self.theoraHeader(page["theora_hdr"], meta)
self.addGroup("video[]", meta, "Video")
if "video_hdr" in page:
meta = Metadata(self)
self.videoHeader(page["video_hdr"], meta)
self.addGroup("video[]", meta, "Video")
if not granule_quotient and meta.has("frame_rate"):
granule_quotient = meta.get('frame_rate')
if "comment" in page:
readVorbisComment(self, page["comment"])
if 3 <= index:
# Only process pages 0..3
break
# Compute duration
if granule_quotient and QUALITY_NORMAL <= self.quality:
page = ogg.createLastPage()
if page and "abs_granule_pos" in page:
try:
self.duration = timedelta(seconds=float(page["abs_granule_pos"].value) / granule_quotient)
except OverflowError:
pass
def videoHeader(self, header, meta):
meta.compression = header["fourcc"].display
meta.width = header["width"].value
meta.height = header["height"].value
meta.bits_per_pixel = header["bits_per_sample"].value
if header["time_unit"].value:
meta.frame_rate = 10000000.0 / header["time_unit"].value
def theoraHeader(self, header, meta):
meta.compression = "Theora"
meta.format_version = "Theora version %u.%u (revision %u)" % (\
header["version_major"].value,
header["version_minor"].value,
header["version_revision"].value)
meta.width = header["frame_width"].value
meta.height = header["frame_height"].value
if header["fps_den"].value:
meta.frame_rate = float(header["fps_num"].value) / header["fps_den"].value
if header["aspect_ratio_den"].value:
meta.aspect_ratio = float(header["aspect_ratio_num"].value) / header["aspect_ratio_den"].value
meta.pixel_format = header["pixel_format"].display
meta.comment = "Quality: %s" % header["quality"].value
def vorbisHeader(self, header, meta):
meta.compression = u"Vorbis"
meta.sample_rate = header["audio_sample_rate"].value
meta.nb_channel = header["audio_channels"].value
meta.format_version = u"Vorbis version %s" % header["vorbis_version"].value
meta.bit_rate = header["bitrate_nominal"].value
class AuMetadata(RootMetadata):
def extract(self, audio):
self.sample_rate = audio["sample_rate"].value
self.nb_channel = audio["channels"].value
self.compression = audio["codec"].display
if "info" in audio:
self.comment = audio["info"].value
self.bits_per_sample = audio.getBitsPerSample()
computeBitRate(self)
if "audio_data" in audio:
if self.has("bit_rate"):
self.duration = timedelta(seconds=float(audio["audio_data"].size) / self.get('bit_rate'))
computeComprRate(self, audio["audio_data"].size)
class RealAudioMetadata(RootMetadata):
FOURCC_TO_BITRATE = {
u"28_8": 15200, # 28.8 kbit/sec (audio bit rate: 15.2 kbit/s)
u"14_4": 8000, # 14.4 kbit/sec
u"lpcJ": 8000, # 14.4 kbit/sec
}
def extract(self, real):
version = real["version"].value
if "metadata" in real:
self.useMetadata(real["metadata"])
self.useRoot(real)
self.format_version = "Real audio version %s" % version
if version == 3:
size = getValue(real, "data_size")
elif "filesize" in real and "headersize" in real:
size = (real["filesize"].value + 40) - (real["headersize"].value + 16)
else:
size = None
if size:
size *= 8
if self.has("bit_rate"):
sec = float(size) / self.get('bit_rate')
self.duration = timedelta(seconds=sec)
computeComprRate(self, size)
@fault_tolerant
def useMetadata(self, info):
self.title = info["title"].value
self.author = info["author"].value
self.copyright = info["copyright"].value
self.comment = info["comment"].value
@fault_tolerant
def useRoot(self, real):
self.bits_per_sample = 16 # FIXME: Is that correct?
if real["version"].value != 3:
self.sample_rate = real["sample_rate"].value
self.nb_channel = real["channels"].value
else:
self.sample_rate = 8000
self.nb_channel = 1
fourcc = getValue(real, "FourCC")
if fourcc:
self.compression = fourcc
try:
self.bit_rate = self.FOURCC_TO_BITRATE[fourcc]
except LookupError:
pass
class RealMediaMetadata(MultipleMetadata):
KEY_TO_ATTR = {
"generated by": "producer",
"creation date": "creation_date",
"modification date": "last_modification",
"description": "comment",
}
def extract(self, media):
if "file_prop" in media:
self.useFileProp(media["file_prop"])
if "content_desc" in media:
self.useContentDesc(media["content_desc"])
for index, stream in enumerate(media.array("stream_prop")):
self.useStreamProp(stream, index)
@fault_tolerant
def useFileInfoProp(self, prop):
key = prop["name"].value.lower()
value = prop["value"].value
if key in self.KEY_TO_ATTR:
setattr(self, self.KEY_TO_ATTR[key], value)
elif value:
self.warning("Skip %s: %s" % (prop["name"].value, value))
@fault_tolerant
def useFileProp(self, prop):
self.bit_rate = prop["avg_bit_rate"].value
self.duration = timedelta(milliseconds=prop["duration"].value)
@fault_tolerant
def useContentDesc(self, content):
self.title = content["title"].value
self.author = content["author"].value
self.copyright = content["copyright"].value
self.comment = content["comment"].value
@fault_tolerant
def useStreamProp(self, stream, index):
meta = Metadata(self)
meta.comment = "Start: %s" % stream["stream_start"].value
if getValue(stream, "mime_type") == "logical-fileinfo":
for prop in stream.array("file_info/prop"):
self.useFileInfoProp(prop)
else:
meta.bit_rate = stream["avg_bit_rate"].value
meta.duration = timedelta(milliseconds=stream["duration"].value)
meta.mime_type = getValue(stream, "mime_type")
meta.title = getValue(stream, "desc")
self.addGroup("stream[%u]" % index, meta, "Stream #%u" % (1+index))
class MpegAudioMetadata(RootMetadata):
TAG_TO_KEY = {
# ID3 version 2.2
"TP1": "author",
"COM": "comment",
"TEN": "producer",
"TRK": "track_number",
"TAL": "album",
"TT2": "title",
"TYE": "creation_date",
"TCO": "music_genre",
# ID3 version 2.3+
"TPE1": "author",
"COMM": "comment",
"TENC": "producer",
"TRCK": "track_number",
"TALB": "album",
"TIT2": "title",
"TYER": "creation_date",
"WXXX": "url",
"TCON": "music_genre",
"TLAN": "language",
"TCOP": "copyright",
"TDAT": "creation_date",
"TRDA": "creation_date",
"TORY": "creation_date",
"TIT1": "title",
}
def processID3v2(self, field):
# Read value
if "content" not in field:
return
content = field["content"]
if "text" not in content:
return
if "title" in content and content["title"].value:
value = "%s: %s" % (content["title"].value, content["text"].value)
else:
value = content["text"].value
# Known tag?
tag = field["tag"].value
if tag not in self.TAG_TO_KEY:
if tag:
if isinstance(tag, str):
tag = makePrintable(tag, "ISO-8859-1", to_unicode=True)
self.warning("Skip ID3v2 tag %s: %s" % (tag, value))
return
key = self.TAG_TO_KEY[tag]
setattr(self, key, value)
def readID3v2(self, id3):
for field in id3:
if field.is_field_set and "tag" in field:
self.processID3v2(field)
def extract(self, mp3):
if "/frames/frame[0]" in mp3:
frame = mp3["/frames/frame[0]"]
self.nb_channel = (frame.getNbChannel(), frame["channel_mode"].display)
self.format_version = u"MPEG version %s layer %s" % \
(frame["version"].display, frame["layer"].display)
self.sample_rate = frame.getSampleRate()
self.bits_per_sample = 16
if mp3["frames"].looksConstantBitRate():
self.computeBitrate(frame)
else:
self.computeVariableBitrate(mp3)
if "id3v1" in mp3:
id3 = mp3["id3v1"]
self.comment = id3["comment"].value
self.author = id3["author"].value
self.title = id3["song"].value
self.album = id3["album"].value
if id3["year"].value != "0":
self.creation_date = id3["year"].value
if "track_nb" in id3:
self.track_number = id3["track_nb"].value
if "id3v2" in mp3:
self.readID3v2(mp3["id3v2"])
if "frames" in mp3:
computeComprRate(self, mp3["frames"].size)
def computeBitrate(self, frame):
bit_rate = frame.getBitRate() # may returns None on error
if not bit_rate:
return
self.bit_rate = (bit_rate, _("%s (constant)") % humanBitRate(bit_rate))
self.duration = timedelta(seconds=float(frame["/frames"].size) / bit_rate)
def computeVariableBitrate(self, mp3):
if self.quality <= QUALITY_FAST:
return
count = 0
if QUALITY_BEST <= self.quality:
self.warning("Process all MPEG audio frames to compute exact duration")
max_count = None
else:
max_count = 500 * self.quality
total_bit_rate = 0.0
for index, frame in enumerate(mp3.array("frames/frame")):
if index < 3:
continue
bit_rate = frame.getBitRate()
if bit_rate:
total_bit_rate += float(bit_rate)
count += 1
if max_count and max_count <= count:
break
if not count:
return
bit_rate = total_bit_rate / count
self.bit_rate = (bit_rate,
_("%s (Variable bit rate)") % humanBitRate(bit_rate))
duration = timedelta(seconds=float(mp3["frames"].size) / bit_rate)
self.duration = duration
class AiffMetadata(RootMetadata):
def extract(self, aiff):
if "common" in aiff:
self.useCommon(aiff["common"])
computeBitRate(self)
@fault_tolerant
def useCommon(self, info):
self.nb_channel = info["nb_channel"].value
self.bits_per_sample = info["sample_size"].value
self.sample_rate = getValue(info, "sample_rate")
if self.has("sample_rate"):
rate = self.get("sample_rate")
if rate:
sec = float(info["nb_sample"].value) / rate
self.duration = timedelta(seconds=sec)
if "codec" in info:
self.compression = info["codec"].display
class FlacMetadata(RootMetadata):
def extract(self, flac):
if "metadata/stream_info/content" in flac:
self.useStreamInfo(flac["metadata/stream_info/content"])
if "metadata/comment/content" in flac:
readVorbisComment(self, flac["metadata/comment/content"])
@fault_tolerant
def useStreamInfo(self, info):
self.nb_channel = info["nb_channel"].value + 1
self.bits_per_sample = info["bits_per_sample"].value + 1
self.sample_rate = info["sample_hertz"].value
sec = info["total_samples"].value
if sec:
sec = float(sec) / info["sample_hertz"].value
self.duration = timedelta(seconds=sec)
registerExtractor(AuFile, AuMetadata)
registerExtractor(MpegAudioFile, MpegAudioMetadata)
registerExtractor(OggFile, OggMetadata)
registerExtractor(RealMediaFile, RealMediaMetadata)
registerExtractor(RealAudioFile, RealAudioMetadata)
registerExtractor(AiffFile, AiffMetadata)
registerExtractor(FlacParser, FlacMetadata)
libs/hachoir_metadata/config.py
-2
View File
@@ -1,2 +0,0 @@
MAX_STR_LENGTH = 300 # characters
RAW_OUTPUT = False
libs/hachoir_metadata/file_system.py
-28
View File
@@ -1,28 +0,0 @@
from hachoir_metadata.metadata import RootMetadata, registerExtractor
from hachoir_metadata.safe import fault_tolerant
from hachoir_parser.file_system import ISO9660
from datetime import datetime
class ISO9660_Metadata(RootMetadata):
def extract(self, iso):
desc = iso['volume[0]/content']
self.title = desc['volume_id'].value
self.title = desc['vol_set_id'].value
self.author = desc['publisher'].value
self.author = desc['data_preparer'].value
self.producer = desc['application'].value
self.copyright = desc['copyright'].value
self.readTimestamp('creation_date', desc['creation_ts'].value)
self.readTimestamp('last_modification', desc['modification_ts'].value)
@fault_tolerant
def readTimestamp(self, key, value):
if value.startswith("0000"):
return
value = datetime(
int(value[0:4]), int(value[4:6]), int(value[6:8]),
int(value[8:10]), int(value[10:12]), int(value[12:14]))
setattr(self, key, value)
registerExtractor(ISO9660, ISO9660_Metadata)
libs/hachoir_metadata/filter.py
-52
View File
@@ -1,52 +0,0 @@
from hachoir_metadata.timezone import UTC
from datetime import date, datetime
# Year in 1850..2030
MIN_YEAR = 1850
MAX_YEAR = 2030
class Filter:
def __init__(self, valid_types, min=None, max=None):
self.types = valid_types
self.min = min
self.max = max
def __call__(self, value):
if not isinstance(value, self.types):
return True
if self.min is not None and value < self.min:
return False
if self.max is not None and self.max < value:
return False
return True
class NumberFilter(Filter):
def __init__(self, min=None, max=None):
Filter.__init__(self, (int, long, float), min, max)
class DatetimeFilter(Filter):
def __init__(self, min=None, max=None):
Filter.__init__(self, (date, datetime),
datetime(MIN_YEAR, 1, 1),
datetime(MAX_YEAR, 12, 31))
self.min_date = date(MIN_YEAR, 1, 1)
self.max_date = date(MAX_YEAR, 12, 31)
self.min_tz = datetime(MIN_YEAR, 1, 1, tzinfo=UTC)
self.max_tz = datetime(MAX_YEAR, 12, 31, tzinfo=UTC)
def __call__(self, value):
"""
Use different min/max values depending on value type
(datetime with timezone, datetime or date).
"""
if not isinstance(value, self.types):
return True
if hasattr(value, "tzinfo") and value.tzinfo:
return (self.min_tz <= value <= self.max_tz)
elif isinstance(value, datetime):
return (self.min <= value <= self.max)
else:
return (self.min_date <= value <= self.max_date)
DATETIME_FILTER = DatetimeFilter()
libs/hachoir_metadata/formatter.py
-25
View File
@@ -1,25 +0,0 @@
from hachoir_core.i18n import _, ngettext
NB_CHANNEL_NAME = {1: _("mono"), 2: _("stereo")}
def humanAudioChannel(value):
return NB_CHANNEL_NAME.get(value, unicode(value))
def humanFrameRate(value):
if isinstance(value, (int, long, float)):
return _("%.1f fps") % value
else:
return value
def humanComprRate(rate):
return u"%.1fx" % rate
def humanAltitude(value):
return ngettext("%.1f meter", "%.1f meters", value) % value
def humanPixelSize(value):
return ngettext("%s pixel", "%s pixels", value) % value
def humanDPI(value):
return u"%s DPI" % value
libs/hachoir_metadata/image.py
-299
View File
@@ -1,299 +0,0 @@
from hachoir_metadata.metadata import (registerExtractor,
Metadata, RootMetadata, MultipleMetadata)
from hachoir_parser.image import (
BmpFile, IcoFile, PcxFile, GifFile, PngFile, TiffFile,
XcfFile, TargaFile, WMF_File, PsdFile)
from hachoir_parser.image.png import getBitsPerPixel as pngBitsPerPixel
from hachoir_parser.image.xcf import XcfProperty
from hachoir_core.i18n import _
from hachoir_metadata.safe import fault_tolerant
def computeComprRate(meta, compr_size):
"""
Compute image compression rate. Skip size of color palette, focus on
image pixels. Original size is width x height x bpp. Compressed size
is an argument (in bits).
Set "compr_data" with a string like "1.52x".
"""
if not meta.has("width") \
or not meta.has("height") \
or not meta.has("bits_per_pixel"):
return
if not compr_size:
return
orig_size = meta.get('width') * meta.get('height') * meta.get('bits_per_pixel')
meta.compr_rate = float(orig_size) / compr_size
class BmpMetadata(RootMetadata):
def extract(self, image):
if "header" not in image:
return
hdr = image["header"]
self.width = hdr["width"].value
self.height = hdr["height"].value
bpp = hdr["bpp"].value
if bpp:
if bpp <= 8 and "used_colors" in hdr:
self.nb_colors = hdr["used_colors"].value
self.bits_per_pixel = bpp
self.compression = hdr["compression"].display
self.format_version = u"Microsoft Bitmap version %s" % hdr.getFormatVersion()
self.width_dpi = hdr["horizontal_dpi"].value
self.height_dpi = hdr["vertical_dpi"].value
if "pixels" in image:
computeComprRate(self, image["pixels"].size)
class TiffMetadata(RootMetadata):
key_to_attr = {
"img_width": "width",
"img_height": "width",
# TODO: Enable that (need link to value)
# "description": "comment",
# "doc_name": "title",
# "orientation": "image_orientation",
}
def extract(self, tiff):
if "ifd" in tiff:
self.useIFD(tiff["ifd"])
def useIFD(self, ifd):
for field in ifd:
try:
attrname = self.key_to_attr[field.name]
except KeyError:
continue
if "value" not in field:
continue
value = field["value"].value
setattr(self, attrname, value)
class IcoMetadata(MultipleMetadata):
color_to_bpp = {
2: 1,
16: 4,
256: 8
}
def extract(self, icon):
for index, header in enumerate(icon.array("icon_header")):
image = Metadata(self)
# Read size and colors from header
image.width = header["width"].value
image.height = header["height"].value
bpp = header["bpp"].value
nb_colors = header["nb_color"].value
if nb_colors != 0:
image.nb_colors = nb_colors
if bpp == 0 and nb_colors in self.color_to_bpp:
bpp = self.color_to_bpp[nb_colors]
elif bpp == 0:
bpp = 8
image.bits_per_pixel = bpp
image.setHeader(_("Icon #%u (%sx%s)")
% (1+index, image.get("width", "?"), image.get("height", "?")))
# Read compression from data (if available)
key = "icon_data[%u]/header/codec" % index
if key in icon:
image.compression = icon[key].display
key = "icon_data[%u]/pixels" % index
if key in icon:
computeComprRate(image, icon[key].size)
# Store new image
self.addGroup("image[%u]" % index, image)
class PcxMetadata(RootMetadata):
@fault_tolerant
def extract(self, pcx):
self.width = 1 + pcx["xmax"].value
self.height = 1 + pcx["ymax"].value
self.width_dpi = pcx["horiz_dpi"].value
self.height_dpi = pcx["vert_dpi"].value
self.bits_per_pixel = pcx["bpp"].value
if 1 <= pcx["bpp"].value <= 8:
self.nb_colors = 2 ** pcx["bpp"].value
self.compression = _("Run-length encoding (RLE)")
self.format_version = "PCX: %s" % pcx["version"].display
if "image_data" in pcx:
computeComprRate(self, pcx["image_data"].size)
class XcfMetadata(RootMetadata):
# Map image type to bits/pixel
TYPE_TO_BPP = {0: 24, 1: 8, 2: 8}
def extract(self, xcf):
self.width = xcf["width"].value
self.height = xcf["height"].value
try:
self.bits_per_pixel = self.TYPE_TO_BPP[ xcf["type"].value ]
except KeyError:
pass
self.format_version = xcf["type"].display
self.readProperties(xcf)
@fault_tolerant
def processProperty(self, prop):
type = prop["type"].value
if type == XcfProperty.PROP_PARASITES:
for field in prop["data"]:
if "name" not in field or "data" not in field:
continue
if field["name"].value == "gimp-comment":
self.comment = field["data"].value
elif type == XcfProperty.PROP_COMPRESSION:
self.compression = prop["data/compression"].display
elif type == XcfProperty.PROP_RESOLUTION:
self.width_dpi = int(prop["data/xres"].value)
self.height_dpi = int(prop["data/yres"].value)
def readProperties(self, xcf):
for prop in xcf.array("property"):
self.processProperty(prop)
class PngMetadata(RootMetadata):
TEXT_TO_ATTR = {
"software": "producer",
}
def extract(self, png):
if "header" in png:
self.useHeader(png["header"])
if "time" in png:
self.useTime(png["time"])
if "physical" in png:
self.usePhysical(png["physical"])
for comment in png.array("text"):
if "text" not in comment:
continue
keyword = comment["keyword"].value
text = comment["text"].value
try:
key = self.TEXT_TO_ATTR[keyword.lower()]
setattr(self, key, text)
except KeyError:
if keyword.lower() != "comment":
self.comment = "%s=%s" % (keyword, text)
else:
self.comment = text
compr_size = sum( data.size for data in png.array("data") )
computeComprRate(self, compr_size)
@fault_tolerant
def useTime(self, field):
self.creation_date = field.value
@fault_tolerant
def usePhysical(self, field):
self.width_dpi = field["pixel_per_unit_x"].value
self.height_dpi = field["pixel_per_unit_y"].value
@fault_tolerant
def useHeader(self, header):
self.width = header["width"].value
self.height = header["height"].value
# Read number of colors and pixel format
if "/palette/size" in header:
nb_colors = header["/palette/size"].value // 3
else:
nb_colors = None
if not header["has_palette"].value:
if header["has_alpha"].value:
self.pixel_format = _("RGBA")
else:
self.pixel_format = _("RGB")
elif "/transparency" in header:
self.pixel_format = _("Color index with transparency")
if nb_colors:
nb_colors -= 1
else:
self.pixel_format = _("Color index")
self.bits_per_pixel = pngBitsPerPixel(header)
if nb_colors:
self.nb_colors = nb_colors
# Read compression, timestamp, etc.
self.compression = header["compression"].display
class GifMetadata(RootMetadata):
def extract(self, gif):
self.useScreen(gif["/screen"])
if self.has("bits_per_pixel"):
self.nb_colors = (1 << self.get('bits_per_pixel'))
self.compression = _("LZW")
self.format_version = "GIF version %s" % gif["version"].value
for comments in gif.array("comments"):
for comment in gif.array(comments.name + "/comment"):
self.comment = comment.value
if "graphic_ctl/has_transp" in gif and gif["graphic_ctl/has_transp"].value:
self.pixel_format = _("Color index with transparency")
else:
self.pixel_format = _("Color index")
@fault_tolerant
def useScreen(self, screen):
self.width = screen["width"].value
self.height = screen["height"].value
self.bits_per_pixel = (1 + screen["bpp"].value)
class TargaMetadata(RootMetadata):
def extract(self, tga):
self.width = tga["width"].value
self.height = tga["height"].value
self.bits_per_pixel = tga["bpp"].value
if tga["nb_color"].value:
self.nb_colors = tga["nb_color"].value
self.compression = tga["codec"].display
if "pixels" in tga:
computeComprRate(self, tga["pixels"].size)
class WmfMetadata(RootMetadata):
def extract(self, wmf):
if wmf.isAPM():
if "amf_header/rect" in wmf:
rect = wmf["amf_header/rect"]
self.width = (rect["right"].value - rect["left"].value)
self.height = (rect["bottom"].value - rect["top"].value)
self.bits_per_pixel = 24
elif wmf.isEMF():
emf = wmf["emf_header"]
if "description" in emf:
desc = emf["description"].value
if "\0" in desc:
self.producer, self.title = desc.split("\0", 1)
else:
self.producer = desc
if emf["nb_colors"].value:
self.nb_colors = emf["nb_colors"].value
self.bits_per_pixel = 8
else:
self.bits_per_pixel = 24
self.width = emf["width_px"].value
self.height = emf["height_px"].value
class PsdMetadata(RootMetadata):
@fault_tolerant
def extract(self, psd):
self.width = psd["width"].value
self.height = psd["height"].value
self.bits_per_pixel = psd["depth"].value * psd["nb_channels"].value
self.pixel_format = psd["color_mode"].display
self.compression = psd["compression"].display
registerExtractor(IcoFile, IcoMetadata)
registerExtractor(GifFile, GifMetadata)
registerExtractor(XcfFile, XcfMetadata)
registerExtractor(TargaFile, TargaMetadata)
registerExtractor(PcxFile, PcxMetadata)
registerExtractor(BmpFile, BmpMetadata)
registerExtractor(PngFile, PngMetadata)
registerExtractor(TiffFile, TiffMetadata)
registerExtractor(WMF_File, WmfMetadata)
registerExtractor(PsdFile, PsdMetadata)
libs/hachoir_metadata/jpeg.py
-289
View File
@@ -1,289 +0,0 @@
from hachoir_metadata.metadata import RootMetadata, registerExtractor
from hachoir_metadata.image import computeComprRate
from hachoir_parser.image.exif import ExifEntry
from hachoir_parser.image.jpeg import (
JpegFile, JpegChunk,
QUALITY_HASH_COLOR, QUALITY_SUM_COLOR,
QUALITY_HASH_GRAY, QUALITY_SUM_GRAY)
from hachoir_core.field import MissingField
from hachoir_core.i18n import _
from hachoir_core.tools import makeUnicode
from hachoir_metadata.safe import fault_tolerant
from datetime import datetime
def deg2float(degree, minute, second):
return degree + (float(minute) + float(second) / 60.0) / 60.0
class JpegMetadata(RootMetadata):
EXIF_KEY = {
# Exif metadatas
ExifEntry.TAG_CAMERA_MANUFACTURER: "camera_manufacturer",
ExifEntry.TAG_CAMERA_MODEL: "camera_model",
ExifEntry.TAG_ORIENTATION: "image_orientation",
ExifEntry.TAG_EXPOSURE: "camera_exposure",
ExifEntry.TAG_FOCAL: "camera_focal",
ExifEntry.TAG_BRIGHTNESS: "camera_brightness",
ExifEntry.TAG_APERTURE: "camera_aperture",
# Generic metadatas
ExifEntry.TAG_IMG_TITLE: "title",
ExifEntry.TAG_SOFTWARE: "producer",
ExifEntry.TAG_FILE_TIMESTAMP: "creation_date",
ExifEntry.TAG_WIDTH: "width",
ExifEntry.TAG_HEIGHT: "height",
ExifEntry.TAG_USER_COMMENT: "comment",
}
IPTC_KEY = {
80: "author",
90: "city",
101: "country",
116: "copyright",
120: "title",
231: "comment",
}
orientation_name = {
1: _('Horizontal (normal)'),
2: _('Mirrored horizontal'),
3: _('Rotated 180'),
4: _('Mirrored vertical'),
5: _('Mirrored horizontal then rotated 90 counter-clock-wise'),
6: _('Rotated 90 clock-wise'),
7: _('Mirrored horizontal then rotated 90 clock-wise'),
8: _('Rotated 90 counter clock-wise'),
}
def extract(self, jpeg):
if "start_frame/content" in jpeg:
self.startOfFrame(jpeg["start_frame/content"])
elif "start_scan/content/nr_components" in jpeg:
self.bits_per_pixel = 8 * jpeg["start_scan/content/nr_components"].value
if "app0/content" in jpeg:
self.extractAPP0(jpeg["app0/content"])
if "exif/content" in jpeg:
for ifd in jpeg.array("exif/content/ifd"):
for entry in ifd.array("entry"):
self.processIfdEntry(ifd, entry)
self.readGPS(ifd)
if "photoshop/content" in jpeg:
psd = jpeg["photoshop/content"]
if "version/content/reader_name" in psd:
self.producer = psd["version/content/reader_name"].value
if "iptc/content" in psd:
self.parseIPTC(psd["iptc/content"])
for field in jpeg.array("comment"):
if "content/comment" in field:
self.comment = field["content/comment"].value
self.computeQuality(jpeg)
if "data" in jpeg:
computeComprRate(self, jpeg["data"].size)
if not self.has("producer") and "photoshop" in jpeg:
self.producer = u"Adobe Photoshop"
if self.has("compression"):
self.compression = "JPEG"
@fault_tolerant
def startOfFrame(self, sof):
# Set compression method
key = sof["../type"].value
self.compression = "JPEG (%s)" % JpegChunk.START_OF_FRAME[key]
# Read image size and bits/pixel
self.width = sof["width"].value
self.height = sof["height"].value
nb_components = sof["nr_components"].value
self.bits_per_pixel = 8 * nb_components
if nb_components == 3:
self.pixel_format = _("YCbCr")
elif nb_components == 1:
self.pixel_format = _("Grayscale")
self.nb_colors = 256
@fault_tolerant
def computeQuality(self, jpeg):
# This function is an adaption to Python of ImageMagick code
# to compute JPEG quality using quantization tables
# Read quantization tables
qtlist = []
for dqt in jpeg.array("quantization"):
for qt in dqt.array("content/qt"):
# TODO: Take care of qt["index"].value?
qtlist.append(qt)
if not qtlist:
return
# Compute sum of all coefficients
sumcoeff = 0
for qt in qtlist:
coeff = qt.array("coeff")
for index in xrange(64):
sumcoeff += coeff[index].value
# Choose the right quality table and compute hash value
try:
hashval= qtlist[0]["coeff[2]"].value + qtlist[0]["coeff[53]"].value
if 2 <= len(qtlist):
hashval += qtlist[1]["coeff[0]"].value + qtlist[1]["coeff[63]"].value
hashtable = QUALITY_HASH_COLOR
sumtable = QUALITY_SUM_COLOR
else:
hashtable = QUALITY_HASH_GRAY
sumtable = QUALITY_SUM_GRAY
except (MissingField, IndexError):
# A coefficient is missing, so don't compute JPEG quality
return
# Find the JPEG quality
for index in xrange(100):
if (hashval >= hashtable[index]) or (sumcoeff >= sumtable[index]):
quality = "%s%%" % (index + 1)
if (hashval > hashtable[index]) or (sumcoeff > sumtable[index]):
quality += " " + _("(approximate)")
self.comment = "JPEG quality: %s" % quality
return
@fault_tolerant
def extractAPP0(self, app0):
self.format_version = u"JFIF %u.%02u" \
% (app0["ver_maj"].value, app0["ver_min"].value)
if "y_density" in app0:
self.width_dpi = app0["x_density"].value
self.height_dpi = app0["y_density"].value
@fault_tolerant
def processIfdEntry(self, ifd, entry):
# Skip unknown tags
tag = entry["tag"].value
if tag not in self.EXIF_KEY:
return
key = self.EXIF_KEY[tag]
if key in ("width", "height") and self.has(key):
# EXIF "valid size" are sometimes not updated when the image is scaled
# so we just ignore it
return
# Read value
if "value" in entry:
value = entry["value"].value
else:
value = ifd["value_%s" % entry.name].value
# Convert value to string
if tag == ExifEntry.TAG_ORIENTATION:
value = self.orientation_name.get(value, value)
elif tag == ExifEntry.TAG_EXPOSURE:
if not value:
return
if isinstance(value, float):
value = (value, u"1/%g" % (1/value))
elif entry["type"].value in (ExifEntry.TYPE_RATIONAL, ExifEntry.TYPE_SIGNED_RATIONAL):
value = (value, u"%.3g" % value)
# Store information
setattr(self, key, value)
@fault_tolerant
def readGPS(self, ifd):
# Read latitude and longitude
latitude_ref = None
longitude_ref = None
latitude = None
longitude = None
altitude_ref = 1
altitude = None
timestamp = None
datestamp = None
for entry in ifd.array("entry"):
tag = entry["tag"].value
if tag == ExifEntry.TAG_GPS_LATITUDE_REF:
if entry["value"].value == "N":
latitude_ref = 1
else:
latitude_ref = -1
elif tag == ExifEntry.TAG_GPS_LONGITUDE_REF:
if entry["value"].value == "E":
longitude_ref = 1
else:
longitude_ref = -1
elif tag == ExifEntry.TAG_GPS_ALTITUDE_REF:
if entry["value"].value == 1:
altitude_ref = -1
else:
altitude_ref = 1
elif tag == ExifEntry.TAG_GPS_LATITUDE:
latitude = [ifd["value_%s[%u]" % (entry.name, index)].value for index in xrange(3)]
elif tag == ExifEntry.TAG_GPS_LONGITUDE:
longitude = [ifd["value_%s[%u]" % (entry.name, index)].value for index in xrange(3)]
elif tag == ExifEntry.TAG_GPS_ALTITUDE:
altitude = ifd["value_%s" % entry.name].value
elif tag == ExifEntry.TAG_GPS_DATESTAMP:
datestamp = ifd["value_%s" % entry.name].value
elif tag == ExifEntry.TAG_GPS_TIMESTAMP:
items = [ifd["value_%s[%u]" % (entry.name, index)].value for index in xrange(3)]
items = map(int, items)
items = map(str, items)
timestamp = ":".join(items)
if latitude_ref and latitude:
value = deg2float(*latitude)
if latitude_ref < 0:
value = -value
self.latitude = value
if longitude and longitude_ref:
value = deg2float(*longitude)
if longitude_ref < 0:
value = -value
self.longitude = value
if altitude:
value = altitude
if altitude_ref < 0:
value = -value
self.altitude = value
if datestamp:
if timestamp:
datestamp += " " + timestamp
self.creation_date = datestamp
def parseIPTC(self, iptc):
datestr = hourstr = None
for field in iptc:
# Skip incomplete field
if "tag" not in field or "content" not in field:
continue
# Get value
value = field["content"].value
if isinstance(value, (str, unicode)):
value = value.replace("\r", " ")
value = value.replace("\n", " ")
# Skip unknown tag
tag = field["tag"].value
if tag == 55:
datestr = value
continue
if tag == 60:
hourstr = value
continue
if tag not in self.IPTC_KEY:
if tag != 0:
self.warning("Skip IPTC key %s: %s" % (
field["tag"].display, makeUnicode(value)))
continue
setattr(self, self.IPTC_KEY[tag], value)
if datestr and hourstr:
try:
year = int(datestr[0:4])
month = int(datestr[4:6])
day = int(datestr[6:8])
hour = int(hourstr[0:2])
min = int(hourstr[2:4])
sec = int(hourstr[4:6])
self.creation_date = datetime(year, month, day, hour, min, sec)
except ValueError:
pass
registerExtractor(JpegFile, JpegMetadata)
libs/hachoir_metadata/metadata.py
-291
View File
@@ -1,291 +0,0 @@
# -*- coding: utf-8 -*-
from hachoir_core.compatibility import any, sorted
from hachoir_core.endian import endian_name
from hachoir_core.tools import makePrintable, makeUnicode
from hachoir_core.dict import Dict
from hachoir_core.error import error, HACHOIR_ERRORS
from hachoir_core.i18n import _
from hachoir_core.log import Logger
from hachoir_metadata.metadata_item import (
MIN_PRIORITY, MAX_PRIORITY, QUALITY_NORMAL)
from hachoir_metadata.register import registerAllItems
extractors = {}
class Metadata(Logger):
header = u"Metadata"
def __init__(self, parent, quality=QUALITY_NORMAL):
assert isinstance(self.header, unicode)
# Limit to 0.0 .. 1.0
if parent:
quality = parent.quality
else:
quality = min(max(0.0, quality), 1.0)
object.__init__(self)
object.__setattr__(self, "_Metadata__data", {})
object.__setattr__(self, "quality", quality)
header = self.__class__.header
object.__setattr__(self, "_Metadata__header", header)
registerAllItems(self)
def _logger(self):
pass
def __setattr__(self, key, value):
"""
Add a new value to data with name 'key'. Skip duplicates.
"""
# Invalid key?
if key not in self.__data:
raise KeyError(_("%s has no metadata '%s'") % (self.__class__.__name__, key))
# Skip duplicates
self.__data[key].add(value)
def setHeader(self, text):
object.__setattr__(self, "header", text)
def getItems(self, key):
try:
return self.__data[key]
except LookupError:
raise ValueError("Metadata has no value '%s'" % key)
def getItem(self, key, index):
try:
return self.getItems(key)[index]
except (LookupError, ValueError):
return None
def has(self, key):
return 1 <= len(self.getItems(key))
def get(self, key, default=None, index=0):
"""
Read first value of tag with name 'key'.
>>> from datetime import timedelta
>>> a = RootMetadata()
>>> a.duration = timedelta(seconds=2300)
>>> a.get('duration')
datetime.timedelta(0, 2300)
>>> a.get('author', u'Anonymous')
u'Anonymous'
"""
item = self.getItem(key, index)
if item is None:
if default is None:
raise ValueError("Metadata has no value '%s' (index %s)" % (key, index))
else:
return default
return item.value
def getValues(self, key):
try:
data = self.__data[key]
except LookupError:
raise ValueError("Metadata has no value '%s'" % key)
return [ item.value for item in data ]
def getText(self, key, default=None, index=0):
"""
Read first value, as unicode string, of tag with name 'key'.
>>> from datetime import timedelta
>>> a = RootMetadata()
>>> a.duration = timedelta(seconds=2300)
>>> a.getText('duration')
u'38 min 20 sec'
>>> a.getText('titre', u'Unknown')
u'Unknown'
"""
item = self.getItem(key, index)
if item is not None:
return item.text
else:
return default
def register(self, data):
assert data.key not in self.__data
data.metadata = self
self.__data[data.key] = data
def __iter__(self):
return self.__data.itervalues()
def __str__(self):
r"""
Create a multi-line ASCII string (end of line is "\n") which
represents all datas.
>>> a = RootMetadata()
>>> a.author = "haypo"
>>> a.copyright = unicode("© Hachoir", "UTF-8")
>>> print a
Metadata:
- Author: haypo
- Copyright: \xa9 Hachoir
@see __unicode__() and exportPlaintext()
"""
text = self.exportPlaintext()
return "\n".join( makePrintable(line, "ASCII") for line in text )
def __unicode__(self):
r"""
Create a multi-line Unicode string (end of line is "\n") which
represents all datas.
>>> a = RootMetadata()
>>> a.copyright = unicode("© Hachoir", "UTF-8")
>>> print repr(unicode(a))
u'Metadata:\n- Copyright: \xa9 Hachoir'
@see __str__() and exportPlaintext()
"""
return "\n".join(self.exportPlaintext())
def exportPlaintext(self, priority=None, human=True, line_prefix=u"- ", title=None):
r"""
Convert metadata to multi-line Unicode string and skip datas
with priority lower than specified priority.
Default priority is Metadata.MAX_PRIORITY. If human flag is True, data
key are translated to better human name (eg. "bit_rate" becomes
"Bit rate") which may be translated using gettext.
If priority is too small, metadata are empty and so None is returned.
>>> print RootMetadata().exportPlaintext()
None
>>> meta = RootMetadata()
>>> meta.copyright = unicode("© Hachoir", "UTF-8")
>>> print repr(meta.exportPlaintext())
[u'Metadata:', u'- Copyright: \xa9 Hachoir']
@see __str__() and __unicode__()
"""
if priority is not None:
priority = max(priority, MIN_PRIORITY)
priority = min(priority, MAX_PRIORITY)
else:
priority = MAX_PRIORITY
if not title:
title = self.header
text = ["%s:" % title]
for data in sorted(self):
if priority < data.priority:
break
if not data.values:
continue
if human:
title = data.description
else:
title = data.key
for item in data.values:
if human:
value = item.text
else:
value = makeUnicode(item.value)
text.append("%s%s: %s" % (line_prefix, title, value))
if 1 < len(text):
return text
else:
return None
def __nonzero__(self):
return any(item for item in self.__data.itervalues())
class RootMetadata(Metadata):
def __init__(self, quality=QUALITY_NORMAL):
Metadata.__init__(self, None, quality)
class MultipleMetadata(RootMetadata):
header = _("Common")
def __init__(self, quality=QUALITY_NORMAL):
RootMetadata.__init__(self, quality)
object.__setattr__(self, "_MultipleMetadata__groups", Dict())
object.__setattr__(self, "_MultipleMetadata__key_counter", {})
def __contains__(self, key):
return key in self.__groups
def __getitem__(self, key):
return self.__groups[key]
def iterGroups(self):
return self.__groups.itervalues()
def __nonzero__(self):
if RootMetadata.__nonzero__(self):
return True
return any(bool(group) for group in self.__groups)
def addGroup(self, key, metadata, header=None):
"""
Add a new group (metadata of a sub-document).
Returns False if the group is skipped, True if it has been added.
"""
if not metadata:
self.warning("Skip empty group %s" % key)
return False
if key.endswith("[]"):
key = key[:-2]
if key in self.__key_counter:
self.__key_counter[key] += 1
else:
self.__key_counter[key] = 1
key += "[%u]" % self.__key_counter[key]
if header:
metadata.setHeader(header)
self.__groups.append(key, metadata)
return True
def exportPlaintext(self, priority=None, human=True, line_prefix=u"- "):
common = Metadata.exportPlaintext(self, priority, human, line_prefix)
if common:
text = common
else:
text = []
for key, metadata in self.__groups.iteritems():
if not human:
title = key
else:
title = None
value = metadata.exportPlaintext(priority, human, line_prefix, title=title)
if value:
text.extend(value)
if len(text):
return text
else:
return None
def registerExtractor(parser, extractor):
assert parser not in extractors
assert issubclass(extractor, RootMetadata)
extractors[parser] = extractor
def extractMetadata(parser, quality=QUALITY_NORMAL):
"""
Create a Metadata class from a parser. Returns None if no metadata
extractor does exist for the parser class.
"""
try:
extractor = extractors[parser.__class__]
except KeyError:
return None
metadata = extractor(quality)
try:
metadata.extract(parser)
except HACHOIR_ERRORS, err:
error("Error during metadata extraction: %s" % unicode(err))
if metadata:
metadata.mime_type = parser.mime_type
metadata.endian = endian_name[parser.endian]
return metadata
libs/hachoir_metadata/metadata_item.py
-146
View File
@@ -1,146 +0,0 @@
from hachoir_core.tools import makeUnicode, normalizeNewline
from hachoir_core.error import HACHOIR_ERRORS
from hachoir_metadata import config
from hachoir_metadata.setter import normalizeString
MIN_PRIORITY = 100
MAX_PRIORITY = 999
QUALITY_FASTEST = 0.0
QUALITY_FAST = 0.25
QUALITY_NORMAL = 0.5
QUALITY_GOOD = 0.75
QUALITY_BEST = 1.0
class DataValue:
def __init__(self, value, text):
self.value = value
self.text = text
class Data:
def __init__(self, key, priority, description,
text_handler=None, type=None, filter=None, conversion=None):
"""
handler is only used if value is not string nor unicode, prototype:
def handler(value) -> str/unicode
"""
assert MIN_PRIORITY <= priority <= MAX_PRIORITY
assert isinstance(description, unicode)
self.metadata = None
self.key = key
self.description = description
self.values = []
if type and not isinstance(type, (tuple, list)):
type = (type,)
self.type = type
self.text_handler = text_handler
self.filter = filter
self.priority = priority
self.conversion = conversion
def _createItem(self, value, text=None):
if text is None:
if isinstance(value, unicode):
text = value
elif self.text_handler:
text = self.text_handler(value)
assert isinstance(text, unicode)
else:
text = makeUnicode(value)
return DataValue(value, text)
def add(self, value):
if isinstance(value, tuple):
if len(value) != 2:
raise ValueError("Data.add() only accept tuple of 2 elements: (value,text)")
value, text = value
else:
text = None
# Skip value 'None'
if value is None:
return
if isinstance(value, (str, unicode)):
value = normalizeString(value)
if not value:
return
# Convert string to Unicode string using charset ISO-8859-1
if self.conversion:
try:
new_value = self.conversion(self.metadata, self.key, value)
except HACHOIR_ERRORS, err:
self.metadata.warning("Error during conversion of %r value: %s" % (
self.key, err))
return
if new_value is None:
dest_types = " or ".join(str(item.__name__) for item in self.type)
self.metadata.warning("Unable to convert %s=%r (%s) to %s" % (
self.key, value, type(value).__name__, dest_types))
return
if isinstance(new_value, tuple):
if text:
value = new_value[0]
else:
value, text = new_value
else:
value = new_value
elif isinstance(value, str):
value = unicode(value, "ISO-8859-1")
if self.type and not isinstance(value, self.type):
dest_types = " or ".join(str(item.__name__) for item in self.type)
self.metadata.warning("Key %r: value %r type (%s) is not %s" % (
self.key, value, type(value).__name__, dest_types))
return
# Skip empty strings
if isinstance(value, unicode):
value = normalizeNewline(value)
if config.MAX_STR_LENGTH \
and config.MAX_STR_LENGTH < len(value):
value = value[:config.MAX_STR_LENGTH] + "(...)"
# Skip duplicates
if value in self:
return
# Use filter
if self.filter and not self.filter(value):
self.metadata.warning("Skip value %s=%r (filter)" % (self.key, value))
return
# For string, if you have "verlongtext" and "verylo",
# keep the longer value
if isinstance(value, unicode):
for index, item in enumerate(self.values):
item = item.value
if not isinstance(item, unicode):
continue
if value.startswith(item):
# Find longer value, replace the old one
self.values[index] = self._createItem(value, text)
return
if item.startswith(value):
# Find truncated value, skip it
return
# Add new value
self.values.append(self._createItem(value, text))
def __len__(self):
return len(self.values)
def __getitem__(self, index):
return self.values[index]
def __contains__(self, value):
for item in self.values:
if value == item.value:
return True
return False
def __cmp__(self, other):
return cmp(self.priority, other.priority)
libs/hachoir_metadata/misc.py
-262
View File
@@ -1,262 +0,0 @@
from hachoir_metadata.metadata import RootMetadata, registerExtractor
from hachoir_metadata.safe import fault_tolerant
from hachoir_parser.container import SwfFile
from hachoir_parser.misc import TorrentFile, TrueTypeFontFile, OLE2_File, PcfFile
from hachoir_core.field import isString
from hachoir_core.error import warning
from hachoir_parser import guessParser
from hachoir_metadata.setter import normalizeString
class TorrentMetadata(RootMetadata):
KEY_TO_ATTR = {
u"announce": "url",
u"comment": "comment",
u"creation_date": "creation_date",
}
INFO_TO_ATTR = {
u"length": "file_size",
u"name": "filename",
}
def extract(self, torrent):
for field in torrent[0]:
self.processRoot(field)
@fault_tolerant
def processRoot(self, field):
if field.name in self.KEY_TO_ATTR:
key = self.KEY_TO_ATTR[field.name]
value = field.value
setattr(self, key, value)
elif field.name == "info" and "value" in field:
for field in field["value"]:
self.processInfo(field)
@fault_tolerant
def processInfo(self, field):
if field.name in self.INFO_TO_ATTR:
key = self.INFO_TO_ATTR[field.name]
value = field.value
setattr(self, key, value)
elif field.name == "piece_length":
self.comment = "Piece length: %s" % field.display
class TTF_Metadata(RootMetadata):
NAMEID_TO_ATTR = {
0: "copyright", # Copyright notice
3: "title", # Unique font identifier
5: "version", # Version string
8: "author", # Manufacturer name
11: "url", # URL Vendor
14: "copyright", # License info URL
}
def extract(self, ttf):
if "header" in ttf:
self.extractHeader(ttf["header"])
if "names" in ttf:
self.extractNames(ttf["names"])
@fault_tolerant
def extractHeader(self, header):
self.creation_date = header["created"].value
self.last_modification = header["modified"].value
self.comment = u"Smallest readable size in pixels: %s pixels" % header["lowest"].value
self.comment = u"Font direction: %s" % header["font_dir"].display
@fault_tolerant
def extractNames(self, names):
offset = names["offset"].value
for header in names.array("header"):
key = header["nameID"].value
foffset = offset + header["offset"].value
field = names.getFieldByAddress(foffset*8)
if not field or not isString(field):
continue
value = field.value
if key not in self.NAMEID_TO_ATTR:
continue
key = self.NAMEID_TO_ATTR[key]
if key == "version" and value.startswith(u"Version "):
# "Version 1.2" => "1.2"
value = value[8:]
setattr(self, key, value)
class OLE2_Metadata(RootMetadata):
SUMMARY_ID_TO_ATTR = {
2: "title", # Title
3: "title", # Subject
4: "author",
6: "comment",
8: "author", # Last saved by
12: "creation_date",
13: "last_modification",
14: "nb_page",
18: "producer",
}
IGNORE_SUMMARY = set((
1, # Code page
))
DOC_SUMMARY_ID_TO_ATTR = {
3: "title", # Subject
14: "author", # Manager
}
IGNORE_DOC_SUMMARY = set((
1, # Code page
))
def extract(self, ole2):
self._extract(ole2)
def _extract(self, fieldset, main_document=True):
if main_document:
# _feedAll() is needed to make sure that we get all root[*] fragments
fieldset._feedAll()
if "root[0]" in fieldset:
self.useRoot(fieldset["root[0]"])
doc_summary = self.getField(fieldset, main_document, "doc_summary[0]")
if doc_summary:
self.useSummary(doc_summary, True)
word_doc = self.getField(fieldset, main_document, "word_doc[0]")
if word_doc:
self.useWordDocument(word_doc)
summary = self.getField(fieldset, main_document, "summary[0]")
if summary:
self.useSummary(summary, False)
@fault_tolerant
def useRoot(self, root):
stream = root.getSubIStream()
ministream = guessParser(stream)
if not ministream:
warning("Unable to create the OLE2 mini stream parser!")
return
self._extract(ministream, main_document=False)
def getField(self, fieldset, main_document, name):
if name not in fieldset:
return None
# _feedAll() is needed to make sure that we get all fragments
# eg. summary[0], summary[1], ..., summary[n]
fieldset._feedAll()
field = fieldset[name]
if main_document:
stream = field.getSubIStream()
field = guessParser(stream)
if not field:
warning("Unable to create the OLE2 parser for %s!" % name)
return None
return field
@fault_tolerant
def useSummary(self, summary, is_doc_summary):
if "os" in summary:
self.os = summary["os"].display
if "section[0]" not in summary:
return
summary = summary["section[0]"]
for property in summary.array("property_index"):
self.useProperty(summary, property, is_doc_summary)
@fault_tolerant
def useWordDocument(self, doc):
self.comment = "Encrypted: %s" % doc["fEncrypted"].value
@fault_tolerant
def useProperty(self, summary, property, is_doc_summary):
field = summary.getFieldByAddress(property["offset"].value*8)
if not field \
or "value" not in field:
return
field = field["value"]
if not field.hasValue():
return
# Get value
value = field.value
if isinstance(value, (str, unicode)):
value = normalizeString(value)
if not value:
return
# Get property identifier
prop_id = property["id"].value
if is_doc_summary:
id_to_attr = self.DOC_SUMMARY_ID_TO_ATTR
ignore = self.IGNORE_DOC_SUMMARY
else:
id_to_attr = self.SUMMARY_ID_TO_ATTR
ignore = self.IGNORE_SUMMARY
if prop_id in ignore:
return
# Get Hachoir metadata key
try:
key = id_to_attr[prop_id]
use_prefix = False
except LookupError:
key = "comment"
use_prefix = True
if use_prefix:
prefix = property["id"].display
if (prefix in ("TotalEditingTime", "LastPrinted")) \
and (not field):
# Ignore null time delta
return
value = "%s: %s" % (prefix, value)
else:
if (key == "last_modification") and (not field):
# Ignore null timestamp
return
setattr(self, key, value)
class PcfMetadata(RootMetadata):
PROP_TO_KEY = {
'CHARSET_REGISTRY': 'charset',
'COPYRIGHT': 'copyright',
'WEIGHT_NAME': 'font_weight',
'FOUNDRY': 'author',
'FONT': 'title',
'_XMBDFED_INFO': 'producer',
}
def extract(self, pcf):
if "properties" in pcf:
self.useProperties(pcf["properties"])
def useProperties(self, properties):
last = properties["total_str_length"]
offset0 = last.address + last.size
for index in properties.array("property"):
# Search name and value
value = properties.getFieldByAddress(offset0+index["value_offset"].value*8)
if not value:
continue
value = value.value
if not value:
continue
name = properties.getFieldByAddress(offset0+index["name_offset"].value*8)
if not name:
continue
name = name.value
if name not in self.PROP_TO_KEY:
warning("Skip %s=%r" % (name, value))
continue
key = self.PROP_TO_KEY[name]
setattr(self, key, value)
class SwfMetadata(RootMetadata):
def extract(self, swf):
self.height = swf["rect/ymax"].value # twips
self.width = swf["rect/xmax"].value # twips
self.format_version = "flash version %s" % swf["version"].value
self.frame_rate = swf["frame_rate"].value
self.comment = "Frame count: %s" % swf["frame_count"].value
registerExtractor(TorrentFile, TorrentMetadata)
registerExtractor(TrueTypeFontFile, TTF_Metadata)
registerExtractor(OLE2_File, OLE2_Metadata)
registerExtractor(PcfFile, PcfMetadata)
registerExtractor(SwfFile, SwfMetadata)
libs/hachoir_metadata/program.py
-100
View File
@@ -1,100 +0,0 @@
from hachoir_metadata.metadata import RootMetadata, registerExtractor
from hachoir_parser.program import ExeFile
from hachoir_metadata.safe import fault_tolerant, getValue
class ExeMetadata(RootMetadata):
KEY_TO_ATTR = {
u"ProductName": "title",
u"LegalCopyright": "copyright",
u"LegalTrademarks": "copyright",
u"LegalTrademarks1": "copyright",
u"LegalTrademarks2": "copyright",
u"CompanyName": "author",
u"BuildDate": "creation_date",
u"FileDescription": "title",
u"ProductVersion": "version",
}
SKIP_KEY = set((u"InternalName", u"OriginalFilename", u"FileVersion", u"BuildVersion"))
def extract(self, exe):
if exe.isPE():
self.extractPE(exe)
elif exe.isNE():
self.extractNE(exe)
def extractNE(self, exe):
if "ne_header" in exe:
self.useNE_Header(exe["ne_header"])
if "info" in exe:
self.useNEInfo(exe["info"])
@fault_tolerant
def useNEInfo(self, info):
for node in info.array("node"):
if node["name"].value == "StringFileInfo":
self.readVersionInfo(node["node[0]"])
def extractPE(self, exe):
# Read information from headers
if "pe_header" in exe:
self.usePE_Header(exe["pe_header"])
if "pe_opt_header" in exe:
self.usePE_OptHeader(exe["pe_opt_header"])
# Use PE resource
resource = exe.getResource()
if resource and "version_info/node[0]" in resource:
for node in resource.array("version_info/node[0]/node"):
if getValue(node, "name") == "StringFileInfo" \
and "node[0]" in node:
self.readVersionInfo(node["node[0]"])
@fault_tolerant
def useNE_Header(self, hdr):
if hdr["is_dll"].value:
self.format_version = u"New-style executable: Dynamic-link library (DLL)"
elif hdr["is_win_app"].value:
self.format_version = u"New-style executable: Windows 3.x application"
else:
self.format_version = u"New-style executable for Windows 3.x"
@fault_tolerant
def usePE_Header(self, hdr):
self.creation_date = hdr["creation_date"].value
self.comment = "CPU: %s" % hdr["cpu"].display
if hdr["is_dll"].value:
self.format_version = u"Portable Executable: Dynamic-link library (DLL)"
else:
self.format_version = u"Portable Executable: Windows application"
@fault_tolerant
def usePE_OptHeader(self, hdr):
self.comment = "Subsystem: %s" % hdr["subsystem"].display
def readVersionInfo(self, info):
values = {}
for node in info.array("node"):
if "value" not in node or "name" not in node:
continue
value = node["value"].value.strip(" \0")
if not value:
continue
key = node["name"].value
values[key] = value
if "ProductName" in values and "FileDescription" in values:
# Make sure that FileDescription is set before ProductName
# as title value
self.title = values["FileDescription"]
self.title = values["ProductName"]
del values["FileDescription"]
del values["ProductName"]
for key, value in values.iteritems():
if key in self.KEY_TO_ATTR:
setattr(self, self.KEY_TO_ATTR[key], value)
elif key not in self.SKIP_KEY:
self.comment = "%s=%s" % (key, value)
registerExtractor(ExeFile, ExeMetadata)
libs/hachoir_metadata/qt/__init__.py
View File
libs/hachoir_metadata/qt/dialog.ui
-64
View File
@@ -1,64 +0,0 @@
<ui version="4.0" >
<class>Form</class>
<widget class="QWidget" name="Form" >
<property name="geometry" >
<rect>
<x>0</x>
<y>0</y>
<width>441</width>
<height>412</height>
</rect>
</property>
<property name="windowTitle" >
<string>hachoir-metadata</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout" >
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2" >
<item>
<widget class="QPushButton" name="open_button" >
<property name="text" >
<string>Open</string>
</property>
</widget>
</item>
<item>
<widget class="QComboBox" name="files_combo" >
<property name="sizePolicy" >
<sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QTableWidget" name="metadata_table" >
<property name="alternatingRowColors" >
<bool>true</bool>
</property>
<property name="showGrid" >
<bool>false</bool>
</property>
<property name="rowCount" >
<number>0</number>
</property>
<property name="columnCount" >
<number>0</number>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="quit_button" >
<property name="text" >
<string>Quit</string>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections/>
</ui>
libs/hachoir_metadata/register.py
-112
View File
@@ -1,112 +0,0 @@
from hachoir_core.i18n import _
from hachoir_core.tools import (
humanDuration, humanBitRate,
humanFrequency, humanBitSize, humanFilesize,
humanDatetime)
from hachoir_core.language import Language
from hachoir_metadata.filter import Filter, NumberFilter, DATETIME_FILTER
from datetime import date, datetime, timedelta
from hachoir_metadata.formatter import (
humanAudioChannel, humanFrameRate, humanComprRate, humanAltitude,
humanPixelSize, humanDPI)
from hachoir_metadata.setter import (
setDatetime, setTrackNumber, setTrackTotal, setLanguage)
from hachoir_metadata.metadata_item import Data
MIN_SAMPLE_RATE = 1000 # 1 kHz
MAX_SAMPLE_RATE = 192000 # 192 kHz
MAX_NB_CHANNEL = 8 # 8 channels
MAX_WIDTH = 20000 # 20 000 pixels
MAX_BIT_RATE = 500 * 1024 * 1024 # 500 Mbit/s
MAX_HEIGHT = MAX_WIDTH
MAX_DPI_WIDTH = 10000
MAX_DPI_HEIGHT = MAX_DPI_WIDTH
MAX_NB_COLOR = 2 ** 24 # 16 million of color
MAX_BITS_PER_PIXEL = 256 # 256 bits/pixel
MAX_FRAME_RATE = 150 # 150 frame/sec
MAX_NB_PAGE = 20000
MAX_COMPR_RATE = 1000.0
MIN_COMPR_RATE = 0.001
MAX_TRACK = 999
DURATION_FILTER = Filter(timedelta,
timedelta(milliseconds=1),
timedelta(days=365))
def registerAllItems(meta):
meta.register(Data("title", 100, _("Title"), type=unicode))
meta.register(Data("artist", 101, _("Artist"), type=unicode))
meta.register(Data("author", 102, _("Author"), type=unicode))
meta.register(Data("music_composer", 103, _("Music composer"), type=unicode))
meta.register(Data("album", 200, _("Album"), type=unicode))
meta.register(Data("duration", 201, _("Duration"), # integer in milliseconde
type=timedelta, text_handler=humanDuration, filter=DURATION_FILTER))
meta.register(Data("nb_page", 202, _("Nb page"), filter=NumberFilter(1, MAX_NB_PAGE)))
meta.register(Data("music_genre", 203, _("Music genre"), type=unicode))
meta.register(Data("language", 204, _("Language"), conversion=setLanguage, type=Language))
meta.register(Data("track_number", 205, _("Track number"), conversion=setTrackNumber,
filter=NumberFilter(1, MAX_TRACK), type=(int, long)))
meta.register(Data("track_total", 206, _("Track total"), conversion=setTrackTotal,
filter=NumberFilter(1, MAX_TRACK), type=(int, long)))
meta.register(Data("organization", 210, _("Organization"), type=unicode))
meta.register(Data("version", 220, _("Version")))
meta.register(Data("width", 301, _("Image width"), filter=NumberFilter(1, MAX_WIDTH), type=(int, long), text_handler=humanPixelSize))
meta.register(Data("height", 302, _("Image height"), filter=NumberFilter(1, MAX_HEIGHT), type=(int, long), text_handler=humanPixelSize))
meta.register(Data("nb_channel", 303, _("Channel"), text_handler=humanAudioChannel, filter=NumberFilter(1, MAX_NB_CHANNEL), type=(int, long)))
meta.register(Data("sample_rate", 304, _("Sample rate"), text_handler=humanFrequency, filter=NumberFilter(MIN_SAMPLE_RATE, MAX_SAMPLE_RATE), type=(int, long, float)))
meta.register(Data("bits_per_sample", 305, _("Bits/sample"), text_handler=humanBitSize, filter=NumberFilter(1, 64), type=(int, long)))
meta.register(Data("image_orientation", 306, _("Image orientation")))
meta.register(Data("nb_colors", 307, _("Number of colors"), filter=NumberFilter(1, MAX_NB_COLOR), type=(int, long)))
meta.register(Data("bits_per_pixel", 308, _("Bits/pixel"), filter=NumberFilter(1, MAX_BITS_PER_PIXEL), type=(int, long)))
meta.register(Data("filename", 309, _("File name"), type=unicode))
meta.register(Data("file_size", 310, _("File size"), text_handler=humanFilesize, type=(int, long)))
meta.register(Data("pixel_format", 311, _("Pixel format")))
meta.register(Data("compr_size", 312, _("Compressed file size"), text_handler=humanFilesize, type=(int, long)))
meta.register(Data("compr_rate", 313, _("Compression rate"), text_handler=humanComprRate, filter=NumberFilter(MIN_COMPR_RATE, MAX_COMPR_RATE), type=(int, long, float)))
meta.register(Data("width_dpi", 320, _("Image DPI width"), filter=NumberFilter(1, MAX_DPI_WIDTH), type=(int, long), text_handler=humanDPI))
meta.register(Data("height_dpi", 321, _("Image DPI height"), filter=NumberFilter(1, MAX_DPI_HEIGHT), type=(int, long), text_handler=humanDPI))
meta.register(Data("file_attr", 400, _("File attributes")))
meta.register(Data("file_type", 401, _("File type")))
meta.register(Data("subtitle_author", 402, _("Subtitle author"), type=unicode))
meta.register(Data("creation_date", 500, _("Creation date"), text_handler=humanDatetime,
filter=DATETIME_FILTER, type=(datetime, date), conversion=setDatetime))
meta.register(Data("last_modification", 501, _("Last modification"), text_handler=humanDatetime,
filter=DATETIME_FILTER, type=(datetime, date), conversion=setDatetime))
meta.register(Data("latitude", 510, _("Latitude"), type=float))
meta.register(Data("longitude", 511, _("Longitude"), type=float))
meta.register(Data("altitude", 511, _("Altitude"), type=float, text_handler=humanAltitude))
meta.register(Data("location", 530, _("Location"), type=unicode))
meta.register(Data("city", 531, _("City"), type=unicode))
meta.register(Data("country", 532, _("Country"), type=unicode))
meta.register(Data("charset", 540, _("Charset"), type=unicode))
meta.register(Data("font_weight", 550, _("Font weight")))
meta.register(Data("camera_aperture", 520, _("Camera aperture")))
meta.register(Data("camera_focal", 521, _("Camera focal")))
meta.register(Data("camera_exposure", 522, _("Camera exposure")))
meta.register(Data("camera_brightness", 530, _("Camera brightness")))
meta.register(Data("camera_model", 531, _("Camera model"), type=unicode))
meta.register(Data("camera_manufacturer", 532, _("Camera manufacturer"), type=unicode))
meta.register(Data("compression", 600, _("Compression")))
meta.register(Data("copyright", 601, _("Copyright"), type=unicode))
meta.register(Data("url", 602, _("URL"), type=unicode))
meta.register(Data("frame_rate", 603, _("Frame rate"), text_handler=humanFrameRate,
filter=NumberFilter(1, MAX_FRAME_RATE), type=(int, long, float)))
meta.register(Data("bit_rate", 604, _("Bit rate"), text_handler=humanBitRate,
filter=NumberFilter(1, MAX_BIT_RATE), type=(int, long, float)))
meta.register(Data("aspect_ratio", 604, _("Aspect ratio"), type=(int, long, float)))
meta.register(Data("os", 900, _("OS"), type=unicode))
meta.register(Data("producer", 901, _("Producer"), type=unicode))
meta.register(Data("comment", 902, _("Comment"), type=unicode))
meta.register(Data("format_version", 950, _("Format version"), type=unicode))
meta.register(Data("mime_type", 951, _("MIME type"), type=unicode))
meta.register(Data("endian", 952, _("Endianness"), type=unicode))
libs/hachoir_metadata/riff.py
-190
View File
@@ -1,190 +0,0 @@
"""
Extract metadata from RIFF file format: AVI video and WAV sound.
"""
from hachoir_metadata.metadata import Metadata, MultipleMetadata, registerExtractor
from hachoir_metadata.safe import fault_tolerant, getValue
from hachoir_parser.container.riff import RiffFile
from hachoir_parser.video.fourcc import UNCOMPRESSED_AUDIO
from hachoir_core.tools import humanFilesize, makeUnicode, timedelta2seconds
from hachoir_core.i18n import _
from hachoir_metadata.audio import computeComprRate as computeAudioComprRate
from datetime import timedelta
class RiffMetadata(MultipleMetadata):
TAG_TO_KEY = {
"INAM": "title",
"IART": "artist",
"ICMT": "comment",
"ICOP": "copyright",
"IENG": "author", # (engineer)
"ISFT": "producer",
"ICRD": "creation_date",
"IDIT": "creation_date",
}
def extract(self, riff):
type = riff["type"].value
if type == "WAVE":
self.extractWAVE(riff)
size = getValue(riff, "audio_data/size")
if size:
computeAudioComprRate(self, size*8)
elif type == "AVI ":
if "headers" in riff:
self.extractAVI(riff["headers"])
self.extractInfo(riff["headers"])
elif type == "ACON":
self.extractAnim(riff)
if "info" in riff:
self.extractInfo(riff["info"])
def processChunk(self, chunk):
if "text" not in chunk:
return
value = chunk["text"].value
tag = chunk["tag"].value
if tag not in self.TAG_TO_KEY:
self.warning("Skip RIFF metadata %s: %s" % (tag, value))
return
key = self.TAG_TO_KEY[tag]
setattr(self, key, value)
@fault_tolerant
def extractWAVE(self, wav):
format = wav["format"]
# Number of channel, bits/sample, sample rate
self.nb_channel = format["nb_channel"].value
self.bits_per_sample = format["bit_per_sample"].value
self.sample_rate = format["sample_per_sec"].value
self.compression = format["codec"].display
if "nb_sample/nb_sample" in wav \
and 0 < format["sample_per_sec"].value:
self.duration = timedelta(seconds=float(wav["nb_sample/nb_sample"].value) / format["sample_per_sec"].value)
if format["codec"].value in UNCOMPRESSED_AUDIO:
# Codec with fixed bit rate
self.bit_rate = format["nb_channel"].value * format["bit_per_sample"].value * format["sample_per_sec"].value
if not self.has("duration") \
and "audio_data/size" in wav \
and self.has("bit_rate"):
duration = float(wav["audio_data/size"].value)*8 / self.get('bit_rate')
self.duration = timedelta(seconds=duration)
def extractInfo(self, fieldset):
for field in fieldset:
if not field.is_field_set:
continue
if "tag" in field:
if field["tag"].value == "LIST":
self.extractInfo(field)
else:
self.processChunk(field)
@fault_tolerant
def extractAVIVideo(self, header, meta):
meta.compression = "%s (fourcc:\"%s\")" \
% (header["fourcc"].display, makeUnicode(header["fourcc"].value))
if header["rate"].value and header["scale"].value:
fps = float(header["rate"].value) / header["scale"].value
meta.frame_rate = fps
if 0 < fps:
self.duration = meta.duration = timedelta(seconds=float(header["length"].value) / fps)
if "../stream_fmt/width" in header:
format = header["../stream_fmt"]
meta.width = format["width"].value
meta.height = format["height"].value
meta.bits_per_pixel = format["depth"].value
else:
meta.width = header["right"].value - header["left"].value
meta.height = header["bottom"].value - header["top"].value
@fault_tolerant
def extractAVIAudio(self, format, meta):
meta.nb_channel = format["channel"].value
meta.sample_rate = format["sample_rate"].value
meta.bit_rate = format["bit_rate"].value * 8
if format["bits_per_sample"].value:
meta.bits_per_sample = format["bits_per_sample"].value
if "../stream_hdr" in format:
header = format["../stream_hdr"]
if header["rate"].value and header["scale"].value:
frame_rate = float(header["rate"].value) / header["scale"].value
meta.duration = timedelta(seconds=float(header["length"].value) / frame_rate)
if header["fourcc"].value != "":
meta.compression = "%s (fourcc:\"%s\")" \
% (format["codec"].display, header["fourcc"].value)
if not meta.has("compression"):
meta.compression = format["codec"].display
self.computeAudioComprRate(meta)
@fault_tolerant
def computeAudioComprRate(self, meta):
uncompr = meta.get('bit_rate', 0)
if not uncompr:
return
compr = meta.get('nb_channel') * meta.get('sample_rate') * meta.get('bits_per_sample', default=16)
if not compr:
return
meta.compr_rate = float(compr) / uncompr
@fault_tolerant
def useAviHeader(self, header):
microsec = header["microsec_per_frame"].value
if microsec:
self.frame_rate = 1000000.0 / microsec
total_frame = getValue(header, "total_frame")
if total_frame and not self.has("duration"):
self.duration = timedelta(microseconds=total_frame * microsec)
self.width = header["width"].value
self.height = header["height"].value
def extractAVI(self, headers):
audio_index = 1
for stream in headers.array("stream"):
if "stream_hdr/stream_type" not in stream:
continue
stream_type = stream["stream_hdr/stream_type"].value
if stream_type == "vids":
if "stream_hdr" in stream:
meta = Metadata(self)
self.extractAVIVideo(stream["stream_hdr"], meta)
self.addGroup("video", meta, "Video stream")
elif stream_type == "auds":
if "stream_fmt" in stream:
meta = Metadata(self)
self.extractAVIAudio(stream["stream_fmt"], meta)
self.addGroup("audio[%u]" % audio_index, meta, "Audio stream")
audio_index += 1
if "avi_hdr" in headers:
self.useAviHeader(headers["avi_hdr"])
# Compute global bit rate
if self.has("duration") and "/movie/size" in headers:
self.bit_rate = float(headers["/movie/size"].value) * 8 / timedelta2seconds(self.get('duration'))
# Video has index?
if "/index" in headers:
self.comment = _("Has audio/video index (%s)") \
% humanFilesize(headers["/index"].size/8)
@fault_tolerant
def extractAnim(self, riff):
if "anim_rate/rate[0]" in riff:
count = 0
total = 0
for rate in riff.array("anim_rate/rate"):
count += 1
if 100 < count:
break
total += rate.value / 60.0
if count and total:
self.frame_rate = count / total
if not self.has("frame_rate") and "anim_hdr/jiffie_rate" in riff:
self.frame_rate = 60.0 / riff["anim_hdr/jiffie_rate"].value
registerExtractor(RiffFile, RiffMetadata)
libs/hachoir_metadata/safe.py
-27
View File
@@ -1,27 +0,0 @@
from hachoir_core.error import HACHOIR_ERRORS, warning
def fault_tolerant(func, *args):
def safe_func(*args, **kw):
try:
func(*args, **kw)
except HACHOIR_ERRORS, err:
warning("Error when calling function %s(): %s" % (
func.__name__, err))
return safe_func
def getFieldAttribute(fieldset, key, attrname):
try:
field = fieldset[key]
if field.hasValue():
return getattr(field, attrname)
except HACHOIR_ERRORS, err:
warning("Unable to get %s of field %s/%s: %s" % (
attrname, fieldset.path, key, err))
return None
def getValue(fieldset, key):
return getFieldAttribute(fieldset, key, "value")
def getDisplay(fieldset, key):
return getFieldAttribute(fieldset, key, "display")
libs/hachoir_metadata/setter.py
-171
View File
@@ -1,171 +0,0 @@
from datetime import date, datetime
import re
from hachoir_core.language import Language
from locale import setlocale, LC_ALL
from time import strptime
from hachoir_metadata.timezone import createTimezone
from hachoir_metadata import config
NORMALIZE_REGEX = re.compile("[-/.: ]+")
YEAR_REGEX1 = re.compile("^([0-9]{4})$")
# Date regex: YYYY-MM-DD (US format)
DATE_REGEX1 = re.compile("^([0-9]{4})~([01][0-9])~([0-9]{2})$")
# Date regex: YYYY-MM-DD HH:MM:SS (US format)
DATETIME_REGEX1 = re.compile("^([0-9]{4})~([01][0-9])~([0-9]{2})~([0-9]{1,2})~([0-9]{2})~([0-9]{2})$")
# Datetime regex: "MM-DD-YYYY HH:MM:SS" (FR format)
DATETIME_REGEX2 = re.compile("^([01]?[0-9])~([0-9]{2})~([0-9]{4})~([0-9]{1,2})~([0-9]{2})~([0-9]{2})$")
# Timezone regex: "(...) +0200"
TIMEZONE_REGEX = re.compile("^(.*)~([+-][0-9]{2})00$")
# Timestmap: 'February 2007'
MONTH_YEAR = "%B~%Y"
# Timestmap: 'Sun Feb 24 15:51:09 2008'
RIFF_TIMESTAMP = "%a~%b~%d~%H~%M~%S~%Y"
# Timestmap: 'Thu, 19 Jul 2007 09:03:57'
ISO_TIMESTAMP = "%a,~%d~%b~%Y~%H~%M~%S"
def parseDatetime(value):
"""
Year and date:
>>> parseDatetime("2000")
(datetime.date(2000, 1, 1), u'2000')
>>> parseDatetime("2004-01-02")
datetime.date(2004, 1, 2)
Timestamp:
>>> parseDatetime("2004-01-02 18:10:45")
datetime.datetime(2004, 1, 2, 18, 10, 45)
>>> parseDatetime("2004-01-02 18:10:45")
datetime.datetime(2004, 1, 2, 18, 10, 45)
Timestamp with timezone:
>>> parseDatetime(u'Thu, 19 Jul 2007 09:03:57 +0000')
datetime.datetime(2007, 7, 19, 9, 3, 57, tzinfo=<TimezoneUTC delta=0, name=u'UTC'>)
>>> parseDatetime(u'Thu, 19 Jul 2007 09:03:57 +0200')
datetime.datetime(2007, 7, 19, 9, 3, 57, tzinfo=<Timezone delta=2:00:00, name='+0200'>)
"""
value = NORMALIZE_REGEX.sub("~", value.strip())
regs = YEAR_REGEX1.match(value)
if regs:
try:
year = int(regs.group(1))
return (date(year, 1, 1), unicode(year))
except ValueError:
pass
regs = DATE_REGEX1.match(value)
if regs:
try:
year = int(regs.group(1))
month = int(regs.group(2))
day = int(regs.group(3))
return date(year, month, day)
except ValueError:
pass
regs = DATETIME_REGEX1.match(value)
if regs:
try:
year = int(regs.group(1))
month = int(regs.group(2))
day = int(regs.group(3))
hour = int(regs.group(4))
min = int(regs.group(5))
sec = int(regs.group(6))
return datetime(year, month, day, hour, min, sec)
except ValueError:
pass
regs = DATETIME_REGEX2.match(value)
if regs:
try:
month = int(regs.group(1))
day = int(regs.group(2))
year = int(regs.group(3))
hour = int(regs.group(4))
min = int(regs.group(5))
sec = int(regs.group(6))
return datetime(year, month, day, hour, min, sec)
except ValueError:
pass
current_locale = setlocale(LC_ALL, "C")
try:
match = TIMEZONE_REGEX.match(value)
if match:
without_timezone = match.group(1)
delta = int(match.group(2))
delta = createTimezone(delta)
else:
without_timezone = value
delta = None
try:
timestamp = strptime(without_timezone, ISO_TIMESTAMP)
arguments = list(timestamp[0:6]) + [0, delta]
return datetime(*arguments)
except ValueError:
pass
try:
timestamp = strptime(without_timezone, RIFF_TIMESTAMP)
arguments = list(timestamp[0:6]) + [0, delta]
return datetime(*arguments)
except ValueError:
pass
try:
timestamp = strptime(value, MONTH_YEAR)
arguments = list(timestamp[0:3])
return date(*arguments)
except ValueError:
pass
finally:
setlocale(LC_ALL, current_locale)
return None
def setDatetime(meta, key, value):
if isinstance(value, (str, unicode)):
return parseDatetime(value)
elif isinstance(value, (date, datetime)):
return value
return None
def setLanguage(meta, key, value):
"""
>>> setLanguage(None, None, "fre")
<Language 'French', code='fre'>
>>> setLanguage(None, None, u"ger")
<Language 'German', code='ger'>
"""
return Language(value)
def setTrackTotal(meta, key, total):
"""
>>> setTrackTotal(None, None, "10")
10
"""
try:
return int(total)
except ValueError:
meta.warning("Invalid track total: %r" % total)
return None
def setTrackNumber(meta, key, number):
if isinstance(number, (int, long)):
return number
if "/" in number:
number, total = number.split("/", 1)
meta.track_total = total
try:
return int(number)
except ValueError:
meta.warning("Invalid track number: %r" % number)
return None
def normalizeString(text):
if config.RAW_OUTPUT:
return text
return text.strip(" \t\v\n\r\0")
libs/hachoir_metadata/timezone.py
-42
View File
@@ -1,42 +0,0 @@
from datetime import tzinfo, timedelta
class TimezoneUTC(tzinfo):
"""UTC timezone"""
ZERO = timedelta(0)
def utcoffset(self, dt):
return TimezoneUTC.ZERO
def tzname(self, dt):
return u"UTC"
def dst(self, dt):
return TimezoneUTC.ZERO
def __repr__(self):
return "<TimezoneUTC delta=0, name=u'UTC'>"
class Timezone(TimezoneUTC):
"""Fixed offset in hour from UTC."""
def __init__(self, offset):
self._offset = timedelta(minutes=offset*60)
self._name = u"%+03u00" % offset
def utcoffset(self, dt):
return self._offset
def tzname(self, dt):
return self._name
def __repr__(self):
return "<Timezone delta=%s, name='%s'>" % (
self._offset, self._name)
UTC = TimezoneUTC()
def createTimezone(offset):
if offset:
return Timezone(offset)
else:
return UTC
libs/hachoir_metadata/version.py
-5
View File
@@ -1,5 +0,0 @@
PACKAGE = "hachoir-metadata"
VERSION = "1.3.3"
WEBSITE = "http://bitbucket.org/haypo/hachoir/wiki/hachoir-metadata"
LICENSE = "GNU GPL v2"
libs/hachoir_metadata/video.py
-412
View File
@@ -1,412 +0,0 @@
from hachoir_core.field import MissingField
from hachoir_metadata.metadata import (registerExtractor,
Metadata, RootMetadata, MultipleMetadata)
from hachoir_metadata.metadata_item import QUALITY_GOOD
from hachoir_metadata.safe import fault_tolerant
from hachoir_parser.video import MovFile, AsfFile, FlvFile
from hachoir_parser.video.asf import Descriptor as ASF_Descriptor
from hachoir_parser.container import MkvFile
from hachoir_parser.container.mkv import dateToDatetime
from hachoir_core.i18n import _
from hachoir_core.tools import makeUnicode, makePrintable, timedelta2seconds
from datetime import timedelta
class MkvMetadata(MultipleMetadata):
tag_key = {
"TITLE": "title",
"URL": "url",
"COPYRIGHT": "copyright",
# TODO: use maybe another name?
# Its value may be different than (...)/Info/DateUTC/date
"DATE_RECORDED": "creation_date",
# TODO: Extract subtitle metadata
"SUBTITLE": "subtitle_author",
}
def extract(self, mkv):
for segment in mkv.array("Segment"):
self.processSegment(segment)
def processSegment(self, segment):
for field in segment:
if field.name.startswith("Info["):
self.processInfo(field)
elif field.name.startswith("Tags["):
for tag in field.array("Tag"):
self.processTag(tag)
elif field.name.startswith("Tracks["):
self.processTracks(field)
elif field.name.startswith("Cluster["):
if self.quality < QUALITY_GOOD:
return
def processTracks(self, tracks):
for entry in tracks.array("TrackEntry"):
self.processTrack(entry)
def processTrack(self, track):
if "TrackType/enum" not in track:
return
if track["TrackType/enum"].display == "video":
self.processVideo(track)
elif track["TrackType/enum"].display == "audio":
self.processAudio(track)
elif track["TrackType/enum"].display == "subtitle":
self.processSubtitle(track)
def trackCommon(self, track, meta):
if "Name/unicode" in track:
meta.title = track["Name/unicode"].value
if "Language/string" in track \
and track["Language/string"].value not in ("mis", "und"):
meta.language = track["Language/string"].value
def processVideo(self, track):
video = Metadata(self)
self.trackCommon(track, video)
try:
video.compression = track["CodecID/string"].value
if "Video" in track:
video.width = track["Video/PixelWidth/unsigned"].value
video.height = track["Video/PixelHeight/unsigned"].value
except MissingField:
pass
self.addGroup("video[]", video, "Video stream")
def getDouble(self, field, parent):
float_key = '%s/float' % parent
if float_key in field:
return field[float_key].value
double_key = '%s/double' % parent
if double_key in field:
return field[double_key].value
return None
def processAudio(self, track):
audio = Metadata(self)
self.trackCommon(track, audio)
if "Audio" in track:
frequency = self.getDouble(track, "Audio/SamplingFrequency")
if frequency is not None:
audio.sample_rate = frequency
if "Audio/Channels/unsigned" in track:
audio.nb_channel = track["Audio/Channels/unsigned"].value
if "Audio/BitDepth/unsigned" in track:
audio.bits_per_sample = track["Audio/BitDepth/unsigned"].value
if "CodecID/string" in track:
audio.compression = track["CodecID/string"].value
self.addGroup("audio[]", audio, "Audio stream")
def processSubtitle(self, track):
sub = Metadata(self)
self.trackCommon(track, sub)
try:
sub.compression = track["CodecID/string"].value
except MissingField:
pass
self.addGroup("subtitle[]", sub, "Subtitle")
def processTag(self, tag):
for field in tag.array("SimpleTag"):
self.processSimpleTag(field)
def processSimpleTag(self, tag):
if "TagName/unicode" not in tag \
or "TagString/unicode" not in tag:
return
name = tag["TagName/unicode"].value
if name not in self.tag_key:
return
key = self.tag_key[name]
value = tag["TagString/unicode"].value
setattr(self, key, value)
def processInfo(self, info):
if "TimecodeScale/unsigned" in info:
duration = self.getDouble(info, "Duration")
if duration is not None:
try:
seconds = duration * info["TimecodeScale/unsigned"].value * 1e-9
self.duration = timedelta(seconds=seconds)
except OverflowError:
# Catch OverflowError for timedelta (long int too large
# to be converted to an int)
pass
if "DateUTC/date" in info:
try:
self.creation_date = dateToDatetime(info["DateUTC/date"].value)
except OverflowError:
pass
if "WritingApp/unicode" in info:
self.producer = info["WritingApp/unicode"].value
if "MuxingApp/unicode" in info:
self.producer = info["MuxingApp/unicode"].value
if "Title/unicode" in info:
self.title = info["Title/unicode"].value
class FlvMetadata(MultipleMetadata):
def extract(self, flv):
if "video[0]" in flv:
meta = Metadata(self)
self.extractVideo(flv["video[0]"], meta)
self.addGroup("video", meta, "Video stream")
if "audio[0]" in flv:
meta = Metadata(self)
self.extractAudio(flv["audio[0]"], meta)
self.addGroup("audio", meta, "Audio stream")
# TODO: Computer duration
# One technic: use last video/audio chunk and use timestamp
# But this is very slow
self.format_version = flv.description
if "metadata/entry[1]" in flv:
self.extractAMF(flv["metadata/entry[1]"])
if self.has('duration'):
self.bit_rate = flv.size / timedelta2seconds(self.get('duration'))
@fault_tolerant
def extractAudio(self, audio, meta):
if audio["codec"].display == "MP3" and "music_data" in audio:
meta.compression = audio["music_data"].description
else:
meta.compression = audio["codec"].display
meta.sample_rate = audio.getSampleRate()
if audio["is_16bit"].value:
meta.bits_per_sample = 16
else:
meta.bits_per_sample = 8
if audio["is_stereo"].value:
meta.nb_channel = 2
else:
meta.nb_channel = 1
@fault_tolerant
def extractVideo(self, video, meta):
meta.compression = video["codec"].display
def extractAMF(self, amf):
for entry in amf.array("item"):
self.useAmfEntry(entry)
@fault_tolerant
def useAmfEntry(self, entry):
key = entry["key"].value
if key == "duration":
self.duration = timedelta(seconds=entry["value"].value)
elif key == "creator":
self.producer = entry["value"].value
elif key == "audiosamplerate":
self.sample_rate = entry["value"].value
elif key == "framerate":
self.frame_rate = entry["value"].value
elif key == "metadatacreator":
self.producer = entry["value"].value
elif key == "metadatadate":
self.creation_date = entry.value
elif key == "width":
self.width = int(entry["value"].value)
elif key == "height":
self.height = int(entry["value"].value)
class MovMetadata(RootMetadata):
def extract(self, mov):
for atom in mov:
if "movie" in atom:
self.processMovie(atom["movie"])
@fault_tolerant
def processMovieHeader(self, hdr):
self.creation_date = hdr["creation_date"].value
self.last_modification = hdr["lastmod_date"].value
self.duration = timedelta(seconds=float(hdr["duration"].value) / hdr["time_scale"].value)
self.comment = _("Play speed: %.1f%%") % (hdr["play_speed"].value*100)
self.comment = _("User volume: %.1f%%") % (float(hdr["volume"].value)*100//255)
@fault_tolerant
def processTrackHeader(self, hdr):
width = int(hdr["frame_size_width"].value)
height = int(hdr["frame_size_height"].value)
if width and height:
self.width = width
self.height = height
def processTrack(self, atom):
for field in atom:
if "track_hdr" in field:
self.processTrackHeader(field["track_hdr"])
def processMovie(self, atom):
for field in atom:
if "track" in field:
self.processTrack(field["track"])
if "movie_hdr" in field:
self.processMovieHeader(field["movie_hdr"])
class AsfMetadata(MultipleMetadata):
EXT_DESC_TO_ATTR = {
"Encoder": "producer",
"ToolName": "producer",
"AlbumTitle": "album",
"Track": "track_number",
"TrackNumber": "track_total",
"Year": "creation_date",
"AlbumArtist": "author",
}
SKIP_EXT_DESC = set((
# Useless informations
"WMFSDKNeeded", "WMFSDKVersion",
"Buffer Average", "VBR Peak", "EncodingTime",
"MediaPrimaryClassID", "UniqueFileIdentifier",
))
def extract(self, asf):
if "header/content" in asf:
self.processHeader(asf["header/content"])
def processHeader(self, header):
compression = []
is_vbr = None
if "ext_desc/content" in header:
# Extract all data from ext_desc
data = {}
for desc in header.array("ext_desc/content/descriptor"):
self.useExtDescItem(desc, data)
# Have ToolName and ToolVersion? If yes, group them to producer key
if "ToolName" in data and "ToolVersion" in data:
self.producer = "%s (version %s)" % (data["ToolName"], data["ToolVersion"])
del data["ToolName"]
del data["ToolVersion"]
# "IsVBR" key
if "IsVBR" in data:
is_vbr = (data["IsVBR"] == 1)
del data["IsVBR"]
# Store data
for key, value in data.iteritems():
if key in self.EXT_DESC_TO_ATTR:
key = self.EXT_DESC_TO_ATTR[key]
else:
if isinstance(key, str):
key = makePrintable(key, "ISO-8859-1", to_unicode=True)
value = "%s=%s" % (key, value)
key = "comment"
setattr(self, key, value)
if "file_prop/content" in header:
self.useFileProp(header["file_prop/content"], is_vbr)
if "codec_list/content" in header:
for codec in header.array("codec_list/content/codec"):
if "name" in codec:
text = codec["name"].value
if "desc" in codec and codec["desc"].value:
text = "%s (%s)" % (text, codec["desc"].value)
compression.append(text)
audio_index = 1
video_index = 1
for index, stream_prop in enumerate(header.array("stream_prop")):
if "content/audio_header" in stream_prop:
meta = Metadata(self)
self.streamProperty(header, index, meta)
self.streamAudioHeader(stream_prop["content/audio_header"], meta)
if self.addGroup("audio[%u]" % audio_index, meta, "Audio stream #%u" % audio_index):
audio_index += 1
elif "content/video_header" in stream_prop:
meta = Metadata(self)
self.streamProperty(header, index, meta)
self.streamVideoHeader(stream_prop["content/video_header"], meta)
if self.addGroup("video[%u]" % video_index, meta, "Video stream #%u" % video_index):
video_index += 1
if "metadata/content" in header:
info = header["metadata/content"]
try:
self.title = info["title"].value
self.author = info["author"].value
self.copyright = info["copyright"].value
except MissingField:
pass
@fault_tolerant
def streamAudioHeader(self, audio, meta):
if not meta.has("compression"):
meta.compression = audio["twocc"].display
meta.nb_channel = audio["channels"].value
meta.sample_rate = audio["sample_rate"].value
meta.bits_per_sample = audio["bits_per_sample"].value
@fault_tolerant
def streamVideoHeader(self, video, meta):
meta.width = video["width"].value
meta.height = video["height"].value
if "bmp_info" in video:
bmp_info = video["bmp_info"]
if not meta.has("compression"):
meta.compression = bmp_info["codec"].display
meta.bits_per_pixel = bmp_info["bpp"].value
@fault_tolerant
def useExtDescItem(self, desc, data):
if desc["type"].value == ASF_Descriptor.TYPE_BYTE_ARRAY:
# Skip binary data
return
key = desc["name"].value
if "/" in key:
# Replace "WM/ToolName" with "ToolName"
key = key.split("/", 1)[1]
if key in self.SKIP_EXT_DESC:
# Skip some keys
return
value = desc["value"].value
if not value:
return
value = makeUnicode(value)
data[key] = value
@fault_tolerant
def useFileProp(self, prop, is_vbr):
self.creation_date = prop["creation_date"].value
self.duration = prop["play_duration"].value
if prop["seekable"].value:
self.comment = u"Is seekable"
value = prop["max_bitrate"].value
text = prop["max_bitrate"].display
if is_vbr is True:
text = "VBR (%s max)" % text
elif is_vbr is False:
text = "%s (CBR)" % text
else:
text = "%s (max)" % text
self.bit_rate = (value, text)
def streamProperty(self, header, index, meta):
key = "bit_rates/content/bit_rate[%u]/avg_bitrate" % index
if key in header:
meta.bit_rate = header[key].value
# TODO: Use codec list
# It doesn't work when the video uses /header/content/bitrate_mutex
# since the codec list are shared between streams but... how is it
# shared?
# key = "codec_list/content/codec[%u]" % index
# if key in header:
# codec = header[key]
# if "name" in codec:
# text = codec["name"].value
# if "desc" in codec and codec["desc"].value:
# meta.compression = "%s (%s)" % (text, codec["desc"].value)
# else:
# meta.compression = text
registerExtractor(MovFile, MovMetadata)
registerExtractor(AsfFile, AsfMetadata)
registerExtractor(FlvFile, FlvMetadata)
registerExtractor(MkvFile, MkvMetadata)
libs/hachoir_parser/__init__.py
-7
View File
@@ -1,7 +0,0 @@
from hachoir_parser.version import __version__
from hachoir_parser.parser import ValidateError, HachoirParser, Parser
from hachoir_parser.parser_list import ParserList, HachoirParserList
from hachoir_parser.guess import (QueryParser, guessParser, createParser)
from hachoir_parser import (archive, audio, container,
file_system, image, game, misc, network, program, video)
libs/hachoir_parser/archive/__init__.py
-12
View File
@@ -1,12 +0,0 @@
from hachoir_parser.archive.ace import AceFile
from hachoir_parser.archive.ar import ArchiveFile
from hachoir_parser.archive.bzip2_parser import Bzip2Parser
from hachoir_parser.archive.cab import CabFile
from hachoir_parser.archive.gzip_parser import GzipParser
from hachoir_parser.archive.tar import TarFile
from hachoir_parser.archive.zip import ZipFile
from hachoir_parser.archive.rar import RarFile
from hachoir_parser.archive.rpm import RpmFile
from hachoir_parser.archive.sevenzip import SevenZipParser
from hachoir_parser.archive.mar import MarFile
libs/hachoir_parser/archive/ace.py
-267
View File
@@ -1,267 +0,0 @@
"""
ACE parser
From wotsit.org and the SDK header (bitflags)
Partial study of a new block type (5) I've called "new_recovery", as its
syntax is very close to the former one (of type 2).
Status: can only read totally file and header blocks.
Author: Christophe Gisquet <christophe.gisquet@free.fr>
Creation date: 19 january 2006
"""
from hachoir_parser import Parser
from hachoir_core.field import (StaticFieldSet, FieldSet,
Bit, Bits, NullBits, RawBytes, Enum,
UInt8, UInt16, UInt32,
PascalString8, PascalString16, String,
TimeDateMSDOS32)
from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_parser.common.msdos import MSDOSFileAttr32
MAGIC = "**ACE**"
OS_MSDOS = 0
OS_WIN32 = 2
HOST_OS = {
0: "MS-DOS",
1: "OS/2",
2: "Win32",
3: "Unix",
4: "MAC-OS",
5: "Win NT",
6: "Primos",
7: "APPLE GS",
8: "ATARI",
9: "VAX VMS",
10: "AMIGA",
11: "NEXT",
}
COMPRESSION_TYPE = {
0: "Store",
1: "Lempel-Ziv 77",
2: "ACE v2.0",
}
COMPRESSION_MODE = {
0: "fastest",
1: "fast",
2: "normal",
3: "good",
4: "best",
}
# TODO: Computing the CRC16 would also prove useful
#def markerValidate(self):
# return not self["extend"].value and self["signature"].value == MAGIC and \
# self["host_os"].value<12
class MarkerFlags(StaticFieldSet):
format = (
(Bit, "extend", "Whether the header is extended"),
(Bit, "has_comment", "Whether the archive has a comment"),
(NullBits, "unused", 7, "Reserved bits"),
(Bit, "sfx", "SFX"),
(Bit, "limited_dict", "Junior SFX with 256K dictionary"),
(Bit, "multi_volume", "Part of a set of ACE archives"),
(Bit, "has_av_string", "This header holds an AV-string"),
(Bit, "recovery_record", "Recovery record preset"),
(Bit, "locked", "Archive is locked"),
(Bit, "solid", "Archive uses solid compression")
)
def markerFlags(self):
yield MarkerFlags(self, "flags", "Marker flags")
def markerHeader(self):
yield String(self, "signature", 7, "Signature")
yield UInt8(self, "ver_extract", "Version needed to extract archive")
yield UInt8(self, "ver_created", "Version used to create archive")
yield Enum(UInt8(self, "host_os", "OS where the files were compressed"), HOST_OS)
yield UInt8(self, "vol_num", "Volume number")
yield TimeDateMSDOS32(self, "time", "Date and time (MS DOS format)")
yield Bits(self, "reserved", 64, "Reserved size for future extensions")
flags = self["flags"]
if flags["has_av_string"].value:
yield PascalString8(self, "av_string", "AV String")
if flags["has_comment"].value:
size = filesizeHandler(UInt16(self, "comment_size", "Comment size"))
yield size
if size.value > 0:
yield RawBytes(self, "compressed_comment", size.value, \
"Compressed comment")
class FileFlags(StaticFieldSet):
format = (
(Bit, "extend", "Whether the header is extended"),
(Bit, "has_comment", "Presence of file comment"),
(Bits, "unused", 10, "Unused bit flags"),
(Bit, "encrypted", "File encrypted with password"),
(Bit, "previous", "File continued from previous volume"),
(Bit, "next", "File continues on the next volume"),
(Bit, "solid", "File compressed using previously archived files")
)
def fileFlags(self):
yield FileFlags(self, "flags", "File flags")
def fileHeader(self):
yield filesizeHandler(UInt32(self, "compressed_size", "Size of the compressed file"))
yield filesizeHandler(UInt32(self, "uncompressed_size", "Uncompressed file size"))
yield TimeDateMSDOS32(self, "ftime", "Date and time (MS DOS format)")
if self["/header/host_os"].value in (OS_MSDOS, OS_WIN32):
yield MSDOSFileAttr32(self, "file_attr", "File attributes")
else:
yield textHandler(UInt32(self, "file_attr", "File attributes"), hexadecimal)
yield textHandler(UInt32(self, "file_crc32", "CRC32 checksum over the compressed file)"), hexadecimal)
yield Enum(UInt8(self, "compression_type", "Type of compression"), COMPRESSION_TYPE)
yield Enum(UInt8(self, "compression_mode", "Quality of compression"), COMPRESSION_MODE)
yield textHandler(UInt16(self, "parameters", "Compression parameters"), hexadecimal)
yield textHandler(UInt16(self, "reserved", "Reserved data"), hexadecimal)
# Filename
yield PascalString16(self, "filename", "Filename")
# Comment
if self["flags/has_comment"].value:
yield filesizeHandler(UInt16(self, "comment_size", "Size of the compressed comment"))
if self["comment_size"].value > 0:
yield RawBytes(self, "comment_data", self["comment_size"].value, "Comment data")
def fileBody(self):
size = self["compressed_size"].value
if size > 0:
yield RawBytes(self, "compressed_data", size, "Compressed data")
def fileDesc(self):
return "File entry: %s (%s)" % (self["filename"].value, self["compressed_size"].display)
def recoveryHeader(self):
yield filesizeHandler(UInt32(self, "rec_blk_size", "Size of recovery data"))
self.body_size = self["rec_blk_size"].size
yield String(self, "signature", 7, "Signature, normally '**ACE**'")
yield textHandler(UInt32(self, "relative_start",
"Relative start (to this block) of the data this block is mode of"),
hexadecimal)
yield UInt32(self, "num_blocks", "Number of blocks the data is split into")
yield UInt32(self, "size_blocks", "Size of these blocks")
yield UInt16(self, "crc16_blocks", "CRC16 over recovery data")
# size_blocks blocks of size size_blocks follow
# The ultimate data is the xor data of all those blocks
size = self["size_blocks"].value
for index in xrange(self["num_blocks"].value):
yield RawBytes(self, "data[]", size, "Recovery block %i" % index)
yield RawBytes(self, "xor_data", size, "The XOR value of the above data blocks")
def recoveryDesc(self):
return "Recovery block, size=%u" % self["body_size"].display
def newRecoveryHeader(self):
"""
This header is described nowhere
"""
if self["flags/extend"].value:
yield filesizeHandler(UInt32(self, "body_size", "Size of the unknown body following"))
self.body_size = self["body_size"].value
yield textHandler(UInt32(self, "unknown[]", "Unknown field, probably 0"),
hexadecimal)
yield String(self, "signature", 7, "Signature, normally '**ACE**'")
yield textHandler(UInt32(self, "relative_start",
"Offset (=crc16's) of this block in the file"), hexadecimal)
yield textHandler(UInt32(self, "unknown[]",
"Unknown field, probably 0"), hexadecimal)
class BaseFlags(StaticFieldSet):
format = (
(Bit, "extend", "Whether the header is extended"),
(NullBits, "unused", 15, "Unused bit flags")
)
def parseFlags(self):
yield BaseFlags(self, "flags", "Unknown flags")
def parseHeader(self):
if self["flags/extend"].value:
yield filesizeHandler(UInt32(self, "body_size", "Size of the unknown body following"))
self.body_size = self["body_size"].value
def parseBody(self):
if self.body_size > 0:
yield RawBytes(self, "body_data", self.body_size, "Body data, unhandled")
class Block(FieldSet):
TAG_INFO = {
0: ("header", "Archiver header", markerFlags, markerHeader, None),
1: ("file[]", fileDesc, fileFlags, fileHeader, fileBody),
2: ("recovery[]", recoveryDesc, recoveryHeader, None, None),
5: ("new_recovery[]", None, None, newRecoveryHeader, None)
}
def __init__(self, parent, name, description=None):
FieldSet.__init__(self, parent, name, description)
self.body_size = 0
self.desc_func = None
type = self["block_type"].value
if type in self.TAG_INFO:
self._name, desc, self.parseFlags, self.parseHeader, self.parseBody = self.TAG_INFO[type]
if desc:
if isinstance(desc, str):
self._description = desc
else:
self.desc_func = desc
else:
self.warning("Processing as unknown block block of type %u" % type)
if not self.parseFlags:
self.parseFlags = parseFlags
if not self.parseHeader:
self.parseHeader = parseHeader
if not self.parseBody:
self.parseBody = parseBody
def createFields(self):
yield textHandler(UInt16(self, "crc16", "Archive CRC16 (from byte 4 on)"), hexadecimal)
yield filesizeHandler(UInt16(self, "head_size", "Block size (from byte 4 on)"))
yield UInt8(self, "block_type", "Block type")
# Flags
for flag in self.parseFlags(self):
yield flag
# Rest of the header
for field in self.parseHeader(self):
yield field
size = self["head_size"].value - (self.current_size//8) + (2+2)
if size > 0:
yield RawBytes(self, "extra_data", size, "Extra header data, unhandled")
# Body in itself
for field in self.parseBody(self):
yield field
def createDescription(self):
if self.desc_func:
return self.desc_func(self)
else:
return "Block: %s" % self["type"].display
class AceFile(Parser):
endian = LITTLE_ENDIAN
PARSER_TAGS = {
"id": "ace",
"category": "archive",
"file_ext": ("ace",),
"mime": (u"application/x-ace-compressed",),
"min_size": 50*8,
"description": "ACE archive"
}
def validate(self):
if self.stream.readBytes(7*8, len(MAGIC)) != MAGIC:
return "Invalid magic"
return True
def createFields(self):
while not self.eof:
yield Block(self, "block[]")
libs/hachoir_parser/archive/ar.py
-52
View File
@@ -1,52 +0,0 @@
"""
GNU ar archive : archive file (.a) and Debian (.deb) archive.
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet, ParserError,
String, RawBytes, UnixLine)
from hachoir_core.endian import BIG_ENDIAN
class ArchiveFileEntry(FieldSet):
def createFields(self):
yield UnixLine(self, "header", "Header")
info = self["header"].value.split()
if len(info) != 7:
raise ParserError("Invalid file entry header")
size = int(info[5])
if 0 < size:
yield RawBytes(self, "content", size, "File data")
def createDescription(self):
return "File entry (%s)" % self["header"].value.split()[0]
class ArchiveFile(Parser):
endian = BIG_ENDIAN
MAGIC = '!<arch>\n'
PARSER_TAGS = {
"id": "unix_archive",
"category": "archive",
"file_ext": ("a", "deb"),
"mime":
(u"application/x-debian-package",
u"application/x-archive",
u"application/x-dpkg"),
"min_size": (8 + 13)*8, # file signature + smallest file as possible
"magic": ((MAGIC, 0),),
"description": "Unix archive"
}
def validate(self):
if self.stream.readBytes(0, len(self.MAGIC)) != self.MAGIC:
return "Invalid magic string"
return True
def createFields(self):
yield String(self, "id", 8, "Unix archive identifier (\"<!arch>\")", charset="ASCII")
while not self.eof:
data = self.stream.readBytes(self.current_size, 1)
if data == "\n":
yield RawBytes(self, "empty_line[]", 1, "Empty line")
else:
yield ArchiveFileEntry(self, "file[]", "File")
libs/hachoir_parser/archive/bzip2_parser.py
-85
View File
@@ -1,85 +0,0 @@
"""
BZIP2 archive file
Author: Victor Stinner
"""
from hachoir_parser import Parser
from hachoir_core.field import (ParserError, String,
Bytes, Character, UInt8, UInt32, CompressedField)
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_core.text_handler import textHandler, hexadecimal
try:
from bz2 import BZ2Decompressor
class Bunzip2:
def __init__(self, stream):
self.bzip2 = BZ2Decompressor()
def __call__(self, size, data=''):
try:
return self.bzip2.decompress(data)
except EOFError:
return ''
has_deflate = True
except ImportError:
has_deflate = False
class Bzip2Parser(Parser):
PARSER_TAGS = {
"id": "bzip2",
"category": "archive",
"file_ext": ("bz2",),
"mime": (u"application/x-bzip2",),
"min_size": 10*8,
"magic": (('BZh', 0),),
"description": "bzip2 archive"
}
endian = LITTLE_ENDIAN
def validate(self):
if self.stream.readBytes(0, 3) != 'BZh':
return "Wrong file signature"
if not("1" <= self["blocksize"].value <= "9"):
return "Wrong blocksize"
return True
def createFields(self):
yield String(self, "id", 3, "Identifier (BZh)", charset="ASCII")
yield Character(self, "blocksize", "Block size (KB of memory needed to uncompress)")
yield UInt8(self, "blockheader", "Block header")
if self["blockheader"].value == 0x17:
yield String(self, "id2", 4, "Identifier2 (re8P)", charset="ASCII")
yield UInt8(self, "id3", "Identifier3 (0x90)")
elif self["blockheader"].value == 0x31:
yield String(self, "id2", 5, "Identifier 2 (AY&SY)", charset="ASCII")
if self["id2"].value != "AY&SY":
raise ParserError("Invalid identifier 2 (AY&SY)!")
else:
raise ParserError("Invalid block header!")
yield textHandler(UInt32(self, "crc32", "CRC32"), hexadecimal)
if self._size is None: # TODO: is it possible to handle piped input?
raise NotImplementedError
size = (self._size - self.current_size)/8
if size:
for tag, filename in self.stream.tags:
if tag == "filename" and filename.endswith(".bz2"):
filename = filename[:-4]
break
else:
filename = None
data = Bytes(self, "file", size)
if has_deflate:
CompressedField(self, Bunzip2)
def createInputStream(**args):
if filename:
args.setdefault("tags",[]).append(("filename", filename))
return self._createInputStream(**args)
data._createInputStream = createInputStream
yield data
libs/hachoir_parser/archive/cab.py
-125
View File
@@ -1,125 +0,0 @@
"""
Microsoft Cabinet (CAB) archive.
Author: Victor Stinner
Creation date: 31 january 2007
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet, Enum,
CString, String,
UInt16, UInt32, Bit, Bits, PaddingBits, NullBits,
DateTimeMSDOS32, RawBytes)
from hachoir_parser.common.msdos import MSDOSFileAttr16
from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
from hachoir_core.endian import LITTLE_ENDIAN
MAX_NB_FOLDER = 30
COMPRESSION_NONE = 0
COMPRESSION_NAME = {
0: "Uncompressed",
1: "Deflate",
2: "Quantum",
3: "LZX",
}
class Folder(FieldSet):
def createFields(self):
yield UInt32(self, "off_data", "Offset of data")
yield UInt16(self, "cf_data")
yield Enum(Bits(self, "compr_method", 4, "Compression method"), COMPRESSION_NAME)
yield Bits(self, "compr_level", 5, "Compression level")
yield PaddingBits(self, "padding", 7)
def createDescription(self):
text= "Folder: compression %s" % self["compr_method"].display
if self["compr_method"].value != COMPRESSION_NONE:
text += " (level %u)" % self["compr_level"].value
return text
class File(FieldSet):
def createFields(self):
yield filesizeHandler(UInt32(self, "filesize", "Uncompressed file size"))
yield UInt32(self, "offset", "File offset after decompression")
yield UInt16(self, "iFolder", "file control id")
yield DateTimeMSDOS32(self, "timestamp")
yield MSDOSFileAttr16(self, "attributes")
yield CString(self, "filename", charset="ASCII")
def createDescription(self):
return "File %s (%s)" % (
self["filename"].display, self["filesize"].display)
class Reserved(FieldSet):
def createFields(self):
yield UInt32(self, "size")
size = self["size"].value
if size:
yield RawBytes(self, "data", size)
class Flags(FieldSet):
static_size = 16
def createFields(self):
yield Bit(self, "has_previous")
yield Bit(self, "has_next")
yield Bit(self, "has_reserved")
yield NullBits(self, "padding", 13)
class CabFile(Parser):
endian = LITTLE_ENDIAN
MAGIC = "MSCF"
PARSER_TAGS = {
"id": "cab",
"category": "archive",
"file_ext": ("cab",),
"mime": (u"application/vnd.ms-cab-compressed",),
"magic": ((MAGIC, 0),),
"min_size": 1*8, # header + file entry
"description": "Microsoft Cabinet archive"
}
def validate(self):
if self.stream.readBytes(0, 4) != self.MAGIC:
return "Invalid magic"
if self["cab_version"].value != 0x0103:
return "Unknown version (%s)" % self["cab_version"].display
if not (1 <= self["nb_folder"].value <= MAX_NB_FOLDER):
return "Invalid number of folder (%s)" % self["nb_folder"].value
return True
def createFields(self):
yield String(self, "magic", 4, "Magic (MSCF)", charset="ASCII")
yield textHandler(UInt32(self, "hdr_checksum", "Header checksum (0 if not used)"), hexadecimal)
yield filesizeHandler(UInt32(self, "filesize", "Cabinet file size"))
yield textHandler(UInt32(self, "fld_checksum", "Folders checksum (0 if not used)"), hexadecimal)
yield UInt32(self, "off_file", "Offset of first file")
yield textHandler(UInt32(self, "files_checksum", "Files checksum (0 if not used)"), hexadecimal)
yield textHandler(UInt16(self, "cab_version", "Cabinet version"), hexadecimal)
yield UInt16(self, "nb_folder", "Number of folders")
yield UInt16(self, "nb_files", "Number of files")
yield Flags(self, "flags")
yield UInt16(self, "setid")
yield UInt16(self, "number", "Zero-based cabinet number")
# --- TODO: Support flags
if self["flags/has_reserved"].value:
yield Reserved(self, "reserved")
#(3) Previous cabinet name, if CAB_HEADER.flags & CAB_FLAG_HASPREV
#(4) Previous disk name, if CAB_HEADER.flags & CAB_FLAG_HASPREV
#(5) Next cabinet name, if CAB_HEADER.flags & CAB_FLAG_HASNEXT
#(6) Next disk name, if CAB_HEADER.flags & CAB_FLAG_HASNEXT
# ----
for index in xrange(self["nb_folder"].value):
yield Folder(self, "folder[]")
for index in xrange(self["nb_files"].value):
yield File(self, "file[]")
end = self.seekBit(self.size, "endraw")
if end:
yield end
def createContentSize(self):
return self["filesize"].value * 8
libs/hachoir_parser/archive/gzip_parser.py
-129
View File
@@ -1,129 +0,0 @@
"""
GZIP archive parser.
Author: Victor Stinner
"""
from hachoir_parser import Parser
from hachoir_core.field import (
UInt8, UInt16, UInt32, Enum, TimestampUnix32,
Bit, CString, SubFile,
NullBits, Bytes, RawBytes)
from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_parser.common.deflate import Deflate
class GzipParser(Parser):
endian = LITTLE_ENDIAN
PARSER_TAGS = {
"id": "gzip",
"category": "archive",
"file_ext": ("gz",),
"mime": (u"application/x-gzip",),
"min_size": 18*8,
#"magic": (('\x1F\x8B\x08', 0),),
"magic_regex": (
# (magic, compression=deflate, <flags>, <mtime>, )
('\x1F\x8B\x08.{5}[\0\2\4\6][\x00-\x0D]', 0),
),
"description": u"gzip archive",
}
os_name = {
0: u"FAT filesystem",
1: u"Amiga",
2: u"VMS (or OpenVMS)",
3: u"Unix",
4: u"VM/CMS",
5: u"Atari TOS",
6: u"HPFS filesystem (OS/2, NT)",
7: u"Macintosh",
8: u"Z-System",
9: u"CP/M",
10: u"TOPS-20",
11: u"NTFS filesystem (NT)",
12: u"QDOS",
13: u"Acorn RISCOS",
}
COMPRESSION_NAME = {
8: u"deflate",
}
def validate(self):
if self["signature"].value != '\x1F\x8B':
return "Invalid signature"
if self["compression"].value not in self.COMPRESSION_NAME:
return "Unknown compression method (%u)" % self["compression"].value
if self["reserved[0]"].value != 0:
return "Invalid reserved[0] value"
if self["reserved[1]"].value != 0:
return "Invalid reserved[1] value"
if self["reserved[2]"].value != 0:
return "Invalid reserved[2] value"
return True
def createFields(self):
# Gzip header
yield Bytes(self, "signature", 2, r"GZip file signature (\x1F\x8B)")
yield Enum(UInt8(self, "compression", "Compression method"), self.COMPRESSION_NAME)
# Flags
yield Bit(self, "is_text", "File content is probably ASCII text")
yield Bit(self, "has_crc16", "Header CRC16")
yield Bit(self, "has_extra", "Extra informations (variable size)")
yield Bit(self, "has_filename", "Contains filename?")
yield Bit(self, "has_comment", "Contains comment?")
yield NullBits(self, "reserved[]", 3)
yield TimestampUnix32(self, "mtime", "Modification time")
# Extra flags
yield NullBits(self, "reserved[]", 1)
yield Bit(self, "slowest", "Compressor used maximum compression (slowest)")
yield Bit(self, "fastest", "Compressor used the fastest compression")
yield NullBits(self, "reserved[]", 5)
yield Enum(UInt8(self, "os", "Operating system"), self.os_name)
# Optional fields
if self["has_extra"].value:
yield UInt16(self, "extra_length", "Extra length")
yield RawBytes(self, "extra", self["extra_length"].value, "Extra")
if self["has_filename"].value:
yield CString(self, "filename", "Filename", charset="ISO-8859-1")
if self["has_comment"].value:
yield CString(self, "comment", "Comment")
if self["has_crc16"].value:
yield textHandler(UInt16(self, "hdr_crc16", "CRC16 of the header"),
hexadecimal)
if self._size is None: # TODO: is it possible to handle piped input?
raise NotImplementedError()
# Read file
size = (self._size - self.current_size) // 8 - 8 # -8: crc32+size
if 0 < size:
if self["has_filename"].value:
filename = self["filename"].value
else:
for tag, filename in self.stream.tags:
if tag == "filename" and filename.endswith(".gz"):
filename = filename[:-3]
break
else:
filename = None
yield Deflate(SubFile(self, "file", size, filename=filename))
# Footer
yield textHandler(UInt32(self, "crc32",
"Uncompressed data content CRC32"), hexadecimal)
yield filesizeHandler(UInt32(self, "size", "Uncompressed size"))
def createDescription(self):
desc = u"gzip archive"
info = []
if "filename" in self:
info.append('filename "%s"' % self["filename"].value)
if "size" in self:
info.append("was %s" % self["size"].display)
if self["mtime"].value:
info.append(self["mtime"].display)
return "%s: %s" % (desc, ", ".join(info))
libs/hachoir_parser/archive/mar.py
-67
View File
@@ -1,67 +0,0 @@
"""
Microsoft Archive parser
Author: Victor Stinner
Creation date: 2007-03-04
"""
MAX_NB_FILE = 100000
from hachoir_parser import Parser
from hachoir_core.field import FieldSet, String, UInt32, SubFile
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
class FileIndex(FieldSet):
static_size = 68*8
def createFields(self):
yield String(self, "filename", 56, truncate="\0", charset="ASCII")
yield filesizeHandler(UInt32(self, "filesize"))
yield textHandler(UInt32(self, "crc32"), hexadecimal)
yield UInt32(self, "offset")
def createDescription(self):
return "File %s (%s) at %s" % (
self["filename"].value, self["filesize"].display, self["offset"].value)
class MarFile(Parser):
MAGIC = "MARC"
PARSER_TAGS = {
"id": "mar",
"category": "archive",
"file_ext": ("mar",),
"min_size": 80*8, # At least one file index
"magic": ((MAGIC, 0),),
"description": "Microsoft Archive",
}
endian = LITTLE_ENDIAN
def validate(self):
if self.stream.readBytes(0, 4) != self.MAGIC:
return "Invalid magic"
if self["version"].value != 3:
return "Invalid version"
if not(1 <= self["nb_file"].value <= MAX_NB_FILE):
return "Invalid number of file"
return True
def createFields(self):
yield String(self, "magic", 4, "File signature (MARC)", charset="ASCII")
yield UInt32(self, "version")
yield UInt32(self, "nb_file")
files = []
for index in xrange(self["nb_file"].value):
item = FileIndex(self, "file[]")
yield item
if item["filesize"].value:
files.append(item)
files.sort(key=lambda item: item["offset"].value)
for index in files:
padding = self.seekByte(index["offset"].value)
if padding:
yield padding
size = index["filesize"].value
desc = "File %s" % index["filename"].value
yield SubFile(self, "data[]", size, desc, filename=index["filename"].value)
libs/hachoir_parser/archive/rar.py
-353
View File
@@ -1,353 +0,0 @@
"""
RAR parser
Status: can only read higher-level attructures
Author: Christophe Gisquet
"""
from hachoir_parser import Parser
from hachoir_core.field import (StaticFieldSet, FieldSet,
Bit, Bits, Enum,
UInt8, UInt16, UInt32, UInt64,
String, TimeDateMSDOS32,
NullBytes, NullBits, RawBytes)
from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_parser.common.msdos import MSDOSFileAttr32
MAX_FILESIZE = 1000 * 1024 * 1024
BLOCK_NAME = {
0x72: "Marker",
0x73: "Archive",
0x74: "File",
0x75: "Comment",
0x76: "Extra info",
0x77: "Subblock",
0x78: "Recovery record",
0x79: "Archive authenticity",
0x7A: "New-format subblock",
0x7B: "Archive end",
}
COMPRESSION_NAME = {
0x30: "Storing",
0x31: "Fastest compression",
0x32: "Fast compression",
0x33: "Normal compression",
0x34: "Good compression",
0x35: "Best compression"
}
OS_MSDOS = 0
OS_WIN32 = 2
OS_NAME = {
0: "MS DOS",
1: "OS/2",
2: "Win32",
3: "Unix",
}
DICTIONARY_SIZE = {
0: "Dictionary size 64 Kb",
1: "Dictionary size 128 Kb",
2: "Dictionary size 256 Kb",
3: "Dictionary size 512 Kb",
4: "Dictionary size 1024 Kb",
7: "File is a directory",
}
def formatRARVersion(field):
"""
Decodes the RAR version stored on 1 byte
"""
return "%u.%u" % divmod(field.value, 10)
def commonFlags(s):
yield Bit(s, "has_added_size", "Additional field indicating additional size")
yield Bit(s, "is_ignorable", "Old versions of RAR should ignore this block when copying data")
class ArchiveFlags(StaticFieldSet):
format = (
(Bit, "vol", "Archive volume"),
(Bit, "has_comment", "Whether there is a comment"),
(Bit, "is_locked", "Archive volume"),
(Bit, "is_solid", "Whether files can be extracted separately"),
(Bit, "new_numbering", "New numbering, or compressed comment"), # From unrar
(Bit, "has_authenticity_information", "The integrity/authenticity of the archive can be checked"),
(Bit, "is_protected", "The integrity/authenticity of the archive can be checked"),
(Bit, "is_passworded", "Needs a password to be decrypted"),
(Bit, "is_first_vol", "Whether it is the first volume"),
(Bit, "is_encrypted", "Whether the encryption version is present"),
(NullBits, "internal", 6, "Reserved for 'internal use'")
)
def archiveFlags(s):
yield ArchiveFlags(s, "flags", "Archiver block flags")
def archiveHeader(s):
yield NullBytes(s, "reserved[]", 2, "Reserved word")
yield NullBytes(s, "reserved[]", 4, "Reserved dword")
def commentHeader(s):
yield filesizeHandler(UInt16(s, "total_size", "Comment header size + comment size"))
yield filesizeHandler(UInt16(s, "uncompressed_size", "Uncompressed comment size"))
yield UInt8(s, "required_version", "RAR version needed to extract comment")
yield UInt8(s, "packing_method", "Comment packing method")
yield UInt16(s, "comment_crc16", "Comment CRC")
def commentBody(s):
size = s["total_size"].value - s.current_size
if size > 0:
yield RawBytes(s, "comment_data", size, "Compressed comment data")
def signatureHeader(s):
yield TimeDateMSDOS32(s, "creation_time")
yield filesizeHandler(UInt16(s, "arc_name_size"))
yield filesizeHandler(UInt16(s, "user_name_size"))
def recoveryHeader(s):
yield filesizeHandler(UInt32(s, "total_size"))
yield textHandler(UInt8(s, "version"), hexadecimal)
yield UInt16(s, "rec_sectors")
yield UInt32(s, "total_blocks")
yield RawBytes(s, "mark", 8)
def avInfoHeader(s):
yield filesizeHandler(UInt16(s, "total_size", "Total block size"))
yield UInt8(s, "version", "Version needed to decompress", handler=hexadecimal)
yield UInt8(s, "method", "Compression method", handler=hexadecimal)
yield UInt8(s, "av_version", "Version for AV", handler=hexadecimal)
yield UInt32(s, "av_crc", "AV info CRC32", handler=hexadecimal)
def avInfoBody(s):
size = s["total_size"].value - s.current_size
if size > 0:
yield RawBytes(s, "av_info_data", size, "AV info")
class FileFlags(FieldSet):
static_size = 16
def createFields(self):
yield Bit(self, "continued_from", "File continued from previous volume")
yield Bit(self, "continued_in", "File continued in next volume")
yield Bit(self, "is_encrypted", "File encrypted with password")
yield Bit(self, "has_comment", "File comment present")
yield Bit(self, "is_solid", "Information from previous files is used (solid flag)")
# The 3 following lines are what blocks more staticity
yield Enum(Bits(self, "dictionary_size", 3, "Dictionary size"), DICTIONARY_SIZE)
for bit in commonFlags(self):
yield bit
yield Bit(self, "is_large", "file64 operations needed")
yield Bit(self, "is_unicode", "Filename also encoded using Unicode")
yield Bit(self, "has_salt", "Has salt for encryption")
yield Bit(self, "uses_file_version", "File versioning is used")
yield Bit(self, "has_ext_time", "Extra time ??")
yield Bit(self, "has_ext_flags", "Extra flag ??")
def fileFlags(s):
yield FileFlags(s, "flags", "File block flags")
class ExtTime(FieldSet):
def createFields(self):
yield textHandler(UInt16(self, "time_flags", "Flags for extended time"), hexadecimal)
flags = self["time_flags"].value
for index in xrange(4):
rmode = flags >> ((3-index)*4)
if rmode & 8:
if index:
yield TimeDateMSDOS32(self, "dos_time[]", "DOS Time")
if rmode & 3:
yield RawBytes(self, "remainder[]", rmode & 3, "Time remainder")
def specialHeader(s, is_file):
yield filesizeHandler(UInt32(s, "compressed_size", "Compressed size (bytes)"))
yield filesizeHandler(UInt32(s, "uncompressed_size", "Uncompressed size (bytes)"))
yield Enum(UInt8(s, "host_os", "Operating system used for archiving"), OS_NAME)
yield textHandler(UInt32(s, "crc32", "File CRC32"), hexadecimal)
yield TimeDateMSDOS32(s, "ftime", "Date and time (MS DOS format)")
yield textHandler(UInt8(s, "version", "RAR version needed to extract file"), formatRARVersion)
yield Enum(UInt8(s, "method", "Packing method"), COMPRESSION_NAME)
yield filesizeHandler(UInt16(s, "filename_length", "File name size"))
if s["host_os"].value in (OS_MSDOS, OS_WIN32):
yield MSDOSFileAttr32(s, "file_attr", "File attributes")
else:
yield textHandler(UInt32(s, "file_attr", "File attributes"), hexadecimal)
# Start additional field from unrar
if s["flags/is_large"].value:
yield filesizeHandler(UInt64(s, "large_size", "Extended 64bits filesize"))
# End additional field
size = s["filename_length"].value
if size > 0:
if s["flags/is_unicode"].value:
charset = "UTF-8"
else:
charset = "ISO-8859-15"
yield String(s, "filename", size, "Filename", charset=charset)
# Start additional fields from unrar - file only
if is_file:
if s["flags/has_salt"].value:
yield textHandler(UInt8(s, "salt", "Salt"), hexadecimal)
if s["flags/has_ext_time"].value:
yield ExtTime(s, "extra_time", "Extra time info")
def fileHeader(s):
return specialHeader(s, True)
def fileBody(s):
# File compressed data
size = s["compressed_size"].value
if s["flags/is_large"].value:
size += s["large_size"].value
if size > 0:
yield RawBytes(s, "compressed_data", size, "File compressed data")
def fileDescription(s):
return "File entry: %s (%s)" % \
(s["filename"].display, s["compressed_size"].display)
def newSubHeader(s):
return specialHeader(s, False)
class EndFlags(StaticFieldSet):
format = (
(Bit, "has_next_vol", "Whether there is another next volume"),
(Bit, "has_data_crc", "Whether a CRC value is present"),
(Bit, "rev_space"),
(Bit, "has_vol_number", "Whether the volume number is present"),
(Bits, "unused[]", 4),
(Bit, "has_added_size", "Additional field indicating additional size"),
(Bit, "is_ignorable", "Old versions of RAR should ignore this block when copying data"),
(Bits, "unused[]", 6),
)
def endFlags(s):
yield EndFlags(s, "flags", "End block flags")
class BlockFlags(FieldSet):
static_size = 16
def createFields(self):
yield textHandler(Bits(self, "unused[]", 8, "Unused flag bits"), hexadecimal)
yield Bit(self, "has_added_size", "Additional field indicating additional size")
yield Bit(self, "is_ignorable", "Old versions of RAR should ignore this block when copying data")
yield Bits(self, "unused[]", 6)
class Block(FieldSet):
BLOCK_INFO = {
# None means 'use default function'
0x72: ("marker", "Archive header", None, None, None),
0x73: ("archive_start", "Archive info", archiveFlags, archiveHeader, None),
0x74: ("file[]", fileDescription, fileFlags, fileHeader, fileBody),
0x75: ("comment[]", "Stray comment", None, commentHeader, commentBody),
0x76: ("av_info[]", "Extra information", None, avInfoHeader, avInfoBody),
0x77: ("sub_block[]", "Stray subblock", None, newSubHeader, fileBody),
0x78: ("recovery[]", "Recovery block", None, recoveryHeader, None),
0x79: ("signature", "Signature block", None, signatureHeader, None),
0x7A: ("new_sub_block[]", "Stray new-format subblock", fileFlags,
newSubHeader, fileBody),
0x7B: ("archive_end", "Archive end block", endFlags, None, None),
}
def __init__(self, parent, name):
FieldSet.__init__(self, parent, name)
t = self["block_type"].value
if t in self.BLOCK_INFO:
self._name, desc, parseFlags, parseHeader, parseBody = self.BLOCK_INFO[t]
if callable(desc):
self.createDescription = lambda: desc(self)
elif desc:
self._description = desc
if parseFlags : self.parseFlags = lambda: parseFlags(self)
if parseHeader : self.parseHeader = lambda: parseHeader(self)
if parseBody : self.parseBody = lambda: parseBody(self)
else:
self.info("Processing as unknown block block of type %u" % type)
self._size = 8*self["block_size"].value
if t == 0x74 or t == 0x7A:
self._size += 8*self["compressed_size"].value
if "is_large" in self["flags"] and self["flags/is_large"].value:
self._size += 8*self["large_size"].value
elif "has_added_size" in self:
self._size += 8*self["added_size"].value
# TODO: check if any other member is needed here
def createFields(self):
yield textHandler(UInt16(self, "crc16", "Block CRC16"), hexadecimal)
yield textHandler(UInt8(self, "block_type", "Block type"), hexadecimal)
# Parse flags
for field in self.parseFlags():
yield field
# Get block size
yield filesizeHandler(UInt16(self, "block_size", "Block size"))
# Parse remaining header
for field in self.parseHeader():
yield field
# Finish header with stuff of unknow size
size = self["block_size"].value - (self.current_size//8)
if size > 0:
yield RawBytes(self, "unknown", size, "Unknow data (UInt32 probably)")
# Parse body
for field in self.parseBody():
yield field
def createDescription(self):
return "Block entry: %s" % self["type"].display
def parseFlags(self):
yield BlockFlags(self, "flags", "Block header flags")
def parseHeader(self):
if "has_added_size" in self["flags"] and \
self["flags/has_added_size"].value:
yield filesizeHandler(UInt32(self, "added_size",
"Supplementary block size"))
def parseBody(self):
"""
Parse what is left of the block
"""
size = self["block_size"].value - (self.current_size//8)
if "has_added_size" in self["flags"] and self["flags/has_added_size"].value:
size += self["added_size"].value
if size > 0:
yield RawBytes(self, "body", size, "Body data")
class RarFile(Parser):
MAGIC = "Rar!\x1A\x07\x00"
PARSER_TAGS = {
"id": "rar",
"category": "archive",
"file_ext": ("rar",),
"mime": (u"application/x-rar-compressed", ),
"min_size": 7*8,
"magic": ((MAGIC, 0),),
"description": "Roshal archive (RAR)",
}
endian = LITTLE_ENDIAN
def validate(self):
magic = self.MAGIC
if self.stream.readBytes(0, len(magic)) != magic:
return "Invalid magic"
return True
def createFields(self):
while not self.eof:
yield Block(self, "block[]")
def createContentSize(self):
start = 0
end = MAX_FILESIZE * 8
pos = self.stream.searchBytes("\xC4\x3D\x7B\x00\x40\x07\x00", start, end)
if pos is not None:
return pos + 7*8
return None
libs/hachoir_parser/archive/rpm.py
-267
View File
@@ -1,267 +0,0 @@
"""
RPM archive parser.
Author: Victor Stinner, 1st December 2005.
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet, ParserError,
UInt8, UInt16, UInt32, UInt64, Enum,
NullBytes, Bytes, RawBytes, SubFile,
Character, CString, String)
from hachoir_core.endian import BIG_ENDIAN
from hachoir_parser.archive.gzip_parser import GzipParser
from hachoir_parser.archive.bzip2_parser import Bzip2Parser
class ItemContent(FieldSet):
format_type = {
0: UInt8,
1: Character,
2: UInt8,
3: UInt16,
4: UInt32,
5: UInt64,
6: CString,
7: RawBytes,
8: CString,
9: CString
}
def __init__(self, parent, name, item):
FieldSet.__init__(self, parent, name, item.description)
self.related_item = item
self._name = "content_%s" % item.name
def createFields(self):
item = self.related_item
type = item["type"].value
cls = self.format_type[type]
count = item["count"].value
if cls is RawBytes: # or type == 8:
if cls is RawBytes:
args = (self, "value", count)
else:
args = (self, "value") # cls is CString
count = 1
else:
if 1 < count:
args = (self, "value[]")
else:
args = (self, "value")
for index in xrange(count):
yield cls(*args)
class Item(FieldSet):
type_name = {
0: "NULL",
1: "CHAR",
2: "INT8",
3: "INT16",
4: "INT32",
5: "INT64",
6: "CSTRING",
7: "BIN",
8: "CSTRING_ARRAY",
9: "CSTRING?"
}
tag_name = {
1000: "File size",
1001: "(Broken) MD5 signature",
1002: "PGP 2.6.3 signature",
1003: "(Broken) MD5 signature",
1004: "MD5 signature",
1005: "GnuPG signature",
1006: "PGP5 signature",
1007: "Uncompressed payload size (bytes)",
256+8: "Broken SHA1 header digest",
256+9: "Broken SHA1 header digest",
256+13: "Broken SHA1 header digest",
256+11: "DSA header signature",
256+12: "RSA header signature"
}
def __init__(self, parent, name, description=None, tag_name_dict=None):
FieldSet.__init__(self, parent, name, description)
if tag_name_dict is None:
tag_name_dict = Item.tag_name
self.tag_name_dict = tag_name_dict
def createFields(self):
yield Enum(UInt32(self, "tag", "Tag"), self.tag_name_dict)
yield Enum(UInt32(self, "type", "Type"), Item.type_name)
yield UInt32(self, "offset", "Offset")
yield UInt32(self, "count", "Count")
def createDescription(self):
return "Item: %s (%s)" % (self["tag"].display, self["type"].display)
class ItemHeader(Item):
tag_name = {
61: "Current image",
62: "Signatures",
63: "Immutable",
64: "Regions",
100: "I18N string locales",
1000: "Name",
1001: "Version",
1002: "Release",
1003: "Epoch",
1004: "Summary",
1005: "Description",
1006: "Build time",
1007: "Build host",
1008: "Install time",
1009: "Size",
1010: "Distribution",
1011: "Vendor",
1012: "Gif",
1013: "Xpm",
1014: "Licence",
1015: "Packager",
1016: "Group",
1017: "Changelog",
1018: "Source",
1019: "Patch",
1020: "Url",
1021: "OS",
1022: "Arch",
1023: "Prein",
1024: "Postin",
1025: "Preun",
1026: "Postun",
1027: "Old filenames",
1028: "File sizes",
1029: "File states",
1030: "File modes",
1031: "File uids",
1032: "File gids",
1033: "File rdevs",
1034: "File mtimes",
1035: "File MD5s",
1036: "File link to's",
1037: "File flags",
1038: "Root",
1039: "File username",
1040: "File groupname",
1043: "Icon",
1044: "Source rpm",
1045: "File verify flags",
1046: "Archive size",
1047: "Provide name",
1048: "Require flags",
1049: "Require name",
1050: "Require version",
1051: "No source",
1052: "No patch",
1053: "Conflict flags",
1054: "Conflict name",
1055: "Conflict version",
1056: "Default prefix",
1057: "Build root",
1058: "Install prefix",
1059: "Exclude arch",
1060: "Exclude OS",
1061: "Exclusive arch",
1062: "Exclusive OS",
1064: "RPM version",
1065: "Trigger scripts",
1066: "Trigger name",
1067: "Trigger version",
1068: "Trigger flags",
1069: "Trigger index",
1079: "Verify script",
#TODO: Finish the list (id 1070..1162 using rpm library source code)
}
def __init__(self, parent, name, description=None):
Item.__init__(self, parent, name, description, self.tag_name)
def sortRpmItem(a,b):
return int( a["offset"].value - b["offset"].value )
class PropertySet(FieldSet):
def __init__(self, parent, name, *args):
FieldSet.__init__(self, parent, name, *args)
self._size = self["content_item[1]"].address + self["size"].value * 8
def createFields(self):
# Read chunk header
yield Bytes(self, "signature", 3, r"Property signature (\x8E\xAD\xE8)")
if self["signature"].value != "\x8E\xAD\xE8":
raise ParserError("Invalid property signature")
yield UInt8(self, "version", "Signature version")
yield NullBytes(self, "reserved", 4, "Reserved")
yield UInt32(self, "count", "Count")
yield UInt32(self, "size", "Size")
# Read item header
items = []
for i in range(0, self["count"].value):
item = ItemHeader(self, "item[]")
yield item
items.append(item)
# Sort items by their offset
items.sort( sortRpmItem )
# Read item content
start = self.current_size/8
for item in items:
offset = item["offset"].value
diff = offset - (self.current_size/8 - start)
if 0 < diff:
yield NullBytes(self, "padding[]", diff)
yield ItemContent(self, "content[]", item)
size = start + self["size"].value - self.current_size/8
if 0 < size:
yield NullBytes(self, "padding[]", size)
class RpmFile(Parser):
PARSER_TAGS = {
"id": "rpm",
"category": "archive",
"file_ext": ("rpm",),
"mime": (u"application/x-rpm",),
"min_size": (96 + 16 + 16)*8, # file header + checksum + content header
"magic": (('\xED\xAB\xEE\xDB', 0),),
"description": "RPM package"
}
TYPE_NAME = {
0: "Binary",
1: "Source"
}
endian = BIG_ENDIAN
def validate(self):
if self["signature"].value != '\xED\xAB\xEE\xDB':
return "Invalid signature"
if self["major_ver"].value != 3:
return "Unknown major version (%u)" % self["major_ver"].value
if self["type"].value not in self.TYPE_NAME:
return "Invalid RPM type"
return True
def createFields(self):
yield Bytes(self, "signature", 4, r"RPM file signature (\xED\xAB\xEE\xDB)")
yield UInt8(self, "major_ver", "Major version")
yield UInt8(self, "minor_ver", "Minor version")
yield Enum(UInt16(self, "type", "RPM type"), RpmFile.TYPE_NAME)
yield UInt16(self, "architecture", "Architecture")
yield String(self, "name", 66, "Archive name", strip="\0", charset="ASCII")
yield UInt16(self, "os", "OS")
yield UInt16(self, "signature_type", "Type of signature")
yield NullBytes(self, "reserved", 16, "Reserved")
yield PropertySet(self, "checksum", "Checksum (signature)")
yield PropertySet(self, "header", "Header")
if self._size is None: # TODO: is it possible to handle piped input?
raise NotImplementedError
size = (self._size - self.current_size) // 8
if size:
if 3 <= size and self.stream.readBytes(self.current_size, 3) == "BZh":
yield SubFile(self, "content", size, "bzip2 content", parser=Bzip2Parser)
else:
yield SubFile(self, "content", size, "gzip content", parser=GzipParser)
libs/hachoir_parser/archive/sevenzip.py
-401
View File
@@ -1,401 +0,0 @@
"""
7zip file parser
Informations:
- File 7zformat.txt of 7-zip SDK:
http://www.7-zip.org/sdk.html
Author: Olivier SCHWAB
Creation date: 6 december 2006
"""
from hachoir_parser import Parser
from hachoir_core.field import (Field, FieldSet, ParserError,
GenericVector,
Enum, UInt8, UInt32, UInt64,
Bytes, RawBytes)
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
class SZUInt64(Field):
"""
Variable length UInt64, where the first byte gives both the number of bytes
needed and the upper byte value.
"""
def __init__(self, parent, name, max_size=None, description=None):
Field.__init__(self, parent, name, size=8, description=description)
value = 0
addr = self.absolute_address
mask = 0x80
firstByte = parent.stream.readBits(addr, 8, LITTLE_ENDIAN)
for i in xrange(8):
addr += 8
if not (firstByte & mask):
value += ((firstByte & (mask-1)) << (8*i))
break
value |= (parent.stream.readBits(addr, 8, LITTLE_ENDIAN) << (8*i))
mask >>= 1
self._size += 8
self.createValue = lambda: value
ID_END, ID_HEADER, ID_ARCHIVE_PROPS, ID_ADD_STREAM_INFO, ID_MAIN_STREAM_INFO, \
ID_FILES_INFO, ID_PACK_INFO, ID_UNPACK_INFO, ID_SUBSTREAMS_INFO, ID_SIZE, \
ID_CRC, ID_FOLDER, ID_CODERS_UNPACK_SIZE, ID_NUM_UNPACK_STREAMS, \
ID_EMPTY_STREAM, ID_EMPTY_FILE, ID_ANTI, ID_NAME, ID_CREATION_TIME, \
ID_LAST_ACCESS_TIME, ID_LAST_WRITE_TIME, ID_WIN_ATTR, ID_COMMENT, \
ID_ENCODED_HEADER = xrange(24)
ID_INFO = {
ID_END : "End",
ID_HEADER : "Header embedding another one",
ID_ARCHIVE_PROPS : "Archive Properties",
ID_ADD_STREAM_INFO : "Additional Streams Info",
ID_MAIN_STREAM_INFO : "Main Streams Info",
ID_FILES_INFO : "Files Info",
ID_PACK_INFO : "Pack Info",
ID_UNPACK_INFO : "Unpack Info",
ID_SUBSTREAMS_INFO : "Substreams Info",
ID_SIZE : "Size",
ID_CRC : "CRC",
ID_FOLDER : "Folder",
ID_CODERS_UNPACK_SIZE: "Coders Unpacked size",
ID_NUM_UNPACK_STREAMS: "Number of Unpacked Streams",
ID_EMPTY_STREAM : "Empty Stream",
ID_EMPTY_FILE : "Empty File",
ID_ANTI : "Anti",
ID_NAME : "Name",
ID_CREATION_TIME : "Creation Time",
ID_LAST_ACCESS_TIME : "Last Access Time",
ID_LAST_WRITE_TIME : "Last Write Time",
ID_WIN_ATTR : "Win Attributes",
ID_COMMENT : "Comment",
ID_ENCODED_HEADER : "Header holding encoded data info",
}
class SkippedData(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "id[]"), ID_INFO)
size = SZUInt64(self, "size")
yield size
if size.value > 0:
yield RawBytes(self, "data", size.value)
def waitForID(s, wait_id, wait_name="waited_id[]"):
while not s.eof:
addr = s.absolute_address+s.current_size
uid = s.stream.readBits(addr, 8, LITTLE_ENDIAN)
if uid == wait_id:
yield Enum(UInt8(s, wait_name), ID_INFO)
s.info("Found ID %s (%u)" % (ID_INFO[uid], uid))
return
s.info("Skipping ID %u!=%u" % (uid, wait_id))
yield SkippedData(s, "skipped_id[]", "%u != %u" % (uid, wait_id))
class HashDigest(FieldSet):
def __init__(self, parent, name, num_digests, desc=None):
FieldSet.__init__(self, parent, name, desc)
self.num_digests = num_digests
def createFields(self):
yield Enum(UInt8(self, "id"), ID_INFO)
bytes = self.stream.readBytes(self.absolute_address, self.num_digests)
if self.num_digests > 0:
yield GenericVector(self, "defined[]", self.num_digests, UInt8, "bool")
for index in xrange(self.num_digests):
if bytes[index]:
yield textHandler(UInt32(self, "hash[]",
"Hash for digest %u" % index), hexadecimal)
class PackInfo(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "id"), ID_INFO)
# Very important, helps determine where the data is
yield SZUInt64(self, "pack_pos", "Position of the packs")
num = SZUInt64(self, "num_pack_streams")
yield num
num = num.value
for field in waitForID(self, ID_SIZE, "size_marker"):
yield field
for size in xrange(num):
yield SZUInt64(self, "pack_size[]")
while not self.eof:
addr = self.absolute_address+self.current_size
uid = self.stream.readBits(addr, 8, LITTLE_ENDIAN)
if uid == ID_END:
yield Enum(UInt8(self, "end_marker"), ID_INFO)
break
elif uid == ID_CRC:
yield HashDigest(self, "hash_digest", size)
else:
yield SkippedData(self, "skipped_data")
def lzmaParams(value):
param = value.value
remainder = param / 9
# Literal coder context bits
lc = param % 9
# Position state bits
pb = remainder / 5
# Literal coder position bits
lp = remainder % 5
return "lc=%u pb=%u lp=%u" % (lc, lp, pb)
class CoderID(FieldSet):
CODECS = {
# Only 2 methods ... and what about PPMD ?
"\0" : "copy",
"\3\1\1": "lzma",
}
def createFields(self):
byte = UInt8(self, "id_size")
yield byte
byte = byte.value
self.info("ID=%u" % byte)
size = byte & 0xF
if size > 0:
name = self.stream.readBytes(self.absolute_address+self.current_size, size)
if name in self.CODECS:
name = self.CODECS[name]
self.info("Codec is %s" % name)
else:
self.info("Undetermined codec %s" % name)
name = "unknown"
yield RawBytes(self, name, size)
#yield textHandler(Bytes(self, "id", size), lambda: name)
if byte & 0x10:
yield SZUInt64(self, "num_stream_in")
yield SZUInt64(self, "num_stream_out")
self.info("Streams: IN=%u OUT=%u" % \
(self["num_stream_in"].value, self["num_stream_out"].value))
if byte & 0x20:
size = SZUInt64(self, "properties_size[]")
yield size
if size.value == 5:
#LzmaDecodeProperties@LZMAStateDecode.c
yield textHandler(UInt8(self, "parameters"), lzmaParams)
yield filesizeHandler(UInt32(self, "dictionary_size"))
elif size.value > 0:
yield RawBytes(self, "properties[]", size.value)
class CoderInfo(FieldSet):
def __init__(self, parent, name, desc=None):
FieldSet.__init__(self, parent, name, desc)
self.in_streams = 1
self.out_streams = 1
def createFields(self):
# The real ID
addr = self.absolute_address + self.current_size
b = self.parent.stream.readBits(addr, 8, LITTLE_ENDIAN)
cid = CoderID(self, "coder_id")
yield cid
if b&0x10: # Work repeated, ...
self.in_streams = cid["num_stream_in"].value
self.out_streams = cid["num_stream_out"].value
# Skip other IDs
while b&0x80:
addr = self.absolute_address + self.current_size
b = self.parent.stream.readBits(addr, 8, LITTLE_ENDIAN)
yield CoderID(self, "unused_codec_id[]")
class BindPairInfo(FieldSet):
def createFields(self):
# 64 bits values then cast to 32 in fact
yield SZUInt64(self, "in_index")
yield SZUInt64(self, "out_index")
self.info("Indexes: IN=%u OUT=%u" % \
(self["in_index"].value, self["out_index"].value))
class FolderItem(FieldSet):
def __init__(self, parent, name, desc=None):
FieldSet.__init__(self, parent, name, desc)
self.in_streams = 0
self.out_streams = 0
def createFields(self):
yield SZUInt64(self, "num_coders")
num = self["num_coders"].value
self.info("Folder: %u codecs" % num)
# Coders info
for index in xrange(num):
ci = CoderInfo(self, "coder_info[]")
yield ci
self.in_streams += ci.in_streams
self.out_streams += ci.out_streams
# Bin pairs
self.info("out streams: %u" % self.out_streams)
for index in xrange(self.out_streams-1):
yield BindPairInfo(self, "bind_pair[]")
# Packed streams
# @todo: Actually find mapping
packed_streams = self.in_streams - self.out_streams + 1
if packed_streams == 1:
pass
else:
for index in xrange(packed_streams):
yield SZUInt64(self, "pack_stream[]")
class UnpackInfo(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "id"), ID_INFO)
# Wait for synch
for field in waitForID(self, ID_FOLDER, "folder_marker"):
yield field
yield SZUInt64(self, "num_folders")
# Get generic info
num = self["num_folders"].value
self.info("%u folders" % num)
yield UInt8(self, "is_external")
# Read folder items
for folder_index in xrange(num):
yield FolderItem(self, "folder_item[]")
# Get unpack sizes for each coder of each folder
for field in waitForID(self, ID_CODERS_UNPACK_SIZE, "coders_unpsize_marker"):
yield field
for folder_index in xrange(num):
folder_item = self["folder_item[%u]" % folder_index]
for index in xrange(folder_item.out_streams):
#yield UInt8(self, "unpack_size[]")
yield SZUInt64(self, "unpack_size[]")
# Extract digests
while not self.eof:
addr = self.absolute_address+self.current_size
uid = self.stream.readBits(addr, 8, LITTLE_ENDIAN)
if uid == ID_END:
yield Enum(UInt8(self, "end_marker"), ID_INFO)
break
elif uid == ID_CRC:
yield HashDigest(self, "hash_digest", num)
else:
yield SkippedData(self, "skip_data")
class SubStreamInfo(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "id"), ID_INFO)
raise ParserError("SubStreamInfo not implemented yet")
class EncodedHeader(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "id"), ID_INFO)
while not self.eof:
addr = self.absolute_address+self.current_size
uid = self.stream.readBits(addr, 8, LITTLE_ENDIAN)
if uid == ID_END:
yield Enum(UInt8(self, "end_marker"), ID_INFO)
break
elif uid == ID_PACK_INFO:
yield PackInfo(self, "pack_info", ID_INFO[ID_PACK_INFO])
elif uid == ID_UNPACK_INFO:
yield UnpackInfo(self, "unpack_info", ID_INFO[ID_UNPACK_INFO])
elif uid == ID_SUBSTREAMS_INFO:
yield SubStreamInfo(self, "substreams_info", ID_INFO[ID_SUBSTREAMS_INFO])
else:
self.info("Unexpected ID (%i)" % uid)
break
class IDHeader(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "id"), ID_INFO)
ParserError("IDHeader not implemented")
class NextHeader(FieldSet):
def __init__(self, parent, name, desc="Next header"):
FieldSet.__init__(self, parent, name, desc)
self._size = 8*self["/signature/start_hdr/next_hdr_size"].value
# Less work, as much interpretable information as the other
# version... what an obnoxious format
def createFields2(self):
yield Enum(UInt8(self, "header_type"), ID_INFO)
yield RawBytes(self, "header_data", self._size-1)
def createFields(self):
uid = self.stream.readBits(self.absolute_address, 8, LITTLE_ENDIAN)
if uid == ID_HEADER:
yield IDHeader(self, "header", ID_INFO[ID_HEADER])
elif uid == ID_ENCODED_HEADER:
yield EncodedHeader(self, "encoded_hdr", ID_INFO[ID_ENCODED_HEADER])
# Game Over: this is usually encoded using LZMA, not copy
# See SzReadAndDecodePackedStreams/SzDecode being called with the
# data position from "/next_hdr/encoded_hdr/pack_info/pack_pos"
# We should process further, yet we can't...
else:
ParserError("Unexpected ID %u" % uid)
size = self._size - self.current_size
if size > 0:
yield RawBytes(self, "next_hdr_data", size//8, "Next header's data")
class Body(FieldSet):
def __init__(self, parent, name, desc="Body data"):
FieldSet.__init__(self, parent, name, desc)
self._size = 8*self["/signature/start_hdr/next_hdr_offset"].value
def createFields(self):
if "encoded_hdr" in self["/next_hdr/"]:
pack_size = sum([s.value for s in self.array("/next_hdr/encoded_hdr/pack_info/pack_size")])
body_size = self["/next_hdr/encoded_hdr/pack_info/pack_pos"].value
yield RawBytes(self, "compressed_data", body_size, "Compressed data")
# Here we could check if copy method was used to "compress" it,
# but this never happens, so just output "compressed file info"
yield RawBytes(self, "compressed_file_info", pack_size,
"Compressed file information")
size = (self._size//8) - pack_size - body_size
if size > 0:
yield RawBytes(self, "unknown_data", size)
elif "header" in self["/next_hdr"]:
yield RawBytes(self, "compressed_data", self._size//8, "Compressed data")
class StartHeader(FieldSet):
static_size = 160
def createFields(self):
yield textHandler(UInt64(self, "next_hdr_offset",
"Next header offset"), hexadecimal)
yield UInt64(self, "next_hdr_size", "Next header size")
yield textHandler(UInt32(self, "next_hdr_crc",
"Next header CRC"), hexadecimal)
class SignatureHeader(FieldSet):
static_size = 96 + StartHeader.static_size
def createFields(self):
yield Bytes(self, "signature", 6, "Signature Header")
yield UInt8(self, "major_ver", "Archive major version")
yield UInt8(self, "minor_ver", "Archive minor version")
yield textHandler(UInt32(self, "start_hdr_crc",
"Start header CRC"), hexadecimal)
yield StartHeader(self, "start_hdr", "Start header")
class SevenZipParser(Parser):
PARSER_TAGS = {
"id": "7zip",
"category": "archive",
"file_ext": ("7z",),
"mime": (u"application/x-7z-compressed",),
"min_size": 32*8,
"magic": (("7z\xbc\xaf\x27\x1c", 0),),
"description": "Compressed archive in 7z format"
}
endian = LITTLE_ENDIAN
def createFields(self):
yield SignatureHeader(self, "signature", "Signature Header")
yield Body(self, "body_data")
yield NextHeader(self, "next_hdr")
def validate(self):
if self.stream.readBytes(0,6) != "7z\xbc\xaf'\x1c":
return "Invalid signature"
return True
def createContentSize(self):
size = self["/signature/start_hdr/next_hdr_offset"].value
size += self["/signature/start_hdr/next_hdr_size"].value
size += 12 # Signature size
size += 20 # Start header size
return size*8
libs/hachoir_parser/archive/tar.py
-124
View File
@@ -1,124 +0,0 @@
"""
Tar archive parser.
Author: Victor Stinner
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet,
Enum, UInt8, SubFile, String, NullBytes)
from hachoir_core.tools import humanFilesize, paddingSize, timestampUNIX
from hachoir_core.endian import BIG_ENDIAN
import re
class FileEntry(FieldSet):
type_name = {
# 48 is "0", 49 is "1", ...
0: u"Normal disk file (old format)",
48: u"Normal disk file",
49: u"Link to previously dumped file",
50: u"Symbolic link",
51: u"Character special file",
52: u"Block special file",
53: u"Directory",
54: u"FIFO special file",
55: u"Contiguous file"
}
def getOctal(self, name):
return self.octal2int(self[name].value)
def getDatetime(self):
"""
Create modification date as Unicode string, may raise ValueError.
"""
timestamp = self.getOctal("mtime")
return timestampUNIX(timestamp)
def createFields(self):
yield String(self, "name", 100, "Name", strip="\0", charset="ISO-8859-1")
yield String(self, "mode", 8, "Mode", strip=" \0", charset="ASCII")
yield String(self, "uid", 8, "User ID", strip=" \0", charset="ASCII")
yield String(self, "gid", 8, "Group ID", strip=" \0", charset="ASCII")
yield String(self, "size", 12, "Size", strip=" \0", charset="ASCII")
yield String(self, "mtime", 12, "Modification time", strip=" \0", charset="ASCII")
yield String(self, "check_sum", 8, "Check sum", strip=" \0", charset="ASCII")
yield Enum(UInt8(self, "type", "Type"), self.type_name)
yield String(self, "lname", 100, "Link name", strip=" \0", charset="ISO-8859-1")
yield String(self, "magic", 8, "Magic", strip=" \0", charset="ASCII")
yield String(self, "uname", 32, "User name", strip=" \0", charset="ISO-8859-1")
yield String(self, "gname", 32, "Group name", strip=" \0", charset="ISO-8859-1")
yield String(self, "devmajor", 8, "Dev major", strip=" \0", charset="ASCII")
yield String(self, "devminor", 8, "Dev minor", strip=" \0", charset="ASCII")
yield NullBytes(self, "padding", 167, "Padding (zero)")
filesize = self.getOctal("size")
if filesize:
yield SubFile(self, "content", filesize, filename=self["name"].value)
size = paddingSize(self.current_size//8, 512)
if size:
yield NullBytes(self, "padding_end", size, "Padding (512 align)")
def convertOctal(self, chunk):
return self.octal2int(chunk.value)
def isEmpty(self):
return self["name"].value == ""
def octal2int(self, text):
try:
return int(text, 8)
except ValueError:
return 0
def createDescription(self):
if self.isEmpty():
desc = "(terminator, empty header)"
else:
filename = self["name"].value
filesize = humanFilesize(self.getOctal("size"))
desc = "(%s: %s, %s)" % \
(filename, self["type"].display, filesize)
return "Tar File " + desc
class TarFile(Parser):
endian = BIG_ENDIAN
PARSER_TAGS = {
"id": "tar",
"category": "archive",
"file_ext": ("tar",),
"mime": (u"application/x-tar", u"application/x-gtar"),
"min_size": 512*8,
"magic": (("ustar \0", 257*8),),
"subfile": "skip",
"description": "TAR archive",
}
_sign = re.compile("ustar *\0|[ \0]*$")
def validate(self):
if not self._sign.match(self.stream.readBytes(257*8, 8)):
return "Invalid magic number"
if self[0].name == "terminator":
return "Don't contain any file"
try:
int(self["file[0]/uid"].value, 8)
int(self["file[0]/gid"].value, 8)
int(self["file[0]/size"].value, 8)
except ValueError:
return "Invalid file size"
return True
def createFields(self):
while not self.eof:
field = FileEntry(self, "file[]")
if field.isEmpty():
yield NullBytes(self, "terminator", 512)
break
yield field
if self.current_size < self._size:
yield self.seekBit(self._size, "end")
def createContentSize(self):
return self["terminator"].address + self["terminator"].size
libs/hachoir_parser/archive/zip.py
-427
View File
@@ -1,427 +0,0 @@
"""
Zip splitter.
Status: can read most important headers
Authors: Christophe Gisquet and Victor Stinner
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet, ParserError,
Bit, Bits, Enum,
TimeDateMSDOS32, SubFile,
UInt8, UInt16, UInt32, UInt64,
String, PascalString16,
RawBytes)
from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
from hachoir_core.error import HACHOIR_ERRORS
from hachoir_core.tools import makeUnicode
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_parser.common.deflate import Deflate
MAX_FILESIZE = 1000 * 1024 * 1024
COMPRESSION_DEFLATE = 8
COMPRESSION_METHOD = {
0: u"no compression",
1: u"Shrunk",
2: u"Reduced (factor 1)",
3: u"Reduced (factor 2)",
4: u"Reduced (factor 3)",
5: u"Reduced (factor 4)",
6: u"Imploded",
7: u"Tokenizing",
8: u"Deflate",
9: u"Deflate64",
10: u"PKWARE Imploding",
11: u"Reserved by PKWARE",
12: u"File is compressed using BZIP2 algorithm",
13: u"Reserved by PKWARE",
14: u"LZMA (EFS)",
15: u"Reserved by PKWARE",
16: u"Reserved by PKWARE",
17: u"Reserved by PKWARE",
18: u"File is compressed using IBM TERSE (new)",
19: u"IBM LZ77 z Architecture (PFS)",
98: u"PPMd version I, Rev 1",
}
def ZipRevision(field):
return "%u.%u" % divmod(field.value, 10)
class ZipVersion(FieldSet):
static_size = 16
HOST_OS = {
0: u"FAT file system (DOS, OS/2, NT)",
1: u"Amiga",
2: u"VMS (VAX or Alpha AXP)",
3: u"Unix",
4: u"VM/CMS",
5: u"Atari",
6: u"HPFS file system (OS/2, NT 3.x)",
7: u"Macintosh",
8: u"Z-System",
9: u"CP/M",
10: u"TOPS-20",
11: u"NTFS file system (NT)",
12: u"SMS/QDOS",
13: u"Acorn RISC OS",
14: u"VFAT file system (Win95, NT)",
15: u"MVS",
16: u"BeOS (BeBox or PowerMac)",
17: u"Tandem",
}
def createFields(self):
yield textHandler(UInt8(self, "zip_version", "ZIP version"), ZipRevision)
yield Enum(UInt8(self, "host_os", "ZIP Host OS"), self.HOST_OS)
class ZipGeneralFlags(FieldSet):
static_size = 16
def createFields(self):
# Need the compression info from the parent, and that is the byte following
method = self.stream.readBits(self.absolute_address+16, 16, LITTLE_ENDIAN)
yield Bits(self, "unused[]", 2, "Unused")
yield Bit(self, "encrypted_central_dir", "Selected data values in the Local Header are masked")
yield Bit(self, "incomplete", "Reserved by PKWARE for enhanced compression.")
yield Bit(self, "uses_unicode", "Filename and comments are in UTF-8")
yield Bits(self, "unused[]", 4, "Unused")
yield Bit(self, "strong_encrypt", "Strong encryption (version >= 50)")
yield Bit(self, "is_patched", "File is compressed with patched data?")
yield Bit(self, "enhanced_deflate", "Reserved for use with method 8")
yield Bit(self, "has_descriptor",
"Compressed data followed by descriptor?")
if method == 6:
yield Bit(self, "use_8k_sliding", "Use 8K sliding dictionary (instead of 4K)")
yield Bit(self, "use_3shannon", "Use a 3 Shannon-Fano tree (instead of 2 Shannon-Fano)")
elif method in (8, 9):
NAME = {
0: "Normal compression",
1: "Maximum compression",
2: "Fast compression",
3: "Super Fast compression"
}
yield Enum(Bits(self, "method", 2), NAME)
elif method == 14: #LZMA
yield Bit(self, "lzma_eos", "LZMA stream is ended with a EndOfStream marker")
yield Bit(self, "unused[]")
else:
yield Bits(self, "compression_info", 2)
yield Bit(self, "is_encrypted", "File is encrypted?")
class ExtraField(FieldSet):
EXTRA_FIELD_ID = {
0x0007: "AV Info",
0x0009: "OS/2 extended attributes (also Info-ZIP)",
0x000a: "PKWARE Win95/WinNT FileTimes", # undocumented!
0x000c: "PKWARE VAX/VMS (also Info-ZIP)",
0x000d: "PKWARE Unix",
0x000f: "Patch Descriptor",
0x07c8: "Info-ZIP Macintosh (old, J. Lee)",
0x2605: "ZipIt Macintosh (first version)",
0x2705: "ZipIt Macintosh v 1.3.5 and newer (w/o full filename)",
0x334d: "Info-ZIP Macintosh (new, D. Haase Mac3 field)",
0x4341: "Acorn/SparkFS (David Pilling)",
0x4453: "Windows NT security descriptor (binary ACL)",
0x4704: "VM/CMS",
0x470f: "MVS",
0x4b46: "FWKCS MD5 (third party, see below)",
0x4c41: "OS/2 access control list (text ACL)",
0x4d49: "Info-ZIP VMS (VAX or Alpha)",
0x5356: "AOS/VS (binary ACL)",
0x5455: "extended timestamp",
0x5855: "Info-ZIP Unix (original; also OS/2, NT, etc.)",
0x6542: "BeOS (BeBox, PowerMac, etc.)",
0x756e: "ASi Unix",
0x7855: "Info-ZIP Unix (new)",
0xfb4a: "SMS/QDOS",
}
def createFields(self):
yield Enum(UInt16(self, "field_id", "Extra field ID"),
self.EXTRA_FIELD_ID)
size = UInt16(self, "field_data_size", "Extra field data size")
yield size
if size.value > 0:
yield RawBytes(self, "field_data", size, "Unknown field data")
def ZipStartCommonFields(self):
yield ZipVersion(self, "version_needed", "Version needed")
yield ZipGeneralFlags(self, "flags", "General purpose flag")
yield Enum(UInt16(self, "compression", "Compression method"),
COMPRESSION_METHOD)
yield TimeDateMSDOS32(self, "last_mod", "Last modification file time")
yield textHandler(UInt32(self, "crc32", "CRC-32"), hexadecimal)
yield UInt32(self, "compressed_size", "Compressed size")
yield UInt32(self, "uncompressed_size", "Uncompressed size")
yield UInt16(self, "filename_length", "Filename length")
yield UInt16(self, "extra_length", "Extra fields length")
def zipGetCharset(self):
if self["flags/uses_unicode"].value:
return "UTF-8"
else:
return "ISO-8859-15"
class ZipCentralDirectory(FieldSet):
HEADER = 0x02014b50
def createFields(self):
yield ZipVersion(self, "version_made_by", "Version made by")
for field in ZipStartCommonFields(self):
yield field
# Check unicode status
charset = zipGetCharset(self)
yield UInt16(self, "comment_length", "Comment length")
yield UInt16(self, "disk_number_start", "Disk number start")
yield UInt16(self, "internal_attr", "Internal file attributes")
yield UInt32(self, "external_attr", "External file attributes")
yield UInt32(self, "offset_header", "Relative offset of local header")
yield String(self, "filename", self["filename_length"].value,
"Filename", charset=charset)
if 0 < self["extra_length"].value:
yield RawBytes(self, "extra", self["extra_length"].value,
"Extra fields")
if 0 < self["comment_length"].value:
yield String(self, "comment", self["comment_length"].value,
"Comment", charset=charset)
def createDescription(self):
return "Central directory: %s" % self["filename"].display
class Zip64EndCentralDirectory(FieldSet):
HEADER = 0x06064b50
def createFields(self):
yield UInt64(self, "zip64_end_size",
"Size of zip64 end of central directory record")
yield ZipVersion(self, "version_made_by", "Version made by")
yield ZipVersion(self, "version_needed", "Version needed to extract")
yield UInt32(self, "number_disk", "Number of this disk")
yield UInt32(self, "number_disk2",
"Number of the disk with the start of the central directory")
yield UInt64(self, "number_entries",
"Total number of entries in the central directory on this disk")
yield UInt64(self, "number_entries2",
"Total number of entries in the central directory")
yield UInt64(self, "size", "Size of the central directory")
yield UInt64(self, "offset", "Offset of start of central directory")
if 0 < self["zip64_end_size"].value:
yield RawBytes(self, "data_sector", self["zip64_end_size"].value,
"zip64 extensible data sector")
class ZipEndCentralDirectory(FieldSet):
HEADER = 0x06054b50
def createFields(self):
yield UInt16(self, "number_disk", "Number of this disk")
yield UInt16(self, "number_disk2", "Number in the central dir")
yield UInt16(self, "total_number_disk",
"Total number of entries in this disk")
yield UInt16(self, "total_number_disk2",
"Total number of entries in the central dir")
yield UInt32(self, "size", "Size of the central directory")
yield UInt32(self, "offset", "Offset of start of central directory")
yield PascalString16(self, "comment", "ZIP comment")
class ZipDataDescriptor(FieldSet):
HEADER_STRING = "\x50\x4B\x07\x08"
HEADER = 0x08074B50
static_size = 96
def createFields(self):
yield textHandler(UInt32(self, "file_crc32",
"Checksum (CRC32)"), hexadecimal)
yield filesizeHandler(UInt32(self, "file_compressed_size",
"Compressed size (bytes)"))
yield filesizeHandler(UInt32(self, "file_uncompressed_size",
"Uncompressed size (bytes)"))
class FileEntry(FieldSet):
HEADER = 0x04034B50
filename = None
def data(self, size):
compression = self["compression"].value
if compression == 0:
return SubFile(self, "data", size, filename=self.filename)
compressed = SubFile(self, "compressed_data", size, filename=self.filename)
if compression == COMPRESSION_DEFLATE:
return Deflate(compressed)
else:
return compressed
def resync(self):
# Non-seekable output, search the next data descriptor
size = self.stream.searchBytesLength(ZipDataDescriptor.HEADER_STRING, False,
self.absolute_address+self.current_size)
if size <= 0:
raise ParserError("Couldn't resync to %s" %
ZipDataDescriptor.HEADER_STRING)
yield self.data(size)
yield textHandler(UInt32(self, "header[]", "Header"), hexadecimal)
data_desc = ZipDataDescriptor(self, "data_desc", "Data descriptor")
#self.info("Resynced!")
yield data_desc
# The above could be checked anytime, but we prefer trying parsing
# than aborting
if self["crc32"].value == 0 and \
data_desc["file_compressed_size"].value != size:
raise ParserError("Bad resync: position=>%i but data_desc=>%i" %
(size, data_desc["file_compressed_size"].value))
def createFields(self):
for field in ZipStartCommonFields(self):
yield field
length = self["filename_length"].value
if length:
filename = String(self, "filename", length, "Filename",
charset=zipGetCharset(self))
yield filename
self.filename = filename.value
if self["extra_length"].value:
yield RawBytes(self, "extra", self["extra_length"].value, "Extra")
size = self["compressed_size"].value
if size > 0:
yield self.data(size)
elif self["flags/incomplete"].value:
for field in self.resync():
yield field
if self["flags/has_descriptor"].value:
yield ZipDataDescriptor(self, "data_desc", "Data descriptor")
def createDescription(self):
return "File entry: %s (%s)" % \
(self["filename"].value, self["compressed_size"].display)
def validate(self):
if self["compression"].value not in COMPRESSION_METHOD:
return "Unknown compression method (%u)" % self["compression"].value
return ""
class ZipSignature(FieldSet):
HEADER = 0x05054B50
def createFields(self):
yield PascalString16(self, "signature", "Signature")
class Zip64EndCentralDirectoryLocator(FieldSet):
HEADER = 0x07064b50
def createFields(self):
yield UInt32(self, "disk_number", \
"Number of the disk with the start of the zip64 end of central directory")
yield UInt64(self, "relative_offset", \
"Relative offset of the zip64 end of central directory record")
yield UInt32(self, "disk_total_number", "Total number of disks")
class ZipFile(Parser):
endian = LITTLE_ENDIAN
MIME_TYPES = {
# Default ZIP archive
u"application/zip": "zip",
u"application/x-zip": "zip",
# Java archive (JAR)
u"application/x-jar": "jar",
u"application/java-archive": "jar",
# OpenOffice 1.0
u"application/vnd.sun.xml.calc": "sxc",
u"application/vnd.sun.xml.draw": "sxd",
u"application/vnd.sun.xml.impress": "sxi",
u"application/vnd.sun.xml.writer": "sxw",
u"application/vnd.sun.xml.math": "sxm",
# OpenOffice 1.0 (template)
u"application/vnd.sun.xml.calc.template": "stc",
u"application/vnd.sun.xml.draw.template": "std",
u"application/vnd.sun.xml.impress.template": "sti",
u"application/vnd.sun.xml.writer.template": "stw",
u"application/vnd.sun.xml.writer.global": "sxg",
# OpenDocument
u"application/vnd.oasis.opendocument.chart": "odc",
u"application/vnd.oasis.opendocument.image": "odi",
u"application/vnd.oasis.opendocument.database": "odb",
u"application/vnd.oasis.opendocument.formula": "odf",
u"application/vnd.oasis.opendocument.graphics": "odg",
u"application/vnd.oasis.opendocument.presentation": "odp",
u"application/vnd.oasis.opendocument.spreadsheet": "ods",
u"application/vnd.oasis.opendocument.text": "odt",
u"application/vnd.oasis.opendocument.text-master": "odm",
# OpenDocument (template)
u"application/vnd.oasis.opendocument.graphics-template": "otg",
u"application/vnd.oasis.opendocument.presentation-template": "otp",
u"application/vnd.oasis.opendocument.spreadsheet-template": "ots",
u"application/vnd.oasis.opendocument.text-template": "ott",
}
PARSER_TAGS = {
"id": "zip",
"category": "archive",
"file_ext": tuple(MIME_TYPES.itervalues()),
"mime": tuple(MIME_TYPES.iterkeys()),
"magic": (("PK\3\4", 0),),
"subfile": "skip",
"min_size": (4 + 26)*8, # header + file entry
"description": "ZIP archive"
}
def validate(self):
if self["header[0]"].value != FileEntry.HEADER:
return "Invalid magic"
try:
file0 = self["file[0]"]
except HACHOIR_ERRORS, err:
return "Unable to get file #0"
err = file0.validate()
if err:
return "File #0: %s" % err
return True
def createFields(self):
# File data
self.signature = None
self.central_directory = []
while not self.eof:
header = textHandler(UInt32(self, "header[]", "Header"), hexadecimal)
yield header
header = header.value
if header == FileEntry.HEADER:
yield FileEntry(self, "file[]")
elif header == ZipDataDescriptor.HEADER:
yield ZipDataDescriptor(self, "spanning[]")
elif header == 0x30304b50:
yield ZipDataDescriptor(self, "temporary_spanning[]")
elif header == ZipCentralDirectory.HEADER:
yield ZipCentralDirectory(self, "central_directory[]")
elif header == ZipEndCentralDirectory.HEADER:
yield ZipEndCentralDirectory(self, "end_central_directory", "End of central directory")
elif header == Zip64EndCentralDirectory.HEADER:
yield Zip64EndCentralDirectory(self, "end64_central_directory", "ZIP64 end of central directory")
elif header == ZipSignature.HEADER:
yield ZipSignature(self, "signature", "Signature")
elif header == Zip64EndCentralDirectoryLocator.HEADER:
yield Zip64EndCentralDirectoryLocator(self, "end_locator", "ZIP64 Enf of central directory locator")
else:
raise ParserError("Error, unknown ZIP header (0x%08X)." % header)
def createMimeType(self):
if self["file[0]/filename"].value == "mimetype":
return makeUnicode(self["file[0]/data"].value)
else:
return u"application/zip"
def createFilenameSuffix(self):
if self["file[0]/filename"].value == "mimetype":
mime = self["file[0]/compressed_data"].value
if mime in self.MIME_TYPES:
return "." + self.MIME_TYPES[mime]
return ".zip"
def createContentSize(self):
start = 0
end = MAX_FILESIZE * 8
end = self.stream.searchBytes("PK\5\6", start, end)
if end is not None:
return end + 22*8
return None
libs/hachoir_parser/audio/__init__.py
-12
View File
@@ -1,12 +0,0 @@
from hachoir_parser.audio.aiff import AiffFile
from hachoir_parser.audio.au import AuFile
from hachoir_parser.audio.itunesdb import ITunesDBFile
from hachoir_parser.audio.midi import MidiFile
from hachoir_parser.audio.mpeg_audio import MpegAudioFile
from hachoir_parser.audio.real_audio import RealAudioFile
from hachoir_parser.audio.xm import XMModule
from hachoir_parser.audio.s3m import S3MModule
from hachoir_parser.audio.s3m import PTMModule
from hachoir_parser.audio.mod import AmigaModule
from hachoir_parser.audio.flac import FlacParser
libs/hachoir_parser/audio/aiff.py
-127
View File
@@ -1,127 +0,0 @@
"""
Audio Interchange File Format (AIFF) parser.
Author: Victor Stinner
Creation: 27 december 2006
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet,
UInt16, UInt32, Float80, TimestampMac32,
RawBytes, NullBytes,
String, Enum, PascalString32)
from hachoir_core.endian import BIG_ENDIAN
from hachoir_core.text_handler import filesizeHandler
from hachoir_core.tools import alignValue
from hachoir_parser.audio.id3 import ID3v2
CODEC_NAME = {
'ACE2': u"ACE 2-to-1",
'ACE8': u"ACE 8-to-3",
'MAC3': u"MAC 3-to-1",
'MAC6': u"MAC 6-to-1",
'NONE': u"None",
'sowt': u"Little-endian, no compression",
}
class Comment(FieldSet):
def createFields(self):
yield TimestampMac32(self, "timestamp")
yield PascalString32(self, "text")
def parseText(self):
yield String(self, "text", self["size"].value)
def parseID3(self):
yield ID3v2(self, "id3v2", size=self["size"].value*8)
def parseComment(self):
yield UInt16(self, "nb_comment")
for index in xrange(self["nb_comment"].value):
yield Comment(self, "comment[]")
def parseCommon(self):
yield UInt16(self, "nb_channel")
yield UInt32(self, "nb_sample")
yield UInt16(self, "sample_size")
yield Float80(self, "sample_rate")
yield Enum(String(self, "codec", 4, strip="\0", charset="ASCII"), CODEC_NAME)
def parseVersion(self):
yield TimestampMac32(self, "timestamp")
def parseSound(self):
yield UInt32(self, "offset")
yield UInt32(self, "block_size")
size = (self.size - self.current_size) // 8
if size:
yield RawBytes(self, "data", size)
class Chunk(FieldSet):
TAG_INFO = {
'COMM': ('common', "Common chunk", parseCommon),
'COMT': ('comment', "Comment", parseComment),
'NAME': ('name', "Name", parseText),
'AUTH': ('author', "Author", parseText),
'FVER': ('version', "Version", parseVersion),
'SSND': ('sound', "Sound data", parseSound),
'ID3 ': ('id3', "ID3", parseID3),
}
def __init__(self, *args):
FieldSet.__init__(self, *args)
self._size = (8 + alignValue(self["size"].value, 2)) * 8
tag = self["type"].value
if tag in self.TAG_INFO:
self._name, self._description, self._parser = self.TAG_INFO[tag]
else:
self._parser = None
def createFields(self):
yield String(self, "type", 4, "Signature (FORM)", charset="ASCII")
yield filesizeHandler(UInt32(self, "size"))
size = self["size"].value
if size:
if self._parser:
for field in self._parser(self):
yield field
if size % 2:
yield NullBytes(self, "padding", 1)
else:
yield RawBytes(self, "data", size)
class AiffFile(Parser):
PARSER_TAGS = {
"id": "aiff",
"category": "audio",
"file_ext": ("aif", "aiff", "aifc"),
"mime": (u"audio/x-aiff",),
"magic_regex": (("FORM.{4}AIF[CF]", 0),),
"min_size": 12*8,
"description": "Audio Interchange File Format (AIFF)"
}
endian = BIG_ENDIAN
def validate(self):
if self.stream.readBytes(0, 4) != "FORM":
return "Invalid signature"
if self.stream.readBytes(8*8, 4) not in ("AIFF", "AIFC"):
return "Invalid type"
return True
def createFields(self):
yield String(self, "signature", 4, "Signature (FORM)", charset="ASCII")
yield filesizeHandler(UInt32(self, "filesize"))
yield String(self, "type", 4, "Form type (AIFF or AIFC)", charset="ASCII")
while not self.eof:
yield Chunk(self, "chunk[]")
def createDescription(self):
if self["type"].value == "AIFC":
return "Audio Interchange File Format Compressed (AIFC)"
else:
return "Audio Interchange File Format (AIFF)"
def createContentSize(self):
return self["filesize"].value * 8
libs/hachoir_parser/audio/au.py
-88
View File
@@ -1,88 +0,0 @@
"""
AU audio file parser
Author: Victor Stinner
Creation: 12 july 2006
"""
from hachoir_parser import Parser
from hachoir_core.field import UInt32, Enum, String, RawBytes
from hachoir_core.endian import BIG_ENDIAN
from hachoir_core.text_handler import displayHandler, filesizeHandler
from hachoir_core.tools import createDict, humanFrequency
class AuFile(Parser):
PARSER_TAGS = {
"id": "sun_next_snd",
"category": "audio",
"file_ext": ("au", "snd"),
"mime": (u"audio/basic",),
"min_size": 24*8,
"magic": ((".snd", 0),),
"description": "Sun/NeXT audio"
}
endian = BIG_ENDIAN
CODEC_INFO = {
1: (8, u"8-bit ISDN u-law"),
2: (8, u"8-bit linear PCM"),
3: (16, u"16-bit linear PCM"),
4: (24, u"24-bit linear PCM"),
5: (32, u"32-bit linear PCM"),
6: (32, u"32-bit IEEE floating point"),
7: (64, u"64-bit IEEE floating point"),
8: (None, u"Fragmented sample data"),
9: (None, u"DSP program"),
10: (8, u"8-bit fixed point"),
11: (16, u"16-bit fixed point"),
12: (24, u"24-bit fixed point"),
13: (32, u"32-bit fixed point"),
18: (16, u"16-bit linear with emphasis"),
19: (16, u"16-bit linear compressed"),
20: (16, u"16-bit linear with emphasis and compression"),
21: (None, u"Music kit DSP commands"),
23: (None, u"4-bit ISDN u-law compressed (CCITT G.721 ADPCM)"),
24: (None, u"ITU-T G.722 ADPCM"),
25: (None, u"ITU-T G.723 3-bit ADPCM"),
26: (None, u"ITU-T G.723 5-bit ADPCM"),
27: (8, u"8-bit ISDN A-law"),
}
# Create bit rate and codec name dictionnaries
BITS_PER_SAMPLE = createDict(CODEC_INFO, 0)
CODEC_NAME = createDict(CODEC_INFO, 1)
VALID_NB_CHANNEL = set((1,2)) # FIXME: 4, 5, 7, 8 channels are supported?
def validate(self):
if self.stream.readBytes(0, 4) != ".snd":
return "Wrong file signature"
if self["channels"].value not in self.VALID_NB_CHANNEL:
return "Invalid number of channel"
return True
def getBitsPerSample(self):
"""
Get bit rate (number of bit per sample per channel),
may returns None if you unable to compute it.
"""
return self.BITS_PER_SAMPLE.get(self["codec"].value)
def createFields(self):
yield String(self, "signature", 4, 'Format signature (".snd")', charset="ASCII")
yield UInt32(self, "data_ofs", "Data offset")
yield filesizeHandler(UInt32(self, "data_size", "Data size"))
yield Enum(UInt32(self, "codec", "Audio codec"), self.CODEC_NAME)
yield displayHandler(UInt32(self, "sample_rate", "Number of samples/second"), humanFrequency)
yield UInt32(self, "channels", "Number of interleaved channels")
size = self["data_ofs"].value - self.current_size // 8
if 0 < size:
yield String(self, "info", size, "Information", strip=" \0", charset="ISO-8859-1")
size = min(self["data_size"].value, (self.size - self.current_size) // 8)
yield RawBytes(self, "audio_data", size, "Audio data")
def createContentSize(self):
return (self["data_ofs"].value + self["data_size"].value) * 8
libs/hachoir_parser/audio/flac.py
-157
View File
@@ -1,157 +0,0 @@
"""
FLAC (audio) parser
Documentation:
* http://flac.sourceforge.net/format.html
Author: Esteban Loiseau <baal AT tuxfamily.org>
Creation date: 2008-04-09
"""
from hachoir_parser import Parser
from hachoir_core.field import FieldSet, String, Bit, Bits, UInt16, UInt24, RawBytes, Enum, NullBytes
from hachoir_core.stream import BIG_ENDIAN, LITTLE_ENDIAN
from hachoir_core.tools import createDict
from hachoir_parser.container.ogg import parseVorbisComment
class VorbisComment(FieldSet):
endian = LITTLE_ENDIAN
createFields = parseVorbisComment
class StreamInfo(FieldSet):
static_size = 34*8
def createFields(self):
yield UInt16(self, "min_block_size", "The minimum block size (in samples) used in the stream")
yield UInt16(self, "max_block_size", "The maximum block size (in samples) used in the stream")
yield UInt24(self, "min_frame_size", "The minimum frame size (in bytes) used in the stream")
yield UInt24(self, "max_frame_size", "The maximum frame size (in bytes) used in the stream")
yield Bits(self, "sample_hertz", 20, "Sample rate in Hertz")
yield Bits(self, "nb_channel", 3, "Number of channels minus one")
yield Bits(self, "bits_per_sample", 5, "Bits per sample minus one")
yield Bits(self, "total_samples", 36, "Total samples in stream")
yield RawBytes(self, "md5sum", 16, "MD5 signature of the unencoded audio data")
class SeekPoint(FieldSet):
def createFields(self):
yield Bits(self, "sample_number", 64, "Sample number")
yield Bits(self, "offset", 64, "Offset in bytes")
yield Bits(self, "nb_sample", 16)
class SeekTable(FieldSet):
def createFields(self):
while not self.eof:
yield SeekPoint(self, "point[]")
class MetadataBlock(FieldSet):
"Metadata block field: http://flac.sourceforge.net/format.html#metadata_block"
BLOCK_TYPES = {
0: ("stream_info", u"Stream info", StreamInfo),
1: ("padding[]", u"Padding", None),
2: ("application[]", u"Application", None),
3: ("seek_table", u"Seek table", SeekTable),
4: ("comment", u"Vorbis comment", VorbisComment),
5: ("cue_sheet[]", u"Cue sheet", None),
6: ("picture[]", u"Picture", None),
}
BLOCK_TYPE_DESC = createDict(BLOCK_TYPES, 1)
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = 32 + self["metadata_length"].value * 8
try:
key = self["block_type"].value
self._name, self._description, self.handler = self.BLOCK_TYPES[key]
except KeyError:
self.handler = None
def createFields(self):
yield Bit(self, "last_metadata_block", "True if this is the last metadata block")
yield Enum(Bits(self, "block_type", 7, "Metadata block header type"), self.BLOCK_TYPE_DESC)
yield UInt24(self, "metadata_length", "Length of following metadata in bytes (doesn't include this header)")
block_type = self["block_type"].value
size = self["metadata_length"].value
if not size:
return
try:
handler = self.BLOCK_TYPES[block_type][2]
except KeyError:
handler = None
if handler:
yield handler(self, "content", size=size*8)
elif self["block_type"].value == 1:
yield NullBytes(self, "padding", size)
else:
yield RawBytes(self, "rawdata", size)
class Metadata(FieldSet):
def createFields(self):
while not self.eof:
field = MetadataBlock(self,"metadata_block[]")
yield field
if field["last_metadata_block"].value:
break
class Frame(FieldSet):
SAMPLE_RATES = {
0: "get from STREAMINFO metadata block",
1: "88.2kHz",
2: "176.4kHz",
3: "192kHz",
4: "8kHz",
5: "16kHz",
6: "22.05kHz",
7: "24kHz",
8: "32kHz",
9: "44.1kHz",
10: "48kHz",
11: "96kHz",
12: "get 8 bit sample rate (in kHz) from end of header",
13: "get 16 bit sample rate (in Hz) from end of header",
14: "get 16 bit sample rate (in tens of Hz) from end of header",
}
def createFields(self):
yield Bits(self, "sync", 14, "Sync code: 11111111111110")
yield Bit(self, "reserved[]")
yield Bit(self, "blocking_strategy")
yield Bits(self, "block_size", 4)
yield Enum(Bits(self, "sample_rate", 4), self.SAMPLE_RATES)
yield Bits(self, "channel_assign", 4)
yield Bits(self, "sample_size", 3)
yield Bit(self, "reserved[]")
# FIXME: Finish frame header parser
class Frames(FieldSet):
def createFields(self):
while not self.eof:
yield Frame(self, "frame[]")
# FIXME: Parse all frames
return
class FlacParser(Parser):
"Parse FLAC audio files: FLAC is a lossless audio codec"
MAGIC = "fLaC\x00"
PARSER_TAGS = {
"id": "flac",
"category": "audio",
"file_ext": ("flac",),
"mime": (u"audio/x-flac",),
"magic": ((MAGIC, 0),),
"min_size": 4*8,
"description": "FLAC audio",
}
endian = BIG_ENDIAN
def validate(self):
if self.stream.readBytes(0, len(self.MAGIC)) != self.MAGIC:
return u"Invalid magic string"
return True
def createFields(self):
yield String(self, "signature", 4,charset="ASCII", description="FLAC signature: fLaC string")
yield Metadata(self,"metadata")
yield Frames(self,"frames")
libs/hachoir_parser/audio/id3.py
-507
View File
@@ -1,507 +0,0 @@
"""
ID3 metadata parser, supported versions: 1.O, 2.2, 2.3 and 2.4
Informations: http://www.id3.org/
Author: Victor Stinner
"""
from hachoir_core.field import (FieldSet, MatchError, ParserError,
Enum, UInt8, UInt24, UInt32,
CString, String, RawBytes,
Bit, Bits, NullBytes, NullBits)
from hachoir_core.text_handler import textHandler
from hachoir_core.tools import humanDuration
from hachoir_core.endian import NETWORK_ENDIAN
class ID3v1(FieldSet):
static_size = 128 * 8
GENRE_NAME = {
0: u"Blues",
1: u"Classic Rock",
2: u"Country",
3: u"Dance",
4: u"Disco",
5: u"Funk",
6: u"Grunge",
7: u"Hip-Hop",
8: u"Jazz",
9: u"Metal",
10: u"New Age",
11: u"Oldies",
12: u"Other",
13: u"Pop",
14: u"R&B",
15: u"Rap",
16: u"Reggae",
17: u"Rock",
18: u"Techno",
19: u"Industrial",
20: u"Alternative",
21: u"Ska",
22: u"Death Metal",
23: u"Pranks",
24: u"Soundtrack",
25: u"Euro-Techno",
26: u"Ambient",
27: u"Trip-Hop",
28: u"Vocal",
29: u"Jazz+Funk",
30: u"Fusion",
31: u"Trance",
32: u"Classical",
33: u"Instrumental",
34: u"Acid",
35: u"House",
36: u"Game",
37: u"Sound Clip",
38: u"Gospel",
39: u"Noise",
40: u"AlternRock",
41: u"Bass",
42: u"Soul",
43: u"Punk",
44: u"Space",
45: u"Meditative",
46: u"Instrumental Pop",
47: u"Instrumental Rock",
48: u"Ethnic",
49: u"Gothic",
50: u"Darkwave",
51: u"Techno-Industrial",
52: u"Electronic",
53: u"Pop-Folk",
54: u"Eurodance",
55: u"Dream",
56: u"Southern Rock",
57: u"Comedy",
58: u"Cult",
59: u"Gangsta",
60: u"Top 40",
61: u"Christian Rap",
62: u"Pop/Funk",
63: u"Jungle",
64: u"Native American",
65: u"Cabaret",
66: u"New Wave",
67: u"Psychadelic",
68: u"Rave",
69: u"Showtunes",
70: u"Trailer",
71: u"Lo-Fi",
72: u"Tribal",
73: u"Acid Punk",
74: u"Acid Jazz",
75: u"Polka",
76: u"Retro",
77: u"Musical",
78: u"Rock & Roll",
79: u"Hard Rock",
# Following are winamp extentions
80: u"Folk",
81: u"Folk-Rock",
82: u"National Folk",
83: u"Swing",
84: u"Fast Fusion",
85: u"Bebob",
86: u"Latin",
87: u"Revival",
88: u"Celtic",
89: u"Bluegrass",
90: u"Avantgarde",
91: u"Gothic Rock",
92: u"Progressive Rock",
93: u"Psychedelic Rock",
94: u"Symphonic Rock",
95: u"Slow Rock",
96: u"Big Band",
97: u"Chorus",
98: u"Easy Listening",
99: u"Acoustic",
100: u"Humour",
101: u"Speech",
102: u"Chanson",
103: u"Opera",
104: u"Chamber Music",
105: u"Sonata",
106: u"Symphony",
107: u"Booty Bass",
108: u"Primus",
109: u"Porn Groove",
110: u"Satire",
111: u"Slow Jam",
112: u"Club",
113: u"Tango",
114: u"Samba",
115: u"Folklore",
116: u"Ballad",
117: u"Power Ballad",
118: u"Rhythmic Soul",
119: u"Freestyle",
120: u"Duet",
121: u"Punk Rock",
122: u"Drum Solo",
123: u"A capella",
124: u"Euro-House",
125: u"Dance Hall",
126: u"Goa",
127: u"Drum & Bass",
128: u"Club-House",
129: u"Hardcore",
130: u"Terror",
131: u"Indie",
132: u"Britpop",
133: u"Negerpunk",
134: u"Polsk Punk",
135: u"Beat",
136: u"Christian Gangsta Rap",
137: u"Heavy Metal",
138: u"Black Metal",
139: u"Crossover",
140: u"Contemporary Christian",
141: u"Christian Rock ",
142: u"Merengue",
143: u"Salsa",
144: u"Trash Metal",
145: u"Anime",
146: u"JPop",
147: u"Synthpop"
}
def createFields(self):
yield String(self, "signature", 3, "IDv1 signature (\"TAG\")", charset="ASCII")
if self["signature"].value != "TAG":
raise MatchError("Stream doesn't look like ID3v1 (wrong signature)!")
# TODO: Charset of below strings?
yield String(self, "song", 30, "Song title", strip=" \0", charset="ISO-8859-1")
yield String(self, "author", 30, "Author", strip=" \0", charset="ISO-8859-1")
yield String(self, "album", 30, "Album title", strip=" \0", charset="ISO-8859-1")
yield String(self, "year", 4, "Year", strip=" \0", charset="ISO-8859-1")
# TODO: Write better algorithm to guess ID3v1 version
version = self.getVersion()
if version in ("v1.1", "v1.1b"):
if version == "v1.1b":
# ID3 v1.1b
yield String(self, "comment", 29, "Comment", strip=" \0", charset="ISO-8859-1")
yield UInt8(self, "track_nb", "Track number")
else:
# ID3 v1.1
yield String(self, "comment", 30, "Comment", strip=" \0", charset="ISO-8859-1")
yield Enum(UInt8(self, "genre", "Genre"), self.GENRE_NAME)
else:
# ID3 v1.0
yield String(self, "comment", 31, "Comment", strip=" \0", charset="ISO-8859-1")
def getVersion(self):
addr = self.absolute_address + 126*8
bytes = self.stream.readBytes(addr, 2)
# last byte (127) is not space?
if bytes[1] != ' ':
# byte 126 is nul?
if bytes[0] == 0x00:
return "v1.1"
else:
return "v1.1b"
else:
return "1.0"
def createDescription(self):
version = self.getVersion()
return "ID3 %s: author=%s, song=%s" % (
version, self["author"].value, self["song"].value)
def getCharset(field):
try:
key = field.value
return ID3_StringCharset.charset_name[key]
except KeyError:
raise ParserError("ID3v2: Invalid charset (%s)." % key)
class ID3_String(FieldSet):
STRIP = " \0"
def createFields(self):
yield String(self, "text", self._size/8, "Text", charset="ISO-8859-1", strip=self.STRIP)
class ID3_StringCharset(ID3_String):
STRIP = " \0"
charset_desc = {
0: "ISO-8859-1",
1: "UTF-16 with BOM",
2: "UTF-16 (big endian)",
3: "UTF-8"
}
charset_name = {
0: "ISO-8859-1",
1: "UTF-16",
2: "UTF-16-BE",
3: "UTF-8"
}
def createFields(self):
yield Enum(UInt8(self, "charset"), self.charset_desc)
size = (self.size - self.current_size)/8
if not size:
return
charset = getCharset(self["charset"])
yield String(self, "text", size, "Text", charset=charset, strip=self.STRIP)
class ID3_GEOB(ID3_StringCharset):
def createFields(self):
yield Enum(UInt8(self, "charset"), self.charset_desc)
charset = getCharset(self["charset"])
yield CString(self, "mime", "MIME type", charset=charset)
yield CString(self, "filename", "File name", charset=charset)
yield CString(self, "description", "Content description", charset=charset)
size = (self.size - self.current_size) // 8
if not size:
return
yield String(self, "text", size, "Text", charset=charset)
class ID3_Comment(ID3_StringCharset):
def createFields(self):
yield Enum(UInt8(self, "charset"), self.charset_desc)
yield String(self, "lang", 3, "Language", charset="ASCII")
charset = getCharset(self["charset"])
yield CString(self, "title", "Title", charset=charset, strip=self.STRIP)
size = (self.size - self.current_size) // 8
if not size:
return
yield String(self, "text", size, "Text", charset=charset, strip=self.STRIP)
class ID3_StringTitle(ID3_StringCharset):
def createFields(self):
yield Enum(UInt8(self, "charset"), self.charset_desc)
if self.current_size == self.size:
return
charset = getCharset(self["charset"])
yield CString(self, "title", "Title", charset=charset, strip=self.STRIP)
size = (self.size - self.current_size)/8
if not size:
return
yield String(self, "text", size, "Text", charset=charset, strip=self.STRIP)
class ID3_Private(FieldSet):
def createFields(self):
size = self._size/8
# TODO: Strings charset?
if self.stream.readBytes(self.absolute_address, 9) == "PeakValue":
yield String(self, "text", 9, "Text")
size -= 9
yield String(self, "content", size, "Content")
class ID3_TrackLength(FieldSet):
def createFields(self):
yield NullBytes(self, "zero", 1)
yield textHandler(String(self, "length", self._size/8 - 1,
"Length in ms", charset="ASCII"), self.computeLength)
def computeLength(self, field):
try:
ms = int(field.value)
return humanDuration(ms)
except:
return field.value
class ID3_Picture23(FieldSet):
pict_type_name = {
0x00: "Other",
0x01: "32x32 pixels 'file icon' (PNG only)",
0x02: "Other file icon",
0x03: "Cover (front)",
0x04: "Cover (back)",
0x05: "Leaflet page",
0x06: "Media (e.g. lable side of CD)",
0x07: "Lead artist/lead performer/soloist",
0x08: "Artist/performer",
0x09: "Conductor",
0x0A: "Band/Orchestra",
0x0B: "Composer",
0x0C: "Lyricist/text writer",
0x0D: "Recording Location",
0x0E: "During recording",
0x0F: "During performance",
0x10: "Movie/video screen capture",
0x11: "A bright coloured fish",
0x12: "Illustration",
0x13: "Band/artist logotype",
0x14: "Publisher/Studio logotype"
}
def createFields(self):
yield Enum(UInt8(self, "charset"), ID3_StringCharset.charset_desc)
charset = getCharset(self["charset"])
yield String(self, "img_fmt", 3, charset="ASCII")
yield Enum(UInt8(self, "pict_type"), self.pict_type_name)
yield CString(self, "text", "Text", charset=charset, strip=" \0")
size = (self._size - self._current_size) / 8
if size:
yield RawBytes(self, "img_data", size)
class ID3_Picture24(FieldSet):
def createFields(self):
yield Enum(UInt8(self, "charset"), ID3_StringCharset.charset_desc)
charset = getCharset(self["charset"])
yield CString(self, "mime", "MIME type", charset=charset)
yield Enum(UInt8(self, "pict_type"), ID3_Picture23.pict_type_name)
yield CString(self, "description", charset=charset)
size = (self._size - self._current_size) / 8
if size:
yield RawBytes(self, "img_data", size)
class ID3_Chunk(FieldSet):
endian = NETWORK_ENDIAN
tag22_name = {
"TT2": "Track title",
"TP1": "Artist",
"TRK": "Track number",
"COM": "Comment",
"TCM": "Composer",
"TAL": "Album",
"TYE": "Year",
"TEN": "Encoder",
"TCO": "Content type",
"PIC": "Picture"
}
tag23_name = {
"COMM": "Comment",
"GEOB": "Encapsulated object",
"PRIV": "Private",
"TPE1": "Artist",
"TCOP": "Copyright",
"TALB": "Album",
"TENC": "Encoder",
"TYER": "Year",
"TSSE": "Encoder settings",
"TCOM": "Composer",
"TRCK": "Track number",
"PCNT": "Play counter",
"TCON": "Content type",
"TLEN": "Track length",
"TIT2": "Track title",
"WXXX": "User defined URL"
}
handler = {
"COMM": ID3_Comment,
"COM": ID3_Comment,
"GEOB": ID3_GEOB,
"PIC": ID3_Picture23,
"APIC": ID3_Picture24,
"PRIV": ID3_Private,
"TXXX": ID3_StringTitle,
"WOAR": ID3_String,
"WXXX": ID3_StringTitle,
}
def __init__(self, *args):
FieldSet.__init__(self, *args)
if 3 <= self["../ver_major"].value:
self._size = (10 + self["size"].value) * 8
else:
self._size = (self["size"].value + 6) * 8
def createFields(self):
if 3 <= self["../ver_major"].value:
# ID3 v2.3 and 2.4
yield Enum(String(self, "tag", 4, "Tag", charset="ASCII", strip="\0"), ID3_Chunk.tag23_name)
if 4 <= self["../ver_major"].value:
yield ID3_Size(self, "size") # ID3 v2.4
else:
yield UInt32(self, "size") # ID3 v2.3
yield Bit(self, "tag_alter", "Tag alter preservation")
yield Bit(self, "file_alter", "Tag alter preservation")
yield Bit(self, "rd_only", "Read only?")
yield NullBits(self, "padding[]", 5)
yield Bit(self, "compressed", "Frame is compressed?")
yield Bit(self, "encrypted", "Frame is encrypted?")
yield Bit(self, "group", "Grouping identity")
yield NullBits(self, "padding[]", 5)
size = self["size"].value
is_compressed = self["compressed"].value
else:
# ID3 v2.2
yield Enum(String(self, "tag", 3, "Tag", charset="ASCII", strip="\0"), ID3_Chunk.tag22_name)
yield UInt24(self, "size")
size = self["size"].value - self.current_size/8 + 6
is_compressed = False
if size:
cls = None
if not(is_compressed):
tag = self["tag"].value
if tag in ID3_Chunk.handler:
cls = ID3_Chunk.handler[tag]
elif tag[0] == "T":
cls = ID3_StringCharset
if cls:
yield cls(self, "content", "Content", size=size*8)
else:
yield RawBytes(self, "content", size, "Raw data content")
def createDescription(self):
if self["size"].value != 0:
return "ID3 Chunk: %s" % self["tag"].display
else:
return "ID3 Chunk: (terminator)"
class ID3_Size(Bits):
static_size = 32
def __init__(self, parent, name, description=None):
Bits.__init__(self, parent, name, 32, description)
def createValue(self):
data = self.parent.stream.readBytes(self.absolute_address, 4)
# TODO: Check that bit #7 of each byte is nul: not(ord(data[i]) & 127)
return reduce(lambda x, y: x*128 + y, (ord(item) for item in data ))
class ID3v2(FieldSet):
endian = NETWORK_ENDIAN
VALID_MAJOR_VERSIONS = (2, 3, 4)
def __init__(self, parent, name, size=None):
FieldSet.__init__(self, parent, name, size=size)
if not self._size:
self._size = (self["size"].value + 10) * 8
def createDescription(self):
return "ID3 v2.%s.%s" % \
(self["ver_major"].value, self["ver_minor"].value)
def createFields(self):
# Signature + version
yield String(self, "header", 3, "Header (ID3)", charset="ASCII")
yield UInt8(self, "ver_major", "Version (major)")
yield UInt8(self, "ver_minor", "Version (minor)")
# Check format
if self["header"].value != "ID3":
raise MatchError("Signature error, should be \"ID3\".")
if self["ver_major"].value not in self.VALID_MAJOR_VERSIONS \
or self["ver_minor"].value != 0:
raise MatchError(
"Unknown ID3 metadata version (2.%u.%u)"
% (self["ver_major"].value, self["ver_minor"].value))
# Flags
yield Bit(self, "unsync", "Unsynchronisation is used?")
yield Bit(self, "ext", "Extended header is used?")
yield Bit(self, "exp", "Experimental indicator")
yield NullBits(self, "padding[]", 5)
# Size
yield ID3_Size(self, "size")
# All tags
while self.current_size < self._size:
field = ID3_Chunk(self, "field[]")
yield field
if field["size"].value == 0:
break
# Search first byte of the MPEG file
padding = self.seekBit(self._size)
if padding:
yield padding
libs/hachoir_parser/audio/itunesdb.py
-433
View File
@@ -1,433 +0,0 @@
"""
iPod iTunesDB parser.
Documentation:
- http://ipodlinux.org/ITunesDB
Author: Romain HERAULT
Creation date: 19 august 2006
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet,
UInt8, UInt16, UInt32, UInt64, TimestampMac32,
String, Float32, NullBytes, Enum)
from hachoir_core.endian import LITTLE_ENDIAN
from hachoir_core.tools import humanDuration
from hachoir_core.text_handler import displayHandler, filesizeHandler
list_order={
1 : "playlist order (manual sort order)",
2 : "???",
3 : "songtitle",
4 : "album",
5 : "artist",
6 : "bitrate",
7 : "genre",
8 : "kind",
9 : "date modified",
10 : "track number",
11 : "size",
12 : "time",
13 : "year",
14 : "sample rate",
15 : "comment",
16 : "date added",
17 : "equalizer",
18 : "composer",
19 : "???",
20 : "play count",
21 : "last played",
22 : "disc number",
23 : "my rating",
24 : "release date",
25 : "BPM",
26 : "grouping",
27 : "category",
28 : "description",
29 : "show",
30 : "season",
31 : "episode number"
}
class DataObject(FieldSet):
type_name={
1:"Title",
2:"Location",
3:"Album",
4:"Artist",
5:"Genre",
6:"Filetype",
7:"EQ Setting",
8:"Comment",
9:"Category",
12:"Composer",
13:"Grouping",
14:"Description text",
15:"Podcast Enclosure URL",
16:"Podcast RSS URL",
17:"Chapter data",
18:"Subtitle",
19:"Show (for TV Shows only)",
20:"Episode",
21:"TV Network",
50:"Smart Playlist Data",
51:"Smart Playlist Rules",
52:"Library Playlist Index",
100:"Column info",
}
mhod52_sort_index_type_name={
3:"Title",
4:"Album, then Disk/Tracknumber, then Title",
5:"Artist, then Album, then Disc/Tracknumber, then Title",
7:"Genre, then Artist, then Album, then Disc/Tracknumber, then Title",
8:"Composer, then Title"
}
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = self["entry_length"].value *8
def createFields(self):
yield String(self, "header_id", 4, "Data Object Header Markup (\"mhod\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "entry_length", "Entry Length")
yield Enum(UInt32(self, "type", "type"),self.type_name)
if(self["type"].value<15):
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "position", "Position")
yield UInt32(self, "length", "String Length in bytes")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield String(self, "string", self["length"].value, "String Data", charset="UTF-16-LE")
elif (self["type"].value<17):
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield String(self, "string", self._size/8-self["header_length"].value, "String Data", charset="UTF-8")
elif (self["type"].value == 52):
yield UInt32(self, "unknown[]", "unk1")
yield UInt32(self, "unknown[]", "unk2")
yield Enum(UInt32(self, "sort_index_type", "Sort Index Type"),self.mhod52_sort_index_type_name)
yield UInt32(self, "entry_count", "Entry Count")
indexes_size = self["entry_count"].value*4
padding_offset = self["entry_length"].value - indexes_size
padding = self.seekByte(padding_offset, "header padding")
if padding:
yield padding
for i in xrange(self["entry_count"].value):
yield UInt32(self, "index["+str(i)+"]", "Index of the "+str(i)+"nth mhit")
else:
padding = self.seekByte(self["header_length"].value, "header padding")
if padding:
yield padding
padding = self.seekBit(self._size, "entry padding")
if padding:
yield padding
class TrackItem(FieldSet):
x1_type_name={
0:"AAC or CBR MP3",
1:"VBR MP3"
}
x2_type_name={
0:"AAC",
1:"MP3"
}
media_type_name={
0x00:"Audio/Video",
0x01:"Audio",
0x02:"Video",
0x04:"Podcast",
0x06:"Video Podcast",
0x08:"Audiobook",
0x20:"Music Video",
0x40:"TV Show",
0X60:"TV Show (Music lists)",
}
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = self["entry_length"].value *8
def createFields(self):
yield String(self, "header_id", 4, "Track Item Header Markup (\"mhit\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "entry_length", "Entry Length")
yield UInt32(self, "string_number", "Number of Strings")
yield UInt32(self, "unique_id", "Unique ID")
yield UInt32(self, "visible_tag", "Visible Tag")
yield String(self, "file_type", 4, "File Type")
yield Enum(UInt8(self, "x1_type", "Extended Type 1"),self.x1_type_name)
yield Enum(UInt8(self, "x2_type", "Extended type 2"),self.x2_type_name)
yield UInt8(self, "compilation_flag", "Compilation Flag")
yield UInt8(self, "rating", "Rating")
yield TimestampMac32(self, "added_date", "Date when the item was added")
yield filesizeHandler(UInt32(self, "size", "Track size in bytes"))
yield displayHandler(UInt32(self, "length", "Track length in milliseconds"), humanDuration)
yield UInt32(self, "track_number", "Number of this track")
yield UInt32(self, "total_track", "Total number of tracks")
yield UInt32(self, "year", "Year of the track")
yield UInt32(self, "bitrate", "Bitrate")
yield UInt32(self, "samplerate", "Sample Rate")
yield UInt32(self, "volume", "volume")
yield UInt32(self, "start_time", "Start playing at, in milliseconds")
yield UInt32(self, "stop_time", "Stop playing at, in milliseconds")
yield UInt32(self, "soundcheck", "SoundCheck preamp")
yield UInt32(self, "playcount_1", "Play count of the track")
yield UInt32(self, "playcount_2", "Play count of the track (identical to playcount_1)")
yield UInt32(self, "last_played_time", "Time the song was last played")
yield UInt32(self, "disc_number", "disc number in multi disc sets")
yield UInt32(self, "total_discs", "Total number of discs in the disc set")
yield UInt32(self, "userid", "User ID in the DRM scheme")
yield TimestampMac32(self, "last_modified", "Time of the last modification of the track")
yield UInt32(self, "bookmark_time", "Bookmark time for AudioBook")
yield UInt64(self, "dbid", "Unique DataBase ID for the song (identical in mhit and in mhii)")
yield UInt8(self, "checked", "song is checked")
yield UInt8(self, "application_rating", "Last Rating before change")
yield UInt16(self, "BPM", "BPM of the track")
yield UInt16(self, "artwork_count", "number of artworks fo this item")
yield UInt16(self, "unknown[]")
yield UInt32(self, "artwork_size", "Total size of artworks in bytes")
yield UInt32(self, "unknown[]")
yield Float32(self, "sample_rate_2", "Sample Rate express in float")
yield UInt32(self, "released_date", "Date of release in Music Store or in Podcast")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt8(self, "has_artwork", "0x01 for track with artwork, 0x02 otherwise")
yield UInt8(self, "skip_wen_shuffling", "Skip that track when shuffling")
yield UInt8(self, "remember_playback_position", "Remember playback position")
yield UInt8(self, "flag4", "Flag 4")
yield UInt64(self, "dbid2", "Unique DataBase ID for the song (identical as above)")
yield UInt8(self, "lyrics_flag", "Lyrics Flag")
yield UInt8(self, "movie_file_flag", "Movie File Flag")
yield UInt8(self, "played_mark", "Track has been played")
yield UInt8(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "sample_count", "Number of samples in the song (only for WAV and AAC files)")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield Enum(UInt32(self, "media_type", "Media Type for video iPod"),self.media_type_name)
yield UInt32(self, "season_number", "Season Number")
yield UInt32(self, "episode_number", "Episode Number")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
yield UInt32(self, "unknown[]")
padding = self.seekByte(self["header_length"].value, "header padding")
if padding:
yield padding
#while ((self.stream.readBytes(0, 4) == 'mhod') and ((self.current_size/8) < self["entry_length"].value)):
for i in xrange(self["string_number"].value):
yield DataObject(self, "data[]")
padding = self.seekBit(self._size, "entry padding")
if padding:
yield padding
class TrackList(FieldSet):
def createFields(self):
yield String(self, "header_id", 4, "Track List Header Markup (\"mhlt\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "track_number", "Number of Tracks")
padding = self.seekByte(self["header_length"].value, "header padding")
if padding:
yield padding
for i in xrange(self["track_number"].value):
yield TrackItem(self, "track[]")
class PlaylistItem(FieldSet):
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = self["entry_length"].value *8
def createFields(self):
yield String(self, "header_id", 4, "Playlist Item Header Markup (\"mhip\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "entry_length", "Entry Length")
yield UInt32(self, "data_object_child_count", "Number of Child Data Objects")
yield UInt32(self, "podcast_grouping_flag", "Podcast Grouping Flag")
yield UInt32(self, "group_id", "Group ID")
yield UInt32(self, "track_id", "Track ID")
yield TimestampMac32(self, "timestamp", "Song Timestamp")
yield UInt32(self, "podcast_grouping_ref", "Podcast Grouping Reference")
padding = self.seekByte(self["header_length"].value, "header padding")
if padding:
yield padding
for i in xrange(self["data_object_child_count"].value):
yield DataObject(self, "mhod[]")
class Playlist(FieldSet):
is_master_pl_name={
0:"Regular playlist",
1:"Master playlist"
}
is_podcast_name={
0:"Normal Playlist List",
1:"Podcast Playlist List"
}
list_sort_order_name={
1:"Manual Sort Order",
2:"???",
3:"Song Title",
4:"Album",
5:"Artist",
6:"Bitrate",
7:"Genre",
8:"Kind",
9:"Date Modified",
10:"Track Number",
11:"Size",
12:"Time",
13:"Year",
14:"Sample Rate",
15:"Comment",
16:"Date Added",
17:"Equalizer",
18:"Composer",
19:"???",
20:"Play Count",
21:"Last Played",
22:"Disc Number",
23:"My Rating",
24:"Release Date",
25:"BPM",
26:"Grouping",
27:"Category",
28:"Description",
29:"Show",
30:"Season",
31:"Episode Number"
}
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = self["entry_length"].value *8
def createFields(self):
yield String(self, "header_id", 4, "Playlist List Header Markup (\"mhyp\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "entry_length", "Entry Length")
yield UInt32(self, "data_object_child_count", "Number of Child Data Objects")
yield UInt32(self, "playlist_count", "Number of Playlist Items")
yield Enum(UInt8(self, "type", "Normal or master playlist?"), self.is_master_pl_name)
yield UInt8(self, "XXX1", "XXX1")
yield UInt8(self, "XXX2", "XXX2")
yield UInt8(self, "XXX3", "XXX3")
yield TimestampMac32(self, "creation_date", "Date when the playlist was created")
yield UInt64(self, "playlistid", "Persistent Playlist ID")
yield UInt32(self, "unk3", "unk3")
yield UInt16(self, "string_mhod_count", "Number of string MHODs for this playlist")
yield Enum(UInt16(self, "is_podcast", "Playlist or Podcast List?"), self.is_podcast_name)
yield Enum(UInt32(self, "sort_order", "Playlist Sort Order"), self.list_sort_order_name)
padding = self.seekByte(self["header_length"].value, "entry padding")
if padding:
yield padding
for i in xrange(self["data_object_child_count"].value):
yield DataObject(self, "mhod[]")
for i in xrange(self["playlist_count"].value):
yield PlaylistItem(self, "playlist_item[]")
class PlaylistList(FieldSet):
def createFields(self):
yield String(self, "header_id", 4, "Playlist List Header Markup (\"mhlp\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "playlist_number", "Number of Playlists")
padding = self.seekByte(self["header_length"].value, "header padding")
if padding:
yield padding
for i in xrange(self["playlist_number"].value):
yield Playlist(self, "playlist[]")
class DataSet(FieldSet):
type_name={
1:"Track List",
2:"Play List",
3:"Podcast List"
}
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = self["entry_length"].value *8
def createFields(self):
yield String(self, "header_id", 4, "DataSet Header Markup (\"mhsd\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "entry_length", "Entry Length")
yield Enum(UInt32(self, "type", "type"),self.type_name)
padding = self.seekByte(self["header_length"].value, "header_raw")
if padding:
yield padding
if self["type"].value == 1:
yield TrackList(self, "tracklist[]")
if self["type"].value == 2:
yield PlaylistList(self, "playlist_list[]");
if self["type"].value == 3:
yield PlaylistList(self, "podcast_list[]");
padding = self.seekBit(self._size, "entry padding")
if padding:
yield padding
class DataBase(FieldSet):
def __init__(self, *args, **kw):
FieldSet.__init__(self, *args, **kw)
self._size = self["entry_length"].value *8
# def createFields(self):
class ITunesDBFile(Parser):
PARSER_TAGS = {
"id": "itunesdb",
"category": "audio",
"min_size": 44*8,
"magic": (('mhbd',0),),
"description": "iPod iTunesDB file"
}
endian = LITTLE_ENDIAN
def validate(self):
return self.stream.readBytes(0, 4) == 'mhbd'
def createFields(self):
yield String(self, "header_id", 4, "DataBase Header Markup (\"mhbd\")", charset="ISO-8859-1")
yield UInt32(self, "header_length", "Header Length")
yield UInt32(self, "entry_length", "Entry Length")
yield UInt32(self, "unknown[]")
yield UInt32(self, "version_number", "Version Number")
yield UInt32(self, "child_number", "Number of Children")
yield UInt64(self, "id", "ID for this database")
yield UInt32(self, "unknown[]")
yield UInt64(self, "initial_dbid", "Initial DBID")
size = self["header_length"].value-self.current_size/ 8
if size>0:
yield NullBytes(self, "padding", size)
for i in xrange(self["child_number"].value):
yield DataSet(self, "dataset[]")
padding = self.seekByte(self["entry_length"].value, "entry padding")
if padding:
yield padding
def createContentSize(self):
return self["entry_length"].value * 8
libs/hachoir_parser/audio/midi.py
-246
View File
@@ -1,246 +0,0 @@
"""
Musical Instrument Digital Interface (MIDI) audio file parser.
Documentation:
- Standard MIDI File Format, Dustin Caldwell (downloaded on wotsit.org)
Author: Victor Stinner
Creation: 27 december 2006
"""
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet, Bits, ParserError,
String, UInt32, UInt24, UInt16, UInt8, Enum, RawBits, RawBytes)
from hachoir_core.endian import BIG_ENDIAN
from hachoir_core.text_handler import textHandler, hexadecimal
from hachoir_core.tools import createDict, humanDurationNanosec
from hachoir_parser.common.tracker import NOTE_NAME
MAX_FILESIZE = 10 * 1024 * 1024
class Integer(Bits):
def __init__(self, parent, name, description=None):
Bits.__init__(self, parent, name, 8, description)
stream = parent.stream
addr = self.absolute_address
value = 0
while True:
bits = stream.readBits(addr, 8, parent.endian)
value = (value << 7) + (bits & 127)
if not(bits & 128):
break
addr += 8
self._size += 8
if 32 < self._size:
raise ParserError("Integer size is bigger than 32-bit")
self.createValue = lambda: value
def parseNote(parser):
yield Enum(UInt8(parser, "note", "Note number"), NOTE_NAME)
yield UInt8(parser, "velocity")
def parseControl(parser):
yield UInt8(parser, "control", "Controller number")
yield UInt8(parser, "value", "New value")
def parsePatch(parser):
yield UInt8(parser, "program", "New program number")
def parseChannel(parser, size=1):
yield UInt8(parser, "channel", "Channel number")
def parsePitch(parser):
yield UInt8(parser, "bottom", "(least sig) 7 bits of value")
yield UInt8(parser, "top", "(most sig) 7 bits of value")
def parseText(parser, size):
yield String(parser, "text", size)
def parseSMPTEOffset(parser, size):
yield RawBits(parser, "padding", 1)
yield Enum(Bits(parser, "frame_rate", 2),
{0:"24 fps", 1:"25 fps", 2:"30 fps (drop frame)", 3:"30 fps"})
yield Bits(parser, "hour", 5)
yield UInt8(parser, "minute")
yield UInt8(parser, "second")
yield UInt8(parser, "frame")
yield UInt8(parser, "subframe", "100 subframes per frame")
def formatTempo(field):
return humanDurationNanosec(field.value*1000)
def parseTempo(parser, size):
yield textHandler(UInt24(parser, "microsec_quarter", "Microseconds per quarter note"), formatTempo)
def parseTimeSignature(parser, size):
yield UInt8(parser, "numerator", "Numerator of time signature")
yield UInt8(parser, "denominator", "denominator of time signature 2=quarter 3=eighth, etc.")
yield UInt8(parser, "nb_tick", "Number of ticks in metronome click")
yield UInt8(parser, "nb_32nd_note", "Number of 32nd notes to the quarter note")
class Command(FieldSet):
COMMAND = {}
for channel in xrange(16):
COMMAND[0x80+channel] = ("Note off (channel %u)" % channel, parseNote)
COMMAND[0x90+channel] = ("Note on (channel %u)" % channel, parseNote)
COMMAND[0xA0+channel] = ("Key after-touch (channel %u)" % channel, parseNote)
COMMAND[0xB0+channel] = ("Control change (channel %u)" % channel, parseControl)
COMMAND[0xC0+channel] = ("Program (patch) change (channel %u)" % channel, parsePatch)
COMMAND[0xD0+channel] = ("Channel after-touch (channel %u)" % channel, parseChannel)
COMMAND[0xE0+channel] = ("Pitch wheel change (channel %u)" % channel, parsePitch)
COMMAND_DESC = createDict(COMMAND, 0)
COMMAND_PARSER = createDict(COMMAND, 1)
META_COMMAND_TEXT = 1
META_COMMAND_NAME = 3
META_COMMAND = {
0x00: ("Sets the track's sequence number", None),
0x01: ("Text event", parseText),
0x02: ("Copyright info", parseText),
0x03: ("Sequence or Track name", parseText),
0x04: ("Track instrument name", parseText),
0x05: ("Lyric", parseText),
0x06: ("Marker", parseText),
0x07: ("Cue point", parseText),
0x20: ("MIDI Channel Prefix", parseChannel),
0x2F: ("End of the track", None),
0x51: ("Set tempo", parseTempo),
0x54: ("SMPTE offset", parseSMPTEOffset),
0x58: ("Time Signature", parseTimeSignature),
0x59: ("Key signature", None),
0x7F: ("Sequencer specific information", None),
}
META_COMMAND_DESC = createDict(META_COMMAND, 0)
META_COMMAND_PARSER = createDict(META_COMMAND, 1)
def __init__(self, *args, **kwargs):
if 'prev_command' in kwargs:
self.prev_command = kwargs['prev_command']
del kwargs['prev_command']
else:
self.prev_command = None
self.command = None
FieldSet.__init__(self, *args, **kwargs)
def createFields(self):
yield Integer(self, "time", "Delta time in ticks")
next = self.stream.readBits(self.absolute_address+self.current_size, 8, self.root.endian)
if next & 0x80 == 0:
# "Running Status" command
if self.prev_command is None:
raise ParserError("Running Status command not preceded by another command.")
self.command = self.prev_command.command
else:
yield Enum(textHandler(UInt8(self, "command"), hexadecimal), self.COMMAND_DESC)
self.command = self["command"].value
if self.command == 0xFF:
yield Enum(textHandler(UInt8(self, "meta_command"), hexadecimal), self.META_COMMAND_DESC)
yield UInt8(self, "data_len")
size = self["data_len"].value
if size:
command = self["meta_command"].value
if command in self.META_COMMAND_PARSER:
parser = self.META_COMMAND_PARSER[command]
else:
parser = None
if parser:
for field in parser(self, size):
yield field
else:
yield RawBytes(self, "data", size)
else:
if self.command not in self.COMMAND_PARSER:
raise ParserError("Unknown command: %s" % self["command"].display)
parser = self.COMMAND_PARSER[self.command]
for field in parser(self):
yield field
def createDescription(self):
if "meta_command" in self:
return self["meta_command"].display
else:
return self.COMMAND_DESC[self.command]
class Track(FieldSet):
def __init__(self, *args):
FieldSet.__init__(self, *args)
self._size = (8 + self["size"].value) * 8
def createFields(self):
yield String(self, "marker", 4, "Track marker (MTrk)", charset="ASCII")
yield UInt32(self, "size")
cur = None
if True:
while not self.eof:
cur = Command(self, "command[]", prev_command=cur)
yield cur
else:
size = self["size"].value
if size:
yield RawBytes(self, "raw", size)
def createDescription(self):
command = self["command[0]"]
if "meta_command" in command \
and command["meta_command"].value in (Command.META_COMMAND_TEXT, Command.META_COMMAND_NAME) \
and "text" in command:
return command["text"].value.strip("\r\n")
else:
return ""
class Header(FieldSet):
static_size = 10*8
FILE_FORMAT = {
0: "Single track",
1: "Multiple tracks, synchronous",
2: "Multiple tracks, asynchronous",
}
def createFields(self):
yield UInt32(self, "size")
yield Enum(UInt16(self, "file_format"), self.FILE_FORMAT)
yield UInt16(self, "nb_track")
yield UInt16(self, "delta_time", "Delta-time ticks per quarter note")
def createDescription(self):
return "%s; %s tracks" % (
self["file_format"].display, self["nb_track"].value)
class MidiFile(Parser):
MAGIC = "MThd"
PARSER_TAGS = {
"id": "midi",
"category": "audio",
"file_ext": ["mid", "midi"],
"mime": (u"audio/mime", ),
"magic": ((MAGIC, 0),),
"min_size": 64,
"description": "MIDI audio"
}
endian = BIG_ENDIAN
def validate(self):
if self.stream.readBytes(0, 4) != self.MAGIC:
return "Invalid signature"
if self["header/size"].value != 6:
return "Invalid header size"
return True
def createFields(self):
yield String(self, "signature", 4, r"MIDI signature (MThd)", charset="ASCII")
yield Header(self, "header")
while not self.eof:
yield Track(self, "track[]")
def createDescription(self):
return "MIDI audio: %s" % self["header"].description
def createContentSize(self):
count = self["/header/nb_track"].value - 1
start = self["track[%u]" % count].absolute_address
# Search "End of track" of last track
end = self.stream.searchBytes("\xff\x2f\x00", start, MAX_FILESIZE*8)
if end is not None:
return end + 3*8
return None
libs/hachoir_parser/audio/mod.py
-149
View File
@@ -1,149 +0,0 @@
"""
Parser of FastTrackerII Extended Module (XM) version 1.4
Documents:
- Modplug source code (file modplug/soundlib/Load_mod.cpp)
http://sourceforge.net/projects/modplug
- Dumb source code (files include/dumb.h and src/it/readmod.c
http://dumb.sf.net/
- Documents on "MOD" format on Wotsit
http://www.wotsit.org
Compressed formats (i.e. starting with "PP20" or having "PACK" as type
are not handled. Also NoiseTracker's NST modules aren't handled, although
it might be possible: no file format and 15 samples
Author: Christophe GISQUET <christophe.gisquet@free.fr>
Creation: 18th February 2007
"""
from math import log10
from hachoir_parser import Parser
from hachoir_core.field import (FieldSet,
Bits, UInt16, UInt8,
RawBytes, String, GenericVector)
from hachoir_core.endian import BIG_ENDIAN
from hachoir_core.text_handler import textHandler
# Old NoiseTracker 15-samples modules can have anything here.
MODULE_TYPE = {
"M.K.": ("Noise/Pro-Tracker", 4),
"M!K!": ("Noise/Pro-Tracker", 4),
"M&K&": ("Noise/Pro-Tracker", 4),
"RASP": ("StarTrekker", 4),
"FLT4": ("StarTrekker", 4),
"FLT8": ("StarTrekker", 8),
"6CHN": ("FastTracker", 6),
"8CHN": ("FastTracker", 8),
"CD81": ("Octalyser", 8),
"OCTA": ("Octalyser", 8),
"FA04": ("Digital Tracker", 4),
"FA06": ("Digital Tracker", 6),
"FA08": ("Digital Tracker", 8),
}
def getFineTune(val):
return ("0", "1", "2", "3", "4", "5", "6", "7", "8",
"-8", "-7", "-6", "-5", "-4", "-3", "-2", "-1")[val.value]
def getVolume(val):
return "%.1f dB" % (20.0*log10(val.value/64.0))
class SampleInfo(FieldSet):
static_size = 30*8
def createFields(self):
yield String(self, "name", 22, strip='\0')
yield UInt16(self, "sample_count")
yield textHandler(UInt8(self, "fine_tune"), getFineTune)
yield textHandler(UInt8(self, "volume"), getVolume)
yield UInt16(self, "loop_start", "Loop start offset in samples")
yield UInt16(self, "loop_len", "Loop length in samples")
def createValue(self):
return self["name"].value
class Header(FieldSet):
static_size = 1084*8
def createFields(self):
yield String(self, "name", 20, strip='\0')
yield GenericVector(self, "samples", 31, SampleInfo, "info")
yield UInt8(self, "length")
yield UInt8(self, "played_patterns_count")
yield GenericVector(self, "patterns", 128, UInt8, "position")
yield String(self, "type", 4)
def getNumChannels(self):
return MODULE_TYPE[self["type"].value][1]
class Note(FieldSet):
static_size = 8*4
def createFields(self):
yield Bits(self, 4, "note_hi_nibble")
yield Bits(self, 12, "period")
yield Bits(self, 4, "note_low_nibble")
yield Bits(self, 4, "effect")
yield UInt8(self, "parameter")
class Row(FieldSet):
def __init__(self, parent, name, channels, desc=None):
FieldSet.__init__(self, parent, name, description=desc)
self.channels = channels
self._size = 8*self.channels*4
def createFields(self):
for index in xrange(self.channels):
yield Note(self, "note[]")
class Pattern(FieldSet):
def __init__(self, parent, name, channels, desc=None):
FieldSet.__init__(self, parent, name, description=desc)
self.channels = channels
self._size = 64*8*self.channels*4
def createFields(self):
for index in xrange(64):
yield Row(self, "row[]", self.channels)
class AmigaModule(Parser):
PARSER_TAGS = {
"id": "mod",
"category": "audio",
"file_ext": ("mod", "nst", "wow", "oct", "sd0" ),
"mime": (u'audio/mod', u'audio/x-mod', u'audio/mod', u'audio/x-mod'),
"min_size": 1084*8,
"description": "Uncompressed amiga module"
}
endian = BIG_ENDIAN
def validate(self):
t = self.stream.readBytes(1080*8, 4)
if t not in MODULE_TYPE:
return "Invalid module type '%s'" % t
self.createValue = lambda t: "%s module, %u channels" % MODULE_TYPE[t]
return True
def createFields(self):
header = Header(self, "header")
yield header
channels = header.getNumChannels()
# Number of patterns
patterns = 0
for index in xrange(128):
patterns = max(patterns,
header["patterns/position[%u]" % index].value)
patterns += 1
# Yield patterns
for index in xrange(patterns):
yield Pattern(self, "pattern[]", channels)
# Yield samples
for index in xrange(31):
count = header["samples/info[%u]/sample_count" % index].value
if count:
self.info("Yielding sample %u: %u samples" % (index, count))
yield RawBytes(self, "sample_data[]", 2*count, \
"Sample %u" % index)

Some files were not shown because too many files have changed in this diff Show More