Update libs
This commit is contained in:
Ruud
@@ -8,23 +8,27 @@ implements a 'serialize' method taking a tree as sole argument and
|
||||
returning an iterator generating tokens.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
import sys
|
||||
|
||||
from ..utils import default_etree
|
||||
|
||||
treeWalkerCache = {}
|
||||
|
||||
|
||||
def getTreeWalker(treeType, implementation=None, **kwargs):
|
||||
"""Get a TreeWalker class for various types of tree with built-in support
|
||||
|
||||
treeType - the name of the tree type required (case-insensitive). Supported
|
||||
values are "simpletree", "dom", "etree" and "beautifulsoup"
|
||||
values are:
|
||||
|
||||
"simpletree" - a built-in DOM-ish tree type with support for some
|
||||
more pythonic idioms.
|
||||
"dom" - The xml.dom.minidom DOM implementation
|
||||
"pulldom" - The xml.dom.pulldom event stream
|
||||
"etree" - A generic walker for tree implementations exposing an
|
||||
elementtree-like interface (known to work with
|
||||
ElementTree, cElementTree and lxml.etree).
|
||||
"lxml" - Optimized walker for lxml.etree
|
||||
"beautifulsoup" - Beautiful soup (if installed)
|
||||
"genshi" - a Genshi stream
|
||||
|
||||
implementation - (Currently applies to the "etree" tree type only). A module
|
||||
@@ -33,20 +37,21 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
|
||||
|
||||
treeType = treeType.lower()
|
||||
if treeType not in treeWalkerCache:
|
||||
if treeType in ("dom", "pulldom", "simpletree"):
|
||||
mod = __import__(treeType, globals())
|
||||
if treeType in ("dom", "pulldom"):
|
||||
name = "%s.%s" % (__name__, treeType)
|
||||
__import__(name)
|
||||
mod = sys.modules[name]
|
||||
treeWalkerCache[treeType] = mod.TreeWalker
|
||||
elif treeType == "genshi":
|
||||
import genshistream
|
||||
from . import genshistream
|
||||
treeWalkerCache[treeType] = genshistream.TreeWalker
|
||||
elif treeType == "beautifulsoup":
|
||||
import soup
|
||||
treeWalkerCache[treeType] = soup.TreeWalker
|
||||
elif treeType == "lxml":
|
||||
import lxmletree
|
||||
from . import lxmletree
|
||||
treeWalkerCache[treeType] = lxmletree.TreeWalker
|
||||
elif treeType == "etree":
|
||||
import etree
|
||||
from . import etree
|
||||
if implementation is None:
|
||||
implementation = default_etree
|
||||
# XXX: NEVER cache here, caching is done in the etree submodule
|
||||
return etree.getETreeModule(implementation, **kwargs).TreeWalker
|
||||
return treeWalkerCache.get(treeType)
|
||||
|
||||
@@ -1,94 +1,9 @@
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
from six import text_type, string_types
|
||||
|
||||
import gettext
|
||||
_ = gettext.gettext
|
||||
|
||||
from html5lib.constants import voidElements, spaceCharacters
|
||||
spaceCharacters = u"".join(spaceCharacters)
|
||||
|
||||
class TreeWalker(object):
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
|
||||
def __iter__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def error(self, msg):
|
||||
return {"type": "SerializeError", "data": msg}
|
||||
|
||||
def normalizeAttrs(self, attrs):
|
||||
newattrs = {}
|
||||
if attrs:
|
||||
#TODO: treewalkers should always have attrs
|
||||
for (namespace,name),value in attrs.iteritems():
|
||||
namespace = unicode(namespace) if namespace else None
|
||||
name = unicode(name)
|
||||
value = unicode(value)
|
||||
newattrs[(namespace,name)] = value
|
||||
return newattrs
|
||||
|
||||
def emptyTag(self, namespace, name, attrs, hasChildren=False):
|
||||
yield {"type": "EmptyTag", "name": unicode(name),
|
||||
"namespace":unicode(namespace),
|
||||
"data": self.normalizeAttrs(attrs)}
|
||||
if hasChildren:
|
||||
yield self.error(_("Void element has children"))
|
||||
|
||||
def startTag(self, namespace, name, attrs):
|
||||
return {"type": "StartTag",
|
||||
"name": unicode(name),
|
||||
"namespace":unicode(namespace),
|
||||
"data": self.normalizeAttrs(attrs)}
|
||||
|
||||
def endTag(self, namespace, name):
|
||||
return {"type": "EndTag",
|
||||
"name": unicode(name),
|
||||
"namespace":unicode(namespace),
|
||||
"data": {}}
|
||||
|
||||
def text(self, data):
|
||||
data = unicode(data)
|
||||
middle = data.lstrip(spaceCharacters)
|
||||
left = data[:len(data)-len(middle)]
|
||||
if left:
|
||||
yield {"type": "SpaceCharacters", "data": left}
|
||||
data = middle
|
||||
middle = data.rstrip(spaceCharacters)
|
||||
right = data[len(middle):]
|
||||
if middle:
|
||||
yield {"type": "Characters", "data": middle}
|
||||
if right:
|
||||
yield {"type": "SpaceCharacters", "data": right}
|
||||
|
||||
def comment(self, data):
|
||||
return {"type": "Comment", "data": unicode(data)}
|
||||
|
||||
def doctype(self, name, publicId=None, systemId=None, correct=True):
|
||||
return {"type": "Doctype",
|
||||
"name": name is not None and unicode(name) or u"",
|
||||
"publicId": publicId,
|
||||
"systemId": systemId,
|
||||
"correct": correct}
|
||||
|
||||
def entity(self, name):
|
||||
return {"type": "Entity", "name": unicode(name)}
|
||||
|
||||
def unknown(self, nodeType):
|
||||
return self.error(_("Unknown node type: ") + nodeType)
|
||||
|
||||
class RecursiveTreeWalker(TreeWalker):
|
||||
def walkChildren(self, node):
|
||||
raise NodeImplementedError
|
||||
|
||||
def element(self, node, namespace, name, attrs, hasChildren):
|
||||
if name in voidElements:
|
||||
for token in self.emptyTag(namespace, name, attrs, hasChildren):
|
||||
yield token
|
||||
else:
|
||||
yield self.startTag(name, attrs)
|
||||
if hasChildren:
|
||||
for token in self.walkChildren(node):
|
||||
yield token
|
||||
yield self.endTag(name)
|
||||
|
||||
from xml.dom import Node
|
||||
|
||||
DOCUMENT = Node.DOCUMENT_NODE
|
||||
@@ -99,16 +14,127 @@ COMMENT = Node.COMMENT_NODE
|
||||
ENTITY = Node.ENTITY_NODE
|
||||
UNKNOWN = "<#UNKNOWN#>"
|
||||
|
||||
from ..constants import voidElements, spaceCharacters
|
||||
spaceCharacters = "".join(spaceCharacters)
|
||||
|
||||
|
||||
def to_text(s, blank_if_none=True):
|
||||
"""Wrapper around six.text_type to convert None to empty string"""
|
||||
if s is None:
|
||||
if blank_if_none:
|
||||
return ""
|
||||
else:
|
||||
return None
|
||||
elif isinstance(s, text_type):
|
||||
return s
|
||||
else:
|
||||
return text_type(s)
|
||||
|
||||
|
||||
def is_text_or_none(string):
|
||||
"""Wrapper around isinstance(string_types) or is None"""
|
||||
return string is None or isinstance(string, string_types)
|
||||
|
||||
|
||||
class TreeWalker(object):
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
|
||||
def __iter__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def error(self, msg):
|
||||
return {"type": "SerializeError", "data": msg}
|
||||
|
||||
def emptyTag(self, namespace, name, attrs, hasChildren=False):
|
||||
assert namespace is None or isinstance(namespace, string_types), type(namespace)
|
||||
assert isinstance(name, string_types), type(name)
|
||||
assert all((namespace is None or isinstance(namespace, string_types)) and
|
||||
isinstance(name, string_types) and
|
||||
isinstance(value, string_types)
|
||||
for (namespace, name), value in attrs.items())
|
||||
|
||||
yield {"type": "EmptyTag", "name": to_text(name, False),
|
||||
"namespace": to_text(namespace),
|
||||
"data": attrs}
|
||||
if hasChildren:
|
||||
yield self.error(_("Void element has children"))
|
||||
|
||||
def startTag(self, namespace, name, attrs):
|
||||
assert namespace is None or isinstance(namespace, string_types), type(namespace)
|
||||
assert isinstance(name, string_types), type(name)
|
||||
assert all((namespace is None or isinstance(namespace, string_types)) and
|
||||
isinstance(name, string_types) and
|
||||
isinstance(value, string_types)
|
||||
for (namespace, name), value in attrs.items())
|
||||
|
||||
return {"type": "StartTag",
|
||||
"name": text_type(name),
|
||||
"namespace": to_text(namespace),
|
||||
"data": dict(((to_text(namespace, False), to_text(name)),
|
||||
to_text(value, False))
|
||||
for (namespace, name), value in attrs.items())}
|
||||
|
||||
def endTag(self, namespace, name):
|
||||
assert namespace is None or isinstance(namespace, string_types), type(namespace)
|
||||
assert isinstance(name, string_types), type(namespace)
|
||||
|
||||
return {"type": "EndTag",
|
||||
"name": to_text(name, False),
|
||||
"namespace": to_text(namespace),
|
||||
"data": {}}
|
||||
|
||||
def text(self, data):
|
||||
assert isinstance(data, string_types), type(data)
|
||||
|
||||
data = to_text(data)
|
||||
middle = data.lstrip(spaceCharacters)
|
||||
left = data[:len(data) - len(middle)]
|
||||
if left:
|
||||
yield {"type": "SpaceCharacters", "data": left}
|
||||
data = middle
|
||||
middle = data.rstrip(spaceCharacters)
|
||||
right = data[len(middle):]
|
||||
if middle:
|
||||
yield {"type": "Characters", "data": middle}
|
||||
if right:
|
||||
yield {"type": "SpaceCharacters", "data": right}
|
||||
|
||||
def comment(self, data):
|
||||
assert isinstance(data, string_types), type(data)
|
||||
|
||||
return {"type": "Comment", "data": text_type(data)}
|
||||
|
||||
def doctype(self, name, publicId=None, systemId=None, correct=True):
|
||||
assert is_text_or_none(name), type(name)
|
||||
assert is_text_or_none(publicId), type(publicId)
|
||||
assert is_text_or_none(systemId), type(systemId)
|
||||
|
||||
return {"type": "Doctype",
|
||||
"name": to_text(name),
|
||||
"publicId": to_text(publicId),
|
||||
"systemId": to_text(systemId),
|
||||
"correct": to_text(correct)}
|
||||
|
||||
def entity(self, name):
|
||||
assert isinstance(name, string_types), type(name)
|
||||
|
||||
return {"type": "Entity", "name": text_type(name)}
|
||||
|
||||
def unknown(self, nodeType):
|
||||
return self.error(_("Unknown node type: ") + nodeType)
|
||||
|
||||
|
||||
class NonRecursiveTreeWalker(TreeWalker):
|
||||
def getNodeDetails(self, node):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def getFirstChild(self, node):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def getNextSibling(self, node):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def getParentNode(self, node):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -118,7 +144,6 @@ class NonRecursiveTreeWalker(TreeWalker):
|
||||
details = self.getNodeDetails(currentNode)
|
||||
type, details = details[0], details[1:]
|
||||
hasChildren = False
|
||||
endTag = None
|
||||
|
||||
if type == DOCTYPE:
|
||||
yield self.doctype(*details)
|
||||
@@ -130,12 +155,11 @@ class NonRecursiveTreeWalker(TreeWalker):
|
||||
elif type == ELEMENT:
|
||||
namespace, name, attributes, hasChildren = details
|
||||
if name in voidElements:
|
||||
for token in self.emptyTag(namespace, name, attributes,
|
||||
for token in self.emptyTag(namespace, name, attributes,
|
||||
hasChildren):
|
||||
yield token
|
||||
hasChildren = False
|
||||
else:
|
||||
endTag = name
|
||||
yield self.startTag(namespace, name, attributes)
|
||||
|
||||
elif type == COMMENT:
|
||||
@@ -149,12 +173,12 @@ class NonRecursiveTreeWalker(TreeWalker):
|
||||
|
||||
else:
|
||||
yield self.unknown(details[0])
|
||||
|
||||
|
||||
if hasChildren:
|
||||
firstChild = self.getFirstChild(currentNode)
|
||||
else:
|
||||
firstChild = None
|
||||
|
||||
|
||||
if firstChild is not None:
|
||||
currentNode = firstChild
|
||||
else:
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from xml.dom import Node
|
||||
|
||||
import gettext
|
||||
_ = gettext.gettext
|
||||
|
||||
import _base
|
||||
from html5lib.constants import voidElements
|
||||
from . import _base
|
||||
|
||||
|
||||
class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
def getNodeDetails(self, node):
|
||||
@@ -16,10 +18,13 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
|
||||
elif node.nodeType == Node.ELEMENT_NODE:
|
||||
attrs = {}
|
||||
for attr in node.attributes.keys():
|
||||
for attr in list(node.attributes.keys()):
|
||||
attr = node.getAttributeNode(attr)
|
||||
attrs[(attr.namespaceURI,attr.localName)] = attr.value
|
||||
return (_base.ELEMENT, node.namespaceURI, node.nodeName,
|
||||
if attr.namespaceURI:
|
||||
attrs[(attr.namespaceURI, attr.localName)] = attr.value
|
||||
else:
|
||||
attrs[(None, attr.name)] = attr.value
|
||||
return (_base.ELEMENT, node.namespaceURI, node.nodeName,
|
||||
attrs, node.hasChildNodes())
|
||||
|
||||
elif node.nodeType == Node.COMMENT_NODE:
|
||||
|
||||
@@ -1,33 +1,28 @@
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
try:
|
||||
from collections import OrderedDict
|
||||
except ImportError:
|
||||
try:
|
||||
from ordereddict import OrderedDict
|
||||
except ImportError:
|
||||
OrderedDict = dict
|
||||
import gettext
|
||||
_ = gettext.gettext
|
||||
|
||||
try:
|
||||
from types import ModuleType
|
||||
except:
|
||||
from new import module as ModuleType
|
||||
import copy
|
||||
import re
|
||||
|
||||
import _base
|
||||
from html5lib.constants import voidElements
|
||||
from six import text_type
|
||||
|
||||
from . import _base
|
||||
from ..utils import moduleFactoryFactory
|
||||
|
||||
tag_regexp = re.compile("{([^}]*)}(.*)")
|
||||
|
||||
moduleCache = {}
|
||||
|
||||
def getETreeModule(ElementTreeImplementation):
|
||||
name = "_" + ElementTreeImplementation.__name__+"builder"
|
||||
if name in moduleCache:
|
||||
return moduleCache[name]
|
||||
else:
|
||||
mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder")
|
||||
objs = getETreeBuilder(ElementTreeImplementation)
|
||||
mod.__dict__.update(objs)
|
||||
moduleCache[name] = mod
|
||||
return mod
|
||||
|
||||
def getETreeBuilder(ElementTreeImplementation):
|
||||
ElementTree = ElementTreeImplementation
|
||||
ElementTreeCommentType = ElementTree.Comment("asd").tag
|
||||
|
||||
class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
"""Given the particular ElementTree representation, this implementation,
|
||||
@@ -35,16 +30,16 @@ def getETreeBuilder(ElementTreeImplementation):
|
||||
content:
|
||||
|
||||
1. The current element
|
||||
|
||||
|
||||
2. The index of the element relative to its parent
|
||||
|
||||
|
||||
3. A stack of ancestor elements
|
||||
|
||||
|
||||
4. A flag "text", "tail" or None to indicate if the current node is a
|
||||
text node; either the text or tail of the current element (1)
|
||||
"""
|
||||
def getNodeDetails(self, node):
|
||||
if isinstance(node, tuple): # It might be the root Element
|
||||
if isinstance(node, tuple): # It might be the root Element
|
||||
elt, key, parents, flag = node
|
||||
if flag in ("text", "tail"):
|
||||
return _base.TEXT, getattr(elt, flag)
|
||||
@@ -54,41 +49,41 @@ def getETreeBuilder(ElementTreeImplementation):
|
||||
if not(hasattr(node, "tag")):
|
||||
node = node.getroot()
|
||||
|
||||
if node.tag in ("<DOCUMENT_ROOT>", "<DOCUMENT_FRAGMENT>"):
|
||||
if node.tag in ("DOCUMENT_ROOT", "DOCUMENT_FRAGMENT"):
|
||||
return (_base.DOCUMENT,)
|
||||
|
||||
elif node.tag == "<!DOCTYPE>":
|
||||
return (_base.DOCTYPE, node.text,
|
||||
return (_base.DOCTYPE, node.text,
|
||||
node.get("publicId"), node.get("systemId"))
|
||||
|
||||
elif node.tag == ElementTree.Comment:
|
||||
elif node.tag == ElementTreeCommentType:
|
||||
return _base.COMMENT, node.text
|
||||
|
||||
else:
|
||||
assert type(node.tag) in (str, unicode), type(node.tag)
|
||||
#This is assumed to be an ordinary element
|
||||
assert type(node.tag) == text_type, type(node.tag)
|
||||
# This is assumed to be an ordinary element
|
||||
match = tag_regexp.match(node.tag)
|
||||
if match:
|
||||
namespace, tag = match.groups()
|
||||
else:
|
||||
namespace = None
|
||||
tag = node.tag
|
||||
attrs = {}
|
||||
for name, value in node.attrib.items():
|
||||
attrs = OrderedDict()
|
||||
for name, value in list(node.attrib.items()):
|
||||
match = tag_regexp.match(name)
|
||||
if match:
|
||||
attrs[(match.group(1),match.group(2))] = value
|
||||
attrs[(match.group(1), match.group(2))] = value
|
||||
else:
|
||||
attrs[(None,name)] = value
|
||||
return (_base.ELEMENT, namespace, tag,
|
||||
attrs[(None, name)] = value
|
||||
return (_base.ELEMENT, namespace, tag,
|
||||
attrs, len(node) or node.text)
|
||||
|
||||
|
||||
def getFirstChild(self, node):
|
||||
if isinstance(node, tuple):
|
||||
element, key, parents, flag = node
|
||||
else:
|
||||
element, key, parents, flag = node, None, [], None
|
||||
|
||||
|
||||
if flag in ("text", "tail"):
|
||||
return None
|
||||
else:
|
||||
@@ -99,13 +94,13 @@ def getETreeBuilder(ElementTreeImplementation):
|
||||
return element[0], 0, parents, None
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def getNextSibling(self, node):
|
||||
if isinstance(node, tuple):
|
||||
element, key, parents, flag = node
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
if flag == "text":
|
||||
if len(element):
|
||||
parents.append(element)
|
||||
@@ -116,16 +111,16 @@ def getETreeBuilder(ElementTreeImplementation):
|
||||
if element.tail and flag != "tail":
|
||||
return element, key, parents, "tail"
|
||||
elif key < len(parents[-1]) - 1:
|
||||
return parents[-1][key+1], key+1, parents, None
|
||||
return parents[-1][key + 1], key + 1, parents, None
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def getParentNode(self, node):
|
||||
if isinstance(node, tuple):
|
||||
element, key, parents, flag = node
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
if flag == "text":
|
||||
if not parents:
|
||||
return element
|
||||
@@ -139,3 +134,5 @@ def getETreeBuilder(ElementTreeImplementation):
|
||||
return parent, list(parents[-1]).index(parent), parents, None
|
||||
|
||||
return locals()
|
||||
|
||||
getETreeModule = moduleFactoryFactory(getETreeBuilder)
|
||||
|
||||
@@ -1,50 +1,49 @@
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from genshi.core import QName
|
||||
from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT
|
||||
from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
|
||||
from genshi.output import NamespaceFlattener
|
||||
from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
|
||||
|
||||
import _base
|
||||
from . import _base
|
||||
|
||||
from ..constants import voidElements, namespaces
|
||||
|
||||
from html5lib.constants import voidElements
|
||||
|
||||
class TreeWalker(_base.TreeWalker):
|
||||
def __iter__(self):
|
||||
depth = 0
|
||||
ignore_until = None
|
||||
# Buffer the events so we can pass in the following one
|
||||
previous = None
|
||||
for event in self.tree:
|
||||
if previous is not None:
|
||||
if previous[0] == START:
|
||||
depth += 1
|
||||
if ignore_until <= depth:
|
||||
ignore_until = None
|
||||
if ignore_until is None:
|
||||
for token in self.tokens(previous, event):
|
||||
yield token
|
||||
if token["type"] == "EmptyTag":
|
||||
ignore_until = depth
|
||||
if previous[0] == END:
|
||||
depth -= 1
|
||||
previous = event
|
||||
if previous is not None:
|
||||
if ignore_until is None or ignore_until <= depth:
|
||||
for token in self.tokens(previous, None):
|
||||
for token in self.tokens(previous, event):
|
||||
yield token
|
||||
elif ignore_until is not None:
|
||||
raise ValueError("Illformed DOM event stream: void element without END_ELEMENT")
|
||||
previous = event
|
||||
|
||||
# Don't forget the final event!
|
||||
if previous is not None:
|
||||
for token in self.tokens(previous, None):
|
||||
yield token
|
||||
|
||||
def tokens(self, event, next):
|
||||
kind, data, pos = event
|
||||
if kind == START:
|
||||
tag, attrib = data
|
||||
tag, attribs = data
|
||||
name = tag.localname
|
||||
namespace = tag.namespace
|
||||
if tag in voidElements:
|
||||
for token in self.emptyTag(namespace, name, list(attrib),
|
||||
not next or next[0] != END
|
||||
converted_attribs = {}
|
||||
for k, v in attribs:
|
||||
if isinstance(k, QName):
|
||||
converted_attribs[(k.namespace, k.localname)] = v
|
||||
else:
|
||||
converted_attribs[(None, k)] = v
|
||||
|
||||
if namespace == namespaces["html"] and name in voidElements:
|
||||
for token in self.emptyTag(namespace, name, converted_attribs,
|
||||
not next or next[0] != END
|
||||
or next[1] != tag):
|
||||
yield token
|
||||
else:
|
||||
yield self.startTag(namespace, name, list(attrib))
|
||||
yield self.startTag(namespace, name, converted_attribs)
|
||||
|
||||
elif kind == END:
|
||||
name = data.localname
|
||||
@@ -62,8 +61,8 @@ class TreeWalker(_base.TreeWalker):
|
||||
elif kind == DOCTYPE:
|
||||
yield self.doctype(*data)
|
||||
|
||||
elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \
|
||||
START_CDATA, END_CDATA, PI):
|
||||
elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS,
|
||||
START_CDATA, END_CDATA, PI):
|
||||
pass
|
||||
|
||||
else:
|
||||
|
||||
@@ -1,186 +1,208 @@
|
||||
from lxml import etree
|
||||
from html5lib.treebuilders.etree import tag_regexp
|
||||
|
||||
from gettext import gettext
|
||||
_ = gettext
|
||||
|
||||
import _base
|
||||
|
||||
from html5lib.constants import voidElements
|
||||
from html5lib import ihatexml
|
||||
|
||||
class Root(object):
|
||||
def __init__(self, et):
|
||||
self.elementtree = et
|
||||
self.children = []
|
||||
if et.docinfo.internalDTD:
|
||||
self.children.append(Doctype(self, et.docinfo.root_name,
|
||||
et.docinfo.public_id,
|
||||
et.docinfo.system_url))
|
||||
root = et.getroot()
|
||||
node = root
|
||||
|
||||
while node.getprevious() is not None:
|
||||
node = node.getprevious()
|
||||
while node is not None:
|
||||
self.children.append(node)
|
||||
node = node.getnext()
|
||||
|
||||
self.text = None
|
||||
self.tail = None
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.children[key]
|
||||
|
||||
def getnext(self):
|
||||
return None
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
class Doctype(object):
|
||||
def __init__(self, root_node, name, public_id, system_id):
|
||||
self.root_node = root_node
|
||||
self.name = name
|
||||
self.public_id = public_id
|
||||
self.system_id = system_id
|
||||
|
||||
self.text = None
|
||||
self.tail = None
|
||||
|
||||
def getnext(self):
|
||||
return self.root_node.children[1]
|
||||
|
||||
class FragmentRoot(Root):
|
||||
def __init__(self, children):
|
||||
self.children = [FragmentWrapper(self, child) for child in children]
|
||||
self.text = self.tail = None
|
||||
|
||||
def getnext(self):
|
||||
return None
|
||||
|
||||
class FragmentWrapper(object):
|
||||
def __init__(self, fragment_root, obj):
|
||||
self.root_node = fragment_root
|
||||
self.obj = obj
|
||||
if hasattr(self.obj, 'text'):
|
||||
self.text = self.obj.text
|
||||
else:
|
||||
self.text = None
|
||||
if hasattr(self.obj, 'tail'):
|
||||
self.tail = self.obj.tail
|
||||
else:
|
||||
self.tail = None
|
||||
self.isstring = isinstance(obj, basestring)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.obj, name)
|
||||
|
||||
def getnext(self):
|
||||
siblings = self.root_node.children
|
||||
idx = siblings.index(self)
|
||||
if idx < len(siblings) - 1:
|
||||
return siblings[idx + 1]
|
||||
else:
|
||||
return None
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.obj[key]
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.obj)
|
||||
|
||||
def getparent(self):
|
||||
return None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.obj)
|
||||
|
||||
def __unicode__(self):
|
||||
return unicode(self.obj)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.obj)
|
||||
|
||||
|
||||
class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
def __init__(self, tree):
|
||||
if hasattr(tree, "getroot"):
|
||||
tree = Root(tree)
|
||||
elif isinstance(tree, list):
|
||||
tree = FragmentRoot(tree)
|
||||
_base.NonRecursiveTreeWalker.__init__(self, tree)
|
||||
self.filter = ihatexml.InfosetFilter()
|
||||
def getNodeDetails(self, node):
|
||||
if isinstance(node, tuple): # Text node
|
||||
node, key = node
|
||||
assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
|
||||
return _base.TEXT, getattr(node, key)
|
||||
|
||||
elif isinstance(node, Root):
|
||||
return (_base.DOCUMENT,)
|
||||
|
||||
elif isinstance(node, Doctype):
|
||||
return _base.DOCTYPE, node.name, node.public_id, node.system_id
|
||||
|
||||
elif isinstance(node, FragmentWrapper) and node.isstring:
|
||||
return _base.TEXT, node
|
||||
|
||||
elif node.tag == etree.Comment:
|
||||
return _base.COMMENT, node.text
|
||||
|
||||
elif node.tag == etree.Entity:
|
||||
return _base.ENTITY, node.text[1:-1] # strip &;
|
||||
|
||||
else:
|
||||
#This is assumed to be an ordinary element
|
||||
match = tag_regexp.match(node.tag)
|
||||
if match:
|
||||
namespace, tag = match.groups()
|
||||
else:
|
||||
namespace = None
|
||||
tag = node.tag
|
||||
attrs = {}
|
||||
for name, value in node.attrib.items():
|
||||
match = tag_regexp.match(name)
|
||||
if match:
|
||||
attrs[(match.group(1),match.group(2))] = value
|
||||
else:
|
||||
attrs[(None,name)] = value
|
||||
return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag),
|
||||
attrs, len(node) > 0 or node.text)
|
||||
|
||||
def getFirstChild(self, node):
|
||||
assert not isinstance(node, tuple), _("Text nodes have no children")
|
||||
|
||||
assert len(node) or node.text, "Node has no children"
|
||||
if node.text:
|
||||
return (node, "text")
|
||||
else:
|
||||
return node[0]
|
||||
|
||||
def getNextSibling(self, node):
|
||||
if isinstance(node, tuple): # Text node
|
||||
node, key = node
|
||||
assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
|
||||
if key == "text":
|
||||
# XXX: we cannot use a "bool(node) and node[0] or None" construct here
|
||||
# because node[0] might evaluate to False if it has no child element
|
||||
if len(node):
|
||||
return node[0]
|
||||
else:
|
||||
return None
|
||||
else: # tail
|
||||
return node.getnext()
|
||||
|
||||
return node.tail and (node, "tail") or node.getnext()
|
||||
|
||||
def getParentNode(self, node):
|
||||
if isinstance(node, tuple): # Text node
|
||||
node, key = node
|
||||
assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
|
||||
if key == "text":
|
||||
return node
|
||||
# else: fallback to "normal" processing
|
||||
|
||||
return node.getparent()
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
from six import text_type
|
||||
|
||||
from lxml import etree
|
||||
from ..treebuilders.etree import tag_regexp
|
||||
|
||||
from gettext import gettext
|
||||
_ = gettext
|
||||
|
||||
from . import _base
|
||||
|
||||
from .. import ihatexml
|
||||
|
||||
|
||||
def ensure_str(s):
|
||||
if s is None:
|
||||
return None
|
||||
elif isinstance(s, text_type):
|
||||
return s
|
||||
else:
|
||||
return s.decode("utf-8", "strict")
|
||||
|
||||
|
||||
class Root(object):
|
||||
def __init__(self, et):
|
||||
self.elementtree = et
|
||||
self.children = []
|
||||
if et.docinfo.internalDTD:
|
||||
self.children.append(Doctype(self,
|
||||
ensure_str(et.docinfo.root_name),
|
||||
ensure_str(et.docinfo.public_id),
|
||||
ensure_str(et.docinfo.system_url)))
|
||||
root = et.getroot()
|
||||
node = root
|
||||
|
||||
while node.getprevious() is not None:
|
||||
node = node.getprevious()
|
||||
while node is not None:
|
||||
self.children.append(node)
|
||||
node = node.getnext()
|
||||
|
||||
self.text = None
|
||||
self.tail = None
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.children[key]
|
||||
|
||||
def getnext(self):
|
||||
return None
|
||||
|
||||
def __len__(self):
|
||||
return 1
|
||||
|
||||
|
||||
class Doctype(object):
|
||||
def __init__(self, root_node, name, public_id, system_id):
|
||||
self.root_node = root_node
|
||||
self.name = name
|
||||
self.public_id = public_id
|
||||
self.system_id = system_id
|
||||
|
||||
self.text = None
|
||||
self.tail = None
|
||||
|
||||
def getnext(self):
|
||||
return self.root_node.children[1]
|
||||
|
||||
|
||||
class FragmentRoot(Root):
|
||||
def __init__(self, children):
|
||||
self.children = [FragmentWrapper(self, child) for child in children]
|
||||
self.text = self.tail = None
|
||||
|
||||
def getnext(self):
|
||||
return None
|
||||
|
||||
|
||||
class FragmentWrapper(object):
|
||||
def __init__(self, fragment_root, obj):
|
||||
self.root_node = fragment_root
|
||||
self.obj = obj
|
||||
if hasattr(self.obj, 'text'):
|
||||
self.text = ensure_str(self.obj.text)
|
||||
else:
|
||||
self.text = None
|
||||
if hasattr(self.obj, 'tail'):
|
||||
self.tail = ensure_str(self.obj.tail)
|
||||
else:
|
||||
self.tail = None
|
||||
self.isstring = isinstance(obj, str) or isinstance(obj, bytes)
|
||||
# Support for bytes here is Py2
|
||||
if self.isstring:
|
||||
self.obj = ensure_str(self.obj)
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.obj, name)
|
||||
|
||||
def getnext(self):
|
||||
siblings = self.root_node.children
|
||||
idx = siblings.index(self)
|
||||
if idx < len(siblings) - 1:
|
||||
return siblings[idx + 1]
|
||||
else:
|
||||
return None
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.obj[key]
|
||||
|
||||
def __bool__(self):
|
||||
return bool(self.obj)
|
||||
|
||||
def getparent(self):
|
||||
return None
|
||||
|
||||
def __str__(self):
|
||||
return str(self.obj)
|
||||
|
||||
def __unicode__(self):
|
||||
return str(self.obj)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.obj)
|
||||
|
||||
|
||||
class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
def __init__(self, tree):
|
||||
if hasattr(tree, "getroot"):
|
||||
tree = Root(tree)
|
||||
elif isinstance(tree, list):
|
||||
tree = FragmentRoot(tree)
|
||||
_base.NonRecursiveTreeWalker.__init__(self, tree)
|
||||
self.filter = ihatexml.InfosetFilter()
|
||||
|
||||
def getNodeDetails(self, node):
|
||||
if isinstance(node, tuple): # Text node
|
||||
node, key = node
|
||||
assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
|
||||
return _base.TEXT, ensure_str(getattr(node, key))
|
||||
|
||||
elif isinstance(node, Root):
|
||||
return (_base.DOCUMENT,)
|
||||
|
||||
elif isinstance(node, Doctype):
|
||||
return _base.DOCTYPE, node.name, node.public_id, node.system_id
|
||||
|
||||
elif isinstance(node, FragmentWrapper) and node.isstring:
|
||||
return _base.TEXT, node.obj
|
||||
|
||||
elif node.tag == etree.Comment:
|
||||
return _base.COMMENT, ensure_str(node.text)
|
||||
|
||||
elif node.tag == etree.Entity:
|
||||
return _base.ENTITY, ensure_str(node.text)[1:-1] # strip &;
|
||||
|
||||
else:
|
||||
# This is assumed to be an ordinary element
|
||||
match = tag_regexp.match(ensure_str(node.tag))
|
||||
if match:
|
||||
namespace, tag = match.groups()
|
||||
else:
|
||||
namespace = None
|
||||
tag = ensure_str(node.tag)
|
||||
attrs = {}
|
||||
for name, value in list(node.attrib.items()):
|
||||
name = ensure_str(name)
|
||||
value = ensure_str(value)
|
||||
match = tag_regexp.match(name)
|
||||
if match:
|
||||
attrs[(match.group(1), match.group(2))] = value
|
||||
else:
|
||||
attrs[(None, name)] = value
|
||||
return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag),
|
||||
attrs, len(node) > 0 or node.text)
|
||||
|
||||
def getFirstChild(self, node):
|
||||
assert not isinstance(node, tuple), _("Text nodes have no children")
|
||||
|
||||
assert len(node) or node.text, "Node has no children"
|
||||
if node.text:
|
||||
return (node, "text")
|
||||
else:
|
||||
return node[0]
|
||||
|
||||
def getNextSibling(self, node):
|
||||
if isinstance(node, tuple): # Text node
|
||||
node, key = node
|
||||
assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
|
||||
if key == "text":
|
||||
# XXX: we cannot use a "bool(node) and node[0] or None" construct here
|
||||
# because node[0] might evaluate to False if it has no child element
|
||||
if len(node):
|
||||
return node[0]
|
||||
else:
|
||||
return None
|
||||
else: # tail
|
||||
return node.getnext()
|
||||
|
||||
return (node, "tail") if node.tail else node.getnext()
|
||||
|
||||
def getParentNode(self, node):
|
||||
if isinstance(node, tuple): # Text node
|
||||
node, key = node
|
||||
assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
|
||||
if key == "text":
|
||||
return node
|
||||
# else: fallback to "normal" processing
|
||||
|
||||
return node.getparent()
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \
|
||||
COMMENT, IGNORABLE_WHITESPACE, CHARACTERS
|
||||
|
||||
import _base
|
||||
from . import _base
|
||||
|
||||
from ..constants import voidElements
|
||||
|
||||
from html5lib.constants import voidElements
|
||||
|
||||
class TreeWalker(_base.TreeWalker):
|
||||
def __iter__(self):
|
||||
@@ -11,7 +14,7 @@ class TreeWalker(_base.TreeWalker):
|
||||
previous = None
|
||||
for event in self.tree:
|
||||
if previous is not None and \
|
||||
(ignore_until is None or previous[1] is ignore_until):
|
||||
(ignore_until is None or previous[1] is ignore_until):
|
||||
if previous[1] is ignore_until:
|
||||
ignore_until = None
|
||||
for token in self.tokens(previous, event):
|
||||
@@ -31,9 +34,9 @@ class TreeWalker(_base.TreeWalker):
|
||||
name = node.nodeName
|
||||
namespace = node.namespaceURI
|
||||
attrs = {}
|
||||
for attr in node.attributes.keys():
|
||||
for attr in list(node.attributes.keys()):
|
||||
attr = node.getAttributeNode(attr)
|
||||
attrs[(attr.namespaceURI,attr.localName)] = attr.value
|
||||
attrs[(attr.namespaceURI, attr.localName)] = attr.value
|
||||
if name in voidElements:
|
||||
for token in self.emptyTag(namespace,
|
||||
name,
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
import gettext
|
||||
_ = gettext.gettext
|
||||
|
||||
import _base
|
||||
|
||||
class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
"""Given that simpletree has no performant way of getting a node's
|
||||
next sibling, this implementation returns "nodes" as tuples with the
|
||||
following content:
|
||||
|
||||
1. The parent Node (Element, Document or DocumentFragment)
|
||||
|
||||
2. The child index of the current node in its parent's children list
|
||||
|
||||
3. A list used as a stack of all ancestors. It is a pair tuple whose
|
||||
first item is a parent Node and second item is a child index.
|
||||
"""
|
||||
|
||||
def getNodeDetails(self, node):
|
||||
if isinstance(node, tuple): # It might be the root Node
|
||||
parent, idx, parents = node
|
||||
node = parent.childNodes[idx]
|
||||
|
||||
# testing node.type allows us not to import treebuilders.simpletree
|
||||
if node.type in (1, 2): # Document or DocumentFragment
|
||||
return (_base.DOCUMENT,)
|
||||
|
||||
elif node.type == 3: # DocumentType
|
||||
return _base.DOCTYPE, node.name, node.publicId, node.systemId
|
||||
|
||||
elif node.type == 4: # TextNode
|
||||
return _base.TEXT, node.value
|
||||
|
||||
elif node.type == 5: # Element
|
||||
attrs = {}
|
||||
for name, value in node.attributes.items():
|
||||
if isinstance(name, tuple):
|
||||
attrs[(name[2],name[1])] = value
|
||||
else:
|
||||
attrs[(None,name)] = value
|
||||
return (_base.ELEMENT, node.namespace, node.name,
|
||||
attrs, node.hasContent())
|
||||
|
||||
elif node.type == 6: # CommentNode
|
||||
return _base.COMMENT, node.data
|
||||
|
||||
else:
|
||||
return _node.UNKNOWN, node.type
|
||||
|
||||
def getFirstChild(self, node):
|
||||
if isinstance(node, tuple): # It might be the root Node
|
||||
parent, idx, parents = node
|
||||
parents.append((parent, idx))
|
||||
node = parent.childNodes[idx]
|
||||
else:
|
||||
parents = []
|
||||
|
||||
assert node.hasContent(), "Node has no children"
|
||||
return (node, 0, parents)
|
||||
|
||||
def getNextSibling(self, node):
|
||||
assert isinstance(node, tuple), "Node is not a tuple: " + str(node)
|
||||
parent, idx, parents = node
|
||||
idx += 1
|
||||
if len(parent.childNodes) > idx:
|
||||
return (parent, idx, parents)
|
||||
else:
|
||||
return None
|
||||
|
||||
def getParentNode(self, node):
|
||||
assert isinstance(node, tuple)
|
||||
parent, idx, parents = node
|
||||
if parents:
|
||||
parent, idx = parents.pop()
|
||||
return parent, idx, parents
|
||||
else:
|
||||
# HACK: We could return ``parent`` but None will stop the algorithm the same way
|
||||
return None
|
||||
@@ -1,60 +0,0 @@
|
||||
import re
|
||||
import gettext
|
||||
_ = gettext.gettext
|
||||
|
||||
from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag
|
||||
from html5lib.constants import namespaces
|
||||
import _base
|
||||
|
||||
class TreeWalker(_base.NonRecursiveTreeWalker):
|
||||
doctype_regexp = re.compile(
|
||||
r'DOCTYPE\s+(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?')
|
||||
def getNodeDetails(self, node):
|
||||
if isinstance(node, BeautifulSoup): # Document or DocumentFragment
|
||||
return (_base.DOCUMENT,)
|
||||
|
||||
elif isinstance(node, Declaration): # DocumentType
|
||||
string = unicode(node.string)
|
||||
#Slice needed to remove markup added during unicode conversion,
|
||||
#but only in some versions of BeautifulSoup/Python
|
||||
if string.startswith('<!') and string.endswith('>'):
|
||||
string = string[2:-1]
|
||||
m = self.doctype_regexp.match(string)
|
||||
#This regexp approach seems wrong and fragile
|
||||
#but beautiful soup stores the doctype as a single thing and we want the seperate bits
|
||||
#It should work as long as the tree is created by html5lib itself but may be wrong if it's
|
||||
#been modified at all
|
||||
#We could just feed to it a html5lib tokenizer, I guess...
|
||||
assert m is not None, "DOCTYPE did not match expected format"
|
||||
|
||||
name = m.group('name')
|
||||
publicId = m.group('publicId')
|
||||
if publicId is not None:
|
||||
systemId = m.group('systemId1')
|
||||
else:
|
||||
systemId = m.group('systemId2')
|
||||
return _base.DOCTYPE, name, publicId or "", systemId or ""
|
||||
|
||||
elif isinstance(node, Comment):
|
||||
string = unicode(node.string)
|
||||
if string.startswith('<!--') and string.endswith('-->'):
|
||||
string = string[4:-3]
|
||||
return _base.COMMENT, string
|
||||
|
||||
elif isinstance(node, unicode): # TextNode
|
||||
return _base.TEXT, node
|
||||
|
||||
elif isinstance(node, Tag): # Element
|
||||
return (_base.ELEMENT, namespaces["html"], node.name,
|
||||
dict(node.attrs).items(), node.contents)
|
||||
else:
|
||||
return _base.UNKNOWN, node.__class__.__name__
|
||||
|
||||
def getFirstChild(self, node):
|
||||
return node.contents[0]
|
||||
|
||||
def getNextSibling(self, node):
|
||||
return node.nextSibling
|
||||
|
||||
def getParentNode(self, node):
|
||||
return node.parent
|
||||
Reference in New Issue
Block a user