20010504
+ # Keep a ref to this for special handling of whitespace in self.processSpaceCharactersNonPre = self.processSpaceCharacters self.startTagHandler = utils.MethodDispatcher([ ("html", self.startTagHtml), - (("base", "basefont", "bgsound", "command", "link", "meta", - "noframes", "script", "style", "title"), + (("base", "basefont", "bgsound", "command", "link", "meta", + "noframes", "script", "style", "title"), self.startTagProcessInHead), ("body", self.startTagBody), ("frameset", self.startTagFrameset), (("address", "article", "aside", "blockquote", "center", "details", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", - "footer", "header", "hgroup", "menu", "nav", "ol", "p", + "footer", "header", "hgroup", "main", "menu", "nav", "ol", "p", "section", "summary", "ul"), - self.startTagCloseP), + self.startTagCloseP), (headingElements, self.startTagHeading), (("pre", "listing"), self.startTagPreListing), ("form", self.startTagForm), (("li", "dd", "dt"), self.startTagListItem), - ("plaintext",self.startTagPlaintext), + ("plaintext", self.startTagPlaintext), ("a", self.startTagA), - (("b", "big", "code", "em", "font", "i", "s", "small", "strike", - "strong", "tt", "u"),self.startTagFormatting), + (("b", "big", "code", "em", "font", "i", "s", "small", "strike", + "strong", "tt", "u"), self.startTagFormatting), ("nobr", self.startTagNobr), ("button", self.startTagButton), (("applet", "marquee", "object"), self.startTagAppletMarqueeObject), @@ -961,21 +912,21 @@ def getPhases(debug): self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([ - ("body",self.endTagBody), - ("html",self.endTagHtml), - (("address", "article", "aside", "blockquote", "center", - "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", - "footer", "header", "hgroup", "listing", "menu", "nav", "ol", "pre", + ("body", self.endTagBody), + ("html", self.endTagHtml), + (("address", "article", "aside", "blockquote", "button", "center", + "details", "dialog", "dir", "div", "dl", "fieldset", "figcaption", "figure", + "footer", "header", "hgroup", "listing", "main", "menu", "nav", "ol", "pre", "section", "summary", "ul"), self.endTagBlock), ("form", self.endTagForm), - ("p",self.endTagP), + ("p", self.endTagP), (("dd", "dt", "li"), self.endTagListItem), (headingElements, self.endTagHeading), (("a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u"), self.endTagFormatting), - (("applet", "marquee", "object"), self.endTagAppletMarqueeObject), + (("applet", "marquee", "object"), self.endTagAppletMarqueeObject), ("br", self.endTagBr), - ]) + ]) self.endTagHandler.default = self.endTagOther def isMatchingFormattingElement(self, node1, node2): @@ -995,14 +946,14 @@ def getPhases(debug): def addFormattingElement(self, token): self.tree.insertElement(token) element = self.tree.openElements[-1] - + matchingElements = [] for node in self.tree.activeFormattingElements[::-1]: if node is Marker: break elif self.isMatchingFormattingElement(node, element): matchingElements.append(node) - + assert len(matchingElements) <= 3 if len(matchingElements) == 3: self.tree.activeFormattingElements.remove(matchingElements[-1]) @@ -1017,7 +968,7 @@ def getPhases(debug): if node.name not in allowed_elements: self.parser.parseError("expected-closing-tag-but-got-eof") break - #Stop parsing + # Stop parsing def processSpaceCharactersDropNewline(self, token): # Sometimes (start of , , and blocks) we @@ -1026,19 +977,19 @@ def getPhases(debug): self.processSpaceCharacters = self.processSpaceCharactersNonPre if (data.startswith("\n") and self.tree.openElements[-1].name in ("pre", "listing", "textarea") - and not self.tree.openElements[-1].hasContent()): + and not self.tree.openElements[-1].hasContent()): data = data[1:] if data: self.tree.reconstructActiveFormattingElements() self.tree.insertText(data) def processCharacters(self, token): - if token["data"] == u"\u0000": - #The tokenizer should always emit null on its own + if token["data"] == "\u0000": + # The tokenizer should always emit null on its own return self.tree.reconstructActiveFormattingElements() self.tree.insertText(token["data"]) - #This must be bad for performance + # This must be bad for performance if (self.parser.framesetOK and any([char not in spaceCharacters for char in token["data"]])): @@ -1054,11 +1005,11 @@ def getPhases(debug): def startTagBody(self, token): self.parser.parseError("unexpected-start-tag", {"name": "body"}) if (len(self.tree.openElements) == 1 - or self.tree.openElements[1].name != "body"): + or self.tree.openElements[1].name != "body"): assert self.parser.innerHTML else: self.parser.framesetOK = False - for attr, value in token["data"].iteritems(): + for attr, value in token["data"].items(): if attr not in self.tree.openElements[1].attributes: self.tree.openElements[1].attributes[attr] = value @@ -1090,7 +1041,7 @@ def getPhases(debug): def startTagForm(self, token): if self.tree.formPointer: - self.parser.parseError(u"unexpected-start-tag", {"name": "form"}) + self.parser.parseError("unexpected-start-tag", {"name": "form"}) else: if self.tree.elementInScope("p", variant="button"): self.endTagP(impliedTagToken("p")) @@ -1100,9 +1051,9 @@ def getPhases(debug): def startTagListItem(self, token): self.parser.framesetOK = False - stopNamesMap = {"li":["li"], - "dt":["dt", "dd"], - "dd":["dt", "dd"]} + stopNamesMap = {"li": ["li"], + "dt": ["dt", "dd"], + "dd": ["dt", "dd"]} stopNames = stopNamesMap[token["name"]] for node in reversed(self.tree.openElements): if node.name in stopNames: @@ -1110,7 +1061,7 @@ def getPhases(debug): impliedTagToken(node.name, "EndTag")) break if (node.nameTuple in specialElements and - node.name not in ("address", "div", "p")): + node.name not in ("address", "div", "p")): break if self.tree.elementInScope("p", variant="button"): @@ -1137,7 +1088,7 @@ def getPhases(debug): afeAElement = self.tree.elementInActiveFormattingElements("a") if afeAElement: self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "a", "endName": "a"}) + {"startName": "a", "endName": "a"}) self.endTagFormatting(impliedTagToken("a")) if afeAElement in self.tree.openElements: self.tree.openElements.remove(afeAElement) @@ -1154,7 +1105,7 @@ def getPhases(debug): self.tree.reconstructActiveFormattingElements() if self.tree.elementInScope("nobr"): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "nobr", "endName": "nobr"}) + {"startName": "nobr", "endName": "nobr"}) self.processEndTag(impliedTagToken("nobr")) # XXX Need tests that trigger the following self.tree.reconstructActiveFormattingElements() @@ -1163,7 +1114,7 @@ def getPhases(debug): def startTagButton(self, token): if self.tree.elementInScope("button"): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "button", "endName": "button"}) + {"startName": "button", "endName": "button"}) self.processEndTag(impliedTagToken("button")) return token else: @@ -1203,8 +1154,8 @@ def getPhases(debug): framesetOK = self.parser.framesetOK self.startTagVoidFormatting(token) if ("type" in token["data"] and - token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): - #input type=hidden doesn't change framesetOK + token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): + # input type=hidden doesn't change framesetOK self.parser.framesetOK = framesetOK def startTagParamSource(self, token): @@ -1223,7 +1174,7 @@ def getPhases(debug): def startTagImage(self, token): # No really... self.parser.parseError("unexpected-start-tag-treated-as", - {"originalName": "image", "newName": "img"}) + {"originalName": "image", "newName": "img"}) self.processStartTag(impliedTagToken("img", "StartTag", attributes=token["data"], selfClosing=token["selfClosing"])) @@ -1243,18 +1194,18 @@ def getPhases(debug): if "prompt" in token["data"]: prompt = token["data"]["prompt"] else: - prompt = u"This is a searchable index. Enter search keywords: " + prompt = "This is a searchable index. Enter search keywords: " self.processCharacters( - {"type":tokenTypes["Characters"], "data":prompt}) + {"type": tokenTypes["Characters"], "data": prompt}) attributes = token["data"].copy() if "action" in attributes: del attributes["action"] if "prompt" in attributes: del attributes["prompt"] attributes["name"] = "isindex" - self.processStartTag(impliedTagToken("input", "StartTag", - attributes = attributes, - selfClosing = + self.processStartTag(impliedTagToken("input", "StartTag", + attributes=attributes, + selfClosing= token["selfClosing"])) self.processEndTag(impliedTagToken("label")) self.processStartTag(impliedTagToken("hr", "StartTag")) @@ -1287,7 +1238,7 @@ def getPhases(debug): if self.parser.phase in (self.parser.phases["inTable"], self.parser.phases["inCaption"], self.parser.phases["inColumnGroup"], - self.parser.phases["inTableBody"], + self.parser.phases["inTableBody"], self.parser.phases["inRow"], self.parser.phases["inCell"]): self.parser.phase = self.parser.phases["inSelectInTable"] @@ -1307,8 +1258,8 @@ def getPhases(debug): self.parser.adjustForeignAttributes(token) token["namespace"] = namespaces["mathml"] self.tree.insertElement(token) - #Need to get the parse error right for the case where the token - #has a namespace not equal to the xmlns attribute + # Need to get the parse error right for the case where the token + # has a namespace not equal to the xmlns attribute if token["selfClosing"]: self.tree.openElements.pop() token["selfClosingAcknowledged"] = True @@ -1319,8 +1270,8 @@ def getPhases(debug): self.parser.adjustForeignAttributes(token) token["namespace"] = namespaces["svg"] self.tree.insertElement(token) - #Need to get the parse error right for the case where the token - #has a namespace not equal to the xmlns attribute + # Need to get the parse error right for the case where the token + # has a namespace not equal to the xmlns attribute if token["selfClosing"]: self.tree.openElements.pop() token["selfClosingAcknowledged"] = True @@ -1362,7 +1313,7 @@ def getPhases(debug): "tbody", "td", "tfoot", "th", "thead", "tr", "body", "html")): - #Not sure this is the correct name for the parse error + # Not sure this is the correct name for the parse error self.parser.parseError( "expected-one-end-tag-but-got-another", {"expectedName": "body", "gotName": node.name}) @@ -1370,20 +1321,20 @@ def getPhases(debug): self.parser.phase = self.parser.phases["afterBody"] def endTagHtml(self, token): - #We repeat the test for the body end tag token being ignored here + # We repeat the test for the body end tag token being ignored here if self.tree.elementInScope("body"): self.endTagBody(impliedTagToken("body")) return token def endTagBlock(self, token): - #Put us back in the right whitespace handling mode + # Put us back in the right whitespace handling mode if token["name"] == "pre": self.processSpaceCharacters = self.processSpaceCharactersNonPre inScope = self.tree.elementInScope(token["name"]) if inScope: self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != token["name"]: - self.parser.parseError("end-tag-too-early", {"name": token["name"]}) + self.parser.parseError("end-tag-too-early", {"name": token["name"]}) if inScope: node = self.tree.openElements.pop() while node.name != token["name"]: @@ -1394,7 +1345,7 @@ def getPhases(debug): self.tree.formPointer = None if node is None or not self.tree.elementInScope(node): self.parser.parseError("unexpected-end-tag", - {"name":"form"}) + {"name": "form"}) else: self.tree.generateImpliedEndTags() if self.tree.openElements[-1] != node: @@ -1410,7 +1361,7 @@ def getPhases(debug): if not self.tree.elementInScope(token["name"], variant=variant): self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) else: - self.tree.generateImpliedEndTags(exclude = token["name"]) + self.tree.generateImpliedEndTags(exclude=token["name"]) if self.tree.openElements[-1].name != token["name"]: self.parser.parseError( "end-tag-too-early", @@ -1436,65 +1387,105 @@ def getPhases(debug): def endTagFormatting(self, token): """The much-feared adoption agency algorithm""" - # http://www.whatwg.org/specs/web-apps/current-work/#adoptionAgency + # http://svn.whatwg.org/webapps/complete.html#adoptionAgency revision 7867 # XXX Better parseError messages appreciated. - name = token["name"] + # Step 1 outerLoopCounter = 0 + + # Step 2 while outerLoopCounter < 8: + + # Step 3 outerLoopCounter += 1 - # Step 1 paragraph 1 + # Step 4: + + # Let the formatting element be the last element in + # the list of active formatting elements that: + # - is between the end of the list and the last scope + # marker in the list, if any, or the start of the list + # otherwise, and + # - has the same tag name as the token. formattingElement = self.tree.elementInActiveFormattingElements( token["name"]) - if (not formattingElement or + if (not formattingElement or (formattingElement in self.tree.openElements and not self.tree.elementInScope(formattingElement.name))): - self.parser.parseError("adoption-agency-1.1", {"name": token["name"]}) + # If there is no such node, then abort these steps + # and instead act as described in the "any other + # end tag" entry below. + self.endTagOther(token) return - # Step 1 paragraph 2 + # Otherwise, if there is such a node, but that node is + # not in the stack of open elements, then this is a + # parse error; remove the element from the list, and + # abort these steps. elif formattingElement not in self.tree.openElements: self.parser.parseError("adoption-agency-1.2", {"name": token["name"]}) self.tree.activeFormattingElements.remove(formattingElement) return - # Step 1 paragraph 3 - if formattingElement != self.tree.openElements[-1]: - self.parser.parseError("adoption-agency-1.3", {"name": token["name"]}) + # Otherwise, if there is such a node, and that node is + # also in the stack of open elements, but the element + # is not in scope, then this is a parse error; ignore + # the token, and abort these steps. + elif not self.tree.elementInScope(formattingElement.name): + self.parser.parseError("adoption-agency-4.4", {"name": token["name"]}) + return - # Step 2 - # Start of the adoption agency algorithm proper + # Otherwise, there is a formatting element and that + # element is in the stack and is in scope. If the + # element is not the current node, this is a parse + # error. In any case, proceed with the algorithm as + # written in the following steps. + else: + if formattingElement != self.tree.openElements[-1]: + self.parser.parseError("adoption-agency-1.3", {"name": token["name"]}) + + # Step 5: + + # Let the furthest block be the topmost node in the + # stack of open elements that is lower in the stack + # than the formatting element, and is an element in + # the special category. There might not be one. afeIndex = self.tree.openElements.index(formattingElement) furthestBlock = None for element in self.tree.openElements[afeIndex:]: if element.nameTuple in specialElements: furthestBlock = element break - # Step 3 + + # Step 6: + + # If there is no furthest block, then the UA must + # first pop all the nodes from the bottom of the stack + # of open elements, from the current node up to and + # including the formatting element, then remove the + # formatting element from the list of active + # formatting elements, and finally abort these steps. if furthestBlock is None: element = self.tree.openElements.pop() while element != formattingElement: element = self.tree.openElements.pop() self.tree.activeFormattingElements.remove(element) return - commonAncestor = self.tree.openElements[afeIndex-1] - # Step 5 - #if furthestBlock.parent: - # furthestBlock.parent.removeChild(furthestBlock) + # Step 7 + commonAncestor = self.tree.openElements[afeIndex - 1] - # Step 5 + # Step 8: # The bookmark is supposed to help us identify where to reinsert - # nodes in step 12. We have to ensure that we reinsert nodes after + # nodes in step 15. We have to ensure that we reinsert nodes after # the node before the active formatting element. Note the bookmark - # can move in step 7.4 + # can move in step 9.7 bookmark = self.tree.activeFormattingElements.index(formattingElement) - # Step 6 + # Step 9 lastNode = node = furthestBlock innerLoopCounter = 0 - + index = self.tree.openElements.index(node) while innerLoopCounter < 3: innerLoopCounter += 1 @@ -1504,15 +1495,13 @@ def getPhases(debug): if node not in self.tree.activeFormattingElements: self.tree.openElements.remove(node) continue - # Step 6.3 + # Step 9.6 if node == formattingElement: break - # Step 6.4 + # Step 9.7 if lastNode == furthestBlock: - bookmark = (self.tree.activeFormattingElements.index(node) - + 1) - # Step 6.5 - #cite = node.parent + bookmark = self.tree.activeFormattingElements.index(node) + 1 + # Step 9.8 clone = node.cloneNode() # Replace node with clone self.tree.activeFormattingElements[ @@ -1520,20 +1509,18 @@ def getPhases(debug): self.tree.openElements[ self.tree.openElements.index(node)] = clone node = clone - - # Step 6.6 + # Step 9.9 # Remove lastNode from its parents, if any if lastNode.parent: lastNode.parent.removeChild(lastNode) node.appendChild(lastNode) - # Step 7.7 + # Step 9.10 lastNode = node - # End of inner loop - # Step 7 + # Step 10 # Foster parent lastNode if commonAncestor is a - # table, tbody, tfoot, thead, or tr we need to foster parent the - # lastNode + # table, tbody, tfoot, thead, or tr we need to foster + # parent the lastNode if lastNode.parent: lastNode.parent.removeChild(lastNode) @@ -1543,23 +1530,23 @@ def getPhases(debug): else: commonAncestor.appendChild(lastNode) - # Step 8 + # Step 11 clone = formattingElement.cloneNode() - # Step 9 + # Step 12 furthestBlock.reparentChildren(clone) - # Step 10 + # Step 13 furthestBlock.appendChild(clone) - # Step 11 + # Step 14 self.tree.activeFormattingElements.remove(formattingElement) self.tree.activeFormattingElements.insert(bookmark, clone) - # Step 12 + # Step 15 self.tree.openElements.remove(formattingElement) self.tree.openElements.insert( - self.tree.openElements.index(furthestBlock) + 1, clone) + self.tree.openElements.index(furthestBlock) + 1, clone) def endTagAppletMarqueeObject(self, token): if self.tree.elementInScope(token["name"]): @@ -1575,7 +1562,7 @@ def getPhases(debug): def endTagBr(self, token): self.parser.parseError("unexpected-end-tag-treated-as", - {"originalName": "br", "newName": "br element"}) + {"originalName": "br", "newName": "br element"}) self.tree.reconstructActiveFormattingElements() self.tree.insertElement(impliedTagToken("br", "StartTag")) self.tree.openElements.pop() @@ -1600,31 +1587,31 @@ def getPhases(debug): self.startTagHandler = utils.MethodDispatcher([]) self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([ - ("script", self.endTagScript)]) + ("script", self.endTagScript)]) self.endTagHandler.default = self.endTagOther def processCharacters(self, token): self.tree.insertText(token["data"]) def processEOF(self): - self.parser.parseError("expected-named-closing-tag-but-got-eof", - self.tree.openElements[-1].name) + self.parser.parseError("expected-named-closing-tag-but-got-eof", + {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() self.parser.phase = self.parser.originalPhase return True def startTagOther(self, token): - assert False, "Tried to process start tag %s in RCDATA/RAWTEXT mode"%token['name'] + assert False, "Tried to process start tag %s in RCDATA/RAWTEXT mode" % token['name'] def endTagScript(self, token): node = self.tree.openElements.pop() assert node.name == "script" self.parser.phase = self.parser.originalPhase - #The rest of this method is all stuff that only happens if - #document.write works + # The rest of this method is all stuff that only happens if + # document.write works def endTagOther(self, token): - node = self.tree.openElements.pop() + self.tree.openElements.pop() self.parser.phase = self.parser.originalPhase class InTablePhase(Phase): @@ -1656,7 +1643,7 @@ def getPhases(debug): def clearStackToTableContext(self): # "clear the stack back to a table context" while self.tree.openElements[-1].name not in ("table", "html"): - #self.parser.parseError("unexpected-implied-end-tag-in-table", + # self.parser.parseError("unexpected-implied-end-tag-in-table", # {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() # When the current node is <html> it's an innerHTML case @@ -1667,7 +1654,7 @@ def getPhases(debug): self.parser.parseError("eof-in-table") else: assert self.parser.innerHTML - #Stop parsing + # Stop parsing def processSpaceCharacters(self, token): originalPhase = self.parser.phase @@ -1682,7 +1669,7 @@ def getPhases(debug): self.parser.phase.processCharacters(token) def insertText(self, token): - #If we get here there must be at least one non-whitespace character + # If we get here there must be at least one non-whitespace character # Do the table magic! self.tree.insertFromTable = True self.parser.phases["inBody"].processCharacters(token) @@ -1714,7 +1701,7 @@ def getPhases(debug): def startTagTable(self, token): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "table", "endName": "table"}) + {"startName": "table", "endName": "table"}) self.parser.phase.processEndTag(impliedTagToken("table")) if not self.parser.innerHTML: return token @@ -1723,8 +1710,8 @@ def getPhases(debug): return self.parser.phases["inHead"].processStartTag(token) def startTagInput(self, token): - if ("type" in token["data"] and - token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): + if ("type" in token["data"] and + token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): self.parser.parseError("unexpected-hidden-input-in-table") self.tree.insertElement(token) # XXX associate with form @@ -1751,8 +1738,8 @@ def getPhases(debug): self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != "table": self.parser.parseError("end-tag-too-early-named", - {"gotName": "table", - "expectedName": self.tree.openElements[-1].name}) + {"gotName": "table", + "expectedName": self.tree.openElements[-1].name}) while self.tree.openElements[-1].name != "table": self.tree.openElements.pop() self.tree.openElements.pop() @@ -1781,7 +1768,7 @@ def getPhases(debug): def flushCharacters(self): data = "".join([item["data"] for item in self.characterTokens]) if any([item not in spaceCharacters for item in data]): - token = {"type":tokenTypes["Characters"], "data":data} + token = {"type": tokenTypes["Characters"], "data": data} self.parser.phases["inTable"].insertText(token) elif data: self.tree.insertText(data) @@ -1798,12 +1785,12 @@ def getPhases(debug): return True def processCharacters(self, token): - if token["data"] == u"\u0000": + if token["data"] == "\u0000": return self.characterTokens.append(token) def processSpaceCharacters(self, token): - #pretty sure we should never reach here + # pretty sure we should never reach here self.characterTokens.append(token) # assert False @@ -1817,7 +1804,6 @@ def getPhases(debug): self.parser.phase = self.originalPhase return token - class InCaptionPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-caption def __init__(self, parser, tree): @@ -1849,7 +1835,7 @@ def getPhases(debug): def startTagTableElement(self, token): self.parser.parseError() - #XXX Have to duplicate logic here to find out if the tag is ignored + # XXX Have to duplicate logic here to find out if the tag is ignored ignoreEndTag = self.ignoreEndTagCaption() self.parser.phase.processEndTag(impliedTagToken("caption")) if not ignoreEndTag: @@ -1864,8 +1850,8 @@ def getPhases(debug): self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != "caption": self.parser.parseError("expected-one-end-tag-but-got-another", - {"gotName": "caption", - "expectedName": self.tree.openElements[-1].name}) + {"gotName": "caption", + "expectedName": self.tree.openElements[-1].name}) while self.tree.openElements[-1].name != "caption": self.tree.openElements.pop() self.tree.openElements.pop() @@ -1889,7 +1875,6 @@ def getPhases(debug): def endTagOther(self, token): return self.parser.phases["inBody"].processEndTag(token) - class InColumnGroupPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-column @@ -1955,7 +1940,6 @@ def getPhases(debug): if not ignoreEndTag: return token - class InTableBodyPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-table0 def __init__(self, parser, tree): @@ -1980,8 +1964,8 @@ def getPhases(debug): # helper methods def clearStackToTableBodyContext(self): while self.tree.openElements[-1].name not in ("tbody", "tfoot", - "thead", "html"): - #self.parser.parseError("unexpected-implied-end-tag-in-table", + "thead", "html"): + # self.parser.parseError("unexpected-implied-end-tag-in-table", # {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() if self.tree.openElements[-1].name == "html": @@ -2003,7 +1987,7 @@ def getPhases(debug): self.parser.phase = self.parser.phases["inRow"] def startTagTableCell(self, token): - self.parser.parseError("unexpected-cell-in-table-body", + self.parser.parseError("unexpected-cell-in-table-body", {"name": token["name"]}) self.startTagTr(impliedTagToken("tr", "StartTag")) return token @@ -2012,7 +1996,7 @@ def getPhases(debug): # XXX AT Any ideas on how to share this with endTagTable? if (self.tree.elementInScope("tbody", variant="table") or self.tree.elementInScope("thead", variant="table") or - self.tree.elementInScope("tfoot", variant="table")): + self.tree.elementInScope("tfoot", variant="table")): self.clearStackToTableBodyContext() self.endTagTableRowGroup( impliedTagToken(self.tree.openElements[-1].name)) @@ -2032,12 +2016,12 @@ def getPhases(debug): self.parser.phase = self.parser.phases["inTable"] else: self.parser.parseError("unexpected-end-tag-in-table-body", - {"name": token["name"]}) + {"name": token["name"]}) def endTagTable(self, token): if (self.tree.elementInScope("tbody", variant="table") or self.tree.elementInScope("thead", variant="table") or - self.tree.elementInScope("tfoot", variant="table")): + self.tree.elementInScope("tfoot", variant="table")): self.clearStackToTableBodyContext() self.endTagTableRowGroup( impliedTagToken(self.tree.openElements[-1].name)) @@ -2049,12 +2033,11 @@ def getPhases(debug): def endTagIgnore(self, token): self.parser.parseError("unexpected-end-tag-in-table-body", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOther(self, token): return self.parser.phases["inTable"].processEndTag(token) - class InRowPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-row def __init__(self, parser, tree): @@ -2072,7 +2055,7 @@ def getPhases(debug): ("table", self.endTagTable), (("tbody", "tfoot", "thead"), self.endTagTableRowGroup), (("body", "caption", "col", "colgroup", "html", "td", "th"), - self.endTagIgnore) + self.endTagIgnore) ]) self.endTagHandler.default = self.endTagOther @@ -2080,7 +2063,7 @@ def getPhases(debug): def clearStackToTableRowContext(self): while self.tree.openElements[-1].name not in ("tr", "html"): self.parser.parseError("unexpected-implied-end-tag-in-table-row", - {"name": self.tree.openElements[-1].name}) + {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() def ignoreEndTagTr(self): @@ -2091,7 +2074,7 @@ def getPhases(debug): self.parser.phases["inTable"].processEOF() def processSpaceCharacters(self, token): - return self.parser.phases["inTable"].processSpaceCharacters(token) + return self.parser.phases["inTable"].processSpaceCharacters(token) def processCharacters(self, token): return self.parser.phases["inTable"].processCharacters(token) @@ -2139,7 +2122,7 @@ def getPhases(debug): def endTagIgnore(self, token): self.parser.parseError("unexpected-end-tag-in-table-row", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOther(self, token): return self.parser.phases["inTable"].processEndTag(token) @@ -2178,7 +2161,7 @@ def getPhases(debug): def startTagTableOther(self, token): if (self.tree.elementInScope("td", variant="table") or - self.tree.elementInScope("th", variant="table")): + self.tree.elementInScope("th", variant="table")): self.closeCell() return token else: @@ -2194,7 +2177,7 @@ def getPhases(debug): self.tree.generateImpliedEndTags(token["name"]) if self.tree.openElements[-1].name != token["name"]: self.parser.parseError("unexpected-cell-end-tag", - {"name": token["name"]}) + {"name": token["name"]}) while True: node = self.tree.openElements.pop() if node.name == token["name"]: @@ -2249,7 +2232,7 @@ def getPhases(debug): assert self.parser.innerHTML def processCharacters(self, token): - if token["data"] == u"\u0000": + if token["data"] == "\u0000": return self.tree.insertText(token["data"]) @@ -2283,19 +2266,19 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-in-select", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOption(self, token): if self.tree.openElements[-1].name == "option": self.tree.openElements.pop() else: self.parser.parseError("unexpected-end-tag-in-select", - {"name": "option"}) + {"name": "option"}) def endTagOptgroup(self, token): # </optgroup> implicitly closes <option> if (self.tree.openElements[-1].name == "option" and - self.tree.openElements[-2].name == "optgroup"): + self.tree.openElements[-2].name == "optgroup"): self.tree.openElements.pop() # It also closes </optgroup> if self.tree.openElements[-1].name == "optgroup": @@ -2303,7 +2286,7 @@ def getPhases(debug): # But nothing else else: self.parser.parseError("unexpected-end-tag-in-select", - {"name": "optgroup"}) + {"name": "optgroup"}) def endTagSelect(self, token): if self.tree.elementInScope("select", variant="select"): @@ -2318,8 +2301,7 @@ def getPhases(debug): def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-in-select", - {"name": token["name"]}) - + {"name": token["name"]}) class InSelectInTablePhase(Phase): def __init__(self, parser, tree): @@ -2360,64 +2342,64 @@ def getPhases(debug): def endTagOther(self, token): return self.parser.phases["inSelect"].processEndTag(token) - class InForeignContentPhase(Phase): - breakoutElements = frozenset(["b", "big", "blockquote", "body", "br", + breakoutElements = frozenset(["b", "big", "blockquote", "body", "br", "center", "code", "dd", "div", "dl", "dt", - "em", "embed", "h1", "h2", "h3", + "em", "embed", "h1", "h2", "h3", "h4", "h5", "h6", "head", "hr", "i", "img", - "li", "listing", "menu", "meta", "nobr", - "ol", "p", "pre", "ruby", "s", "small", - "span", "strong", "strike", "sub", "sup", + "li", "listing", "menu", "meta", "nobr", + "ol", "p", "pre", "ruby", "s", "small", + "span", "strong", "strike", "sub", "sup", "table", "tt", "u", "ul", "var"]) + def __init__(self, parser, tree): Phase.__init__(self, parser, tree) def adjustSVGTagNames(self, token): - replacements = {u"altglyph":u"altGlyph", - u"altglyphdef":u"altGlyphDef", - u"altglyphitem":u"altGlyphItem", - u"animatecolor":u"animateColor", - u"animatemotion":u"animateMotion", - u"animatetransform":u"animateTransform", - u"clippath":u"clipPath", - u"feblend":u"feBlend", - u"fecolormatrix":u"feColorMatrix", - u"fecomponenttransfer":u"feComponentTransfer", - u"fecomposite":u"feComposite", - u"feconvolvematrix":u"feConvolveMatrix", - u"fediffuselighting":u"feDiffuseLighting", - u"fedisplacementmap":u"feDisplacementMap", - u"fedistantlight":u"feDistantLight", - u"feflood":u"feFlood", - u"fefunca":u"feFuncA", - u"fefuncb":u"feFuncB", - u"fefuncg":u"feFuncG", - u"fefuncr":u"feFuncR", - u"fegaussianblur":u"feGaussianBlur", - u"feimage":u"feImage", - u"femerge":u"feMerge", - u"femergenode":u"feMergeNode", - u"femorphology":u"feMorphology", - u"feoffset":u"feOffset", - u"fepointlight":u"fePointLight", - u"fespecularlighting":u"feSpecularLighting", - u"fespotlight":u"feSpotLight", - u"fetile":u"feTile", - u"feturbulence":u"feTurbulence", - u"foreignobject":u"foreignObject", - u"glyphref":u"glyphRef", - u"lineargradient":u"linearGradient", - u"radialgradient":u"radialGradient", - u"textpath":u"textPath"} + replacements = {"altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath"} if token["name"] in replacements: token["name"] = replacements[token["name"]] def processCharacters(self, token): - if token["data"] == u"\u0000": - token["data"] = u"\uFFFD" - elif (self.parser.framesetOK and + if token["data"] == "\u0000": + token["data"] = "\uFFFD" + elif (self.parser.framesetOK and any(char not in spaceCharacters for char in token["data"])): self.parser.framesetOK = False Phase.processCharacters(self, token) @@ -2428,9 +2410,9 @@ def getPhases(debug): (token["name"] == "font" and set(token["data"].keys()) & set(["color", "face", "size"]))): self.parser.parseError("unexpected-html-element-in-foreign-content", - token["name"]) + {"name": token["name"]}) while (self.tree.openElements[-1].namespace != - self.tree.defaultNamespace and + self.tree.defaultNamespace and not self.parser.isHTMLIntegrationPoint(self.tree.openElements[-1]) and not self.parser.isMathMLTextIntegrationPoint(self.tree.openElements[-1])): self.tree.openElements.pop() @@ -2453,11 +2435,11 @@ def getPhases(debug): nodeIndex = len(self.tree.openElements) - 1 node = self.tree.openElements[-1] if node.name != token["name"]: - self.parser.parseError("unexpected-end-tag", token["name"]) + self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) while True: if node.name.translate(asciiUpper2Lower) == token["name"]: - #XXX this isn't in the spec but it seems necessary + # XXX this isn't in the spec but it seems necessary if self.parser.phase == self.parser.phases["inTableText"]: self.parser.phase.flushCharacters() self.parser.phase = self.parser.phase.originalPhase @@ -2475,21 +2457,20 @@ def getPhases(debug): break return new_token - class AfterBodyPhase(Phase): def __init__(self, parser, tree): Phase.__init__(self, parser, tree) self.startTagHandler = utils.MethodDispatcher([ - ("html", self.startTagHtml) - ]) + ("html", self.startTagHtml) + ]) self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([("html", self.endTagHtml)]) self.endTagHandler.default = self.endTagOther def processEOF(self): - #Stop parsing + # Stop parsing pass def processComment(self, token): @@ -2507,11 +2488,11 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-after-body", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token - def endTagHtml(self,name): + def endTagHtml(self, name): if self.parser.innerHTML: self.parser.parseError("unexpected-end-tag-after-body-innerhtml") else: @@ -2519,7 +2500,7 @@ def getPhases(debug): def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-after-body", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token @@ -2562,7 +2543,7 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-in-frameset", - {"name": token["name"]}) + {"name": token["name"]}) def endTagFrameset(self, token): if self.tree.openElements[-1].name == "html": @@ -2571,15 +2552,14 @@ def getPhases(debug): else: self.tree.openElements.pop() if (not self.parser.innerHTML and - self.tree.openElements[-1].name != "frameset"): + self.tree.openElements[-1].name != "frameset"): # If we're not in innerHTML mode and the the current node is not a # "frameset" element (anymore) then switch. self.parser.phase = self.parser.phases["afterFrameset"] def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-in-frameset", - {"name": token["name"]}) - + {"name": token["name"]}) class AfterFramesetPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#after3 @@ -2598,7 +2578,7 @@ def getPhases(debug): self.endTagHandler.default = self.endTagOther def processEOF(self): - #Stop parsing + # Stop parsing pass def processCharacters(self, token): @@ -2609,15 +2589,14 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-after-frameset", - {"name": token["name"]}) + {"name": token["name"]}) def endTagHtml(self, token): self.parser.phase = self.parser.phases["afterAfterFrameset"] def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-after-frameset", - {"name": token["name"]}) - + {"name": token["name"]}) class AfterAfterBodyPhase(Phase): def __init__(self, parser, tree): @@ -2647,13 +2626,13 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("expected-eof-but-got-start-tag", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token def processEndTag(self, token): self.parser.parseError("expected-eof-but-got-end-tag", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token @@ -2687,12 +2666,11 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("expected-eof-but-got-start-tag", - {"name": token["name"]}) + {"name": token["name"]}) def processEndTag(self, token): self.parser.parseError("expected-eof-but-got-end-tag", - {"name": token["name"]}) - + {"name": token["name"]}) return { "initial": InitialPhase, @@ -2719,14 +2697,16 @@ def getPhases(debug): "afterAfterBody": AfterAfterBodyPhase, "afterAfterFrameset": AfterAfterFramesetPhase, # XXX after after frameset - } + } -def impliedTagToken(name, type="EndTag", attributes = None, - selfClosing = False): + +def impliedTagToken(name, type="EndTag", attributes=None, + selfClosing=False): if attributes is None: attributes = {} - return {"type":tokenTypes[type], "name":unicode(name), "data":attributes, - "selfClosing":selfClosing} + return {"type": tokenTypes[type], "name": name, "data": attributes, + "selfClosing": selfClosing} + class ParseError(Exception): """Error in parsed document""" diff --git a/libs/html5lib/ihatexml.py b/libs/html5lib/ihatexml.py index dd785639..0fc79308 100644 --- a/libs/html5lib/ihatexml.py +++ b/libs/html5lib/ihatexml.py @@ -1,25 +1,105 @@ -import re +from __future__ import absolute_import, division, unicode_literals -baseChar = """[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | [#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | [#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | [#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | [#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | [#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | [#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | [#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | [#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | [#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | [#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | [#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | [#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | [#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | [#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | [#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | [#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | [#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | [#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | [#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | [#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | [#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | [#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | [#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | [#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | [#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | [#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | [#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | [#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | #x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | #x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | #x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | [#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | [#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | #x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | [#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | [#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | [#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | [#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | [#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | #x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | [#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | [#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | [#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | [#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]""" +import re +import warnings + +from .constants import DataLossWarning + +baseChar = """ +[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | +[#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | +[#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | +[#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | +[#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | +[#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | +[#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | +[#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | +[#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | +[#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | +[#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | +[#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | +[#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | +[#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | +[#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | +[#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | +[#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | +[#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | +[#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | +[#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | +[#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | +[#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | +[#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | +[#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | +[#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | +[#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | +[#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | +[#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | +[#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | +[#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | +#x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | +#x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | +#x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | +[#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | +[#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | +#x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | +[#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | +[#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | +[#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | +[#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | +[#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | +#x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | +[#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | +[#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | +[#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | +[#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]""" ideographic = """[#x4E00-#x9FA5] | #x3007 | [#x3021-#x3029]""" -combiningCharacter = """[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | [#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | [#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | [#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | #x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | [#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | [#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | #x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | [#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | [#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | #x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | [#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | [#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | [#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | [#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | [#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | #x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | [#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | #x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | [#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | [#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | #x3099 | #x309A""" +combiningCharacter = """ +[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | +[#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | +[#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | +[#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | +#x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | +[#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | +[#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | +#x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | +[#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | +[#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | +#x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | +[#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | +[#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | +[#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | +[#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | +[#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | +#x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | +[#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | +#x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | +[#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | +[#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | +#x3099 | #x309A""" -digit = """[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | [#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | [#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | [#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]""" +digit = """ +[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | +[#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | +[#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | +[#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]""" -extender = """#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | [#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]""" +extender = """ +#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | +#[#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]""" letter = " | ".join([baseChar, ideographic]) -#Without the -name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, - extender]) +# Without the +name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, + extender]) nameFirst = " | ".join([letter, "_"]) reChar = re.compile(r"#x([\d|A-F]{4,4})") reCharRange = re.compile(r"\[#x([\d|A-F]{4,4})-#x([\d|A-F]{4,4})\]") + def charStringToList(chars): charRanges = [item.strip() for item in chars.split(" | ")] rv = [] @@ -30,16 +110,17 @@ def charStringToList(chars): if match is not None: rv.append([hexToInt(item) for item in match.groups()]) if len(rv[-1]) == 1: - rv[-1] = rv[-1]*2 + rv[-1] = rv[-1] * 2 foundMatch = True break if not foundMatch: assert len(item) == 1 - + rv.append([ord(item)] * 2) rv = normaliseCharList(rv) return rv + def normaliseCharList(charList): charList = sorted(charList) for item in charList: @@ -49,61 +130,69 @@ def normaliseCharList(charList): while i < len(charList): j = 1 rv.append(charList[i]) - while i + j < len(charList) and charList[i+j][0] <= rv[-1][1] + 1: - rv[-1][1] = charList[i+j][1] + while i + j < len(charList) and charList[i + j][0] <= rv[-1][1] + 1: + rv[-1][1] = charList[i + j][1] j += 1 i += j return rv -#We don't really support characters above the BMP :( +# We don't really support characters above the BMP :( max_unicode = int("FFFF", 16) - + + def missingRanges(charList): rv = [] if charList[0] != 0: rv.append([0, charList[0][0] - 1]) for i, item in enumerate(charList[:-1]): - rv.append([item[1]+1, charList[i+1][0] - 1]) + rv.append([item[1] + 1, charList[i + 1][0] - 1]) if charList[-1][1] != max_unicode: rv.append([charList[-1][1] + 1, max_unicode]) return rv + def listToRegexpStr(charList): rv = [] for item in charList: if item[0] == item[1]: - rv.append(escapeRegexp(unichr(item[0]))) + rv.append(escapeRegexp(chr(item[0]))) else: - rv.append(escapeRegexp(unichr(item[0])) + "-" + - escapeRegexp(unichr(item[1]))) - return "[%s]"%"".join(rv) + rv.append(escapeRegexp(chr(item[0])) + "-" + + escapeRegexp(chr(item[1]))) + return "[%s]" % "".join(rv) + def hexToInt(hex_str): return int(hex_str, 16) + def escapeRegexp(string): specialCharacters = (".", "^", "$", "*", "+", "?", "{", "}", - "[", "]", "|", "(", ")", "-") + "[", "]", "|", "(", ")", "-") for char in specialCharacters: string = string.replace(char, "\\" + char) - if char in string: - print string return string -#output from the above -nonXmlNameBMPRegexp = re.compile(u'[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') +# output from the above +nonXmlNameBMPRegexp = re.compile('[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') + +nonXmlNameFirstBMPRegexp = re.compile('[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') + +# Simpler things +nonPubidCharRegexp = re.compile("[^\x20\x0D\x0Aa-zA-Z0-9\-\'()+,./:=?;!*#@$_%]") -nonXmlNameFirstBMPRegexp = re.compile(u'[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') class InfosetFilter(object): replacementRegexp = re.compile(r"U[\dA-F]{5,5}") - def __init__(self, replaceChars = None, - dropXmlnsLocalName = False, - dropXmlnsAttrNs = False, - preventDoubleDashComments = False, - preventDashAtCommentEnd = False, - replaceFormFeedCharacters = True): + + def __init__(self, replaceChars=None, + dropXmlnsLocalName=False, + dropXmlnsAttrNs=False, + preventDoubleDashComments=False, + preventDashAtCommentEnd=False, + replaceFormFeedCharacters=True, + preventSingleQuotePubid=False): self.dropXmlnsLocalName = dropXmlnsLocalName self.dropXmlnsAttrNs = dropXmlnsAttrNs @@ -113,14 +202,17 @@ class InfosetFilter(object): self.replaceFormFeedCharacters = replaceFormFeedCharacters + self.preventSingleQuotePubid = preventSingleQuotePubid + self.replaceCache = {} def coerceAttribute(self, name, namespace=None): if self.dropXmlnsLocalName and name.startswith("xmlns:"): - #Need a datalosswarning here + warnings.warn("Attributes cannot begin with xmlns", DataLossWarning) return None - elif (self.dropXmlnsAttrNs and + elif (self.dropXmlnsAttrNs and namespace == "http://www.w3.org/2000/xmlns/"): + warnings.warn("Attributes cannot be in the xml namespace", DataLossWarning) return None else: return self.toXmlName(name) @@ -131,20 +223,35 @@ class InfosetFilter(object): def coerceComment(self, data): if self.preventDoubleDashComments: while "--" in data: + warnings.warn("Comments cannot contain adjacent dashes", DataLossWarning) data = data.replace("--", "- -") return data - + def coerceCharacters(self, data): if self.replaceFormFeedCharacters: + for i in range(data.count("\x0C")): + warnings.warn("Text cannot contain U+000C", DataLossWarning) data = data.replace("\x0C", " ") - #Other non-xml characters + # Other non-xml characters return data + def coercePubid(self, data): + dataOutput = data + for char in nonPubidCharRegexp.findall(data): + warnings.warn("Coercing non-XML pubid", DataLossWarning) + replacement = self.getReplacementCharacter(char) + dataOutput = dataOutput.replace(char, replacement) + if self.preventSingleQuotePubid and dataOutput.find("'") >= 0: + warnings.warn("Pubid cannot contain single quote", DataLossWarning) + dataOutput = dataOutput.replace("'", self.getReplacementCharacter("'")) + return dataOutput + def toXmlName(self, name): nameFirst = name[0] nameRest = name[1:] m = nonXmlNameFirstBMPRegexp.match(nameFirst) if m: + warnings.warn("Coercing non-XML name", DataLossWarning) nameFirstOutput = self.getReplacementCharacter(nameFirst) else: nameFirstOutput = nameFirst @@ -152,10 +259,11 @@ class InfosetFilter(object): nameRestOutput = nameRest replaceChars = set(nonXmlNameBMPRegexp.findall(nameRest)) for char in replaceChars: + warnings.warn("Coercing non-XML name", DataLossWarning) replacement = self.getReplacementCharacter(char) nameRestOutput = nameRestOutput.replace(char, replacement) return nameFirstOutput + nameRestOutput - + def getReplacementCharacter(self, char): if char in self.replaceCache: replacement = self.replaceCache[char] @@ -169,9 +277,9 @@ class InfosetFilter(object): return name def escapeChar(self, char): - replacement = "U" + hex(ord(char))[2:].upper().rjust(5, "0") + replacement = "U%05X" % ord(char) self.replaceCache[char] = replacement return replacement def unescapeChar(self, charcode): - return unichr(int(charcode[1:], 16)) + return chr(int(charcode[1:], 16)) diff --git a/libs/html5lib/inputstream.py b/libs/html5lib/inputstream.py index edec1329..004bdd4a 100644 --- a/libs/html5lib/inputstream.py +++ b/libs/html5lib/inputstream.py @@ -1,19 +1,33 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type + import codecs import re -import types -import sys -from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase -from constants import encodings, ReparseException -import utils +from .constants import EOF, spaceCharacters, asciiLetters, asciiUppercase +from .constants import encodings, ReparseException +from . import utils -#Non-unicode versions of constants for use in the pre-parser -spaceCharactersBytes = frozenset([str(item) for item in spaceCharacters]) -asciiLettersBytes = frozenset([str(item) for item in asciiLetters]) -asciiUppercaseBytes = frozenset([str(item) for item in asciiUppercase]) -spacesAngleBrackets = spaceCharactersBytes | frozenset([">", "<"]) +from io import StringIO -invalid_unicode_re = re.compile(u"[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]") +try: + from io import BytesIO +except ImportError: + BytesIO = StringIO + +try: + from io import BufferedIOBase +except ImportError: + class BufferedIOBase(object): + pass + +# Non-unicode versions of constants for use in the pre-parser +spaceCharactersBytes = frozenset([item.encode("ascii") for item in spaceCharacters]) +asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters]) +asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase]) +spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"]) + +invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]") non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, @@ -23,22 +37,23 @@ non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF]) -ascii_punctuation_re = re.compile(ur"[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]") +ascii_punctuation_re = re.compile("[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]") # Cache for charsUntil() charsUntilRegEx = {} - -class BufferedStream: + + +class BufferedStream(object): """Buffering for streams that do not have buffering of their own - The buffer is implemented as a list of chunks on the assumption that + The buffer is implemented as a list of chunks on the assumption that joining many strings will be slow since it is O(n**2) """ - + def __init__(self, stream): self.stream = stream self.buffer = [] - self.position = [-1,0] #chunk number, offset + self.position = [-1, 0] # chunk number, offset def tell(self): pos = 0 @@ -48,11 +63,11 @@ class BufferedStream: return pos def seek(self, pos): - assert pos < self._bufferedBytes() + assert pos <= self._bufferedBytes() offset = pos i = 0 while len(self.buffer[i]) < offset: - offset -= pos + offset -= len(self.buffer[i]) i += 1 self.position = [i, offset] @@ -64,7 +79,7 @@ class BufferedStream: return self._readStream(bytes) else: return self._readFromBuffer(bytes) - + def _bufferedBytes(self): return sum([len(item) for item in self.buffer]) @@ -83,7 +98,7 @@ class BufferedStream: while bufferIndex < len(self.buffer) and remainingBytes != 0: assert remainingBytes > 0 bufferedData = self.buffer[bufferIndex] - + if remainingBytes <= len(bufferedData) - bufferOffset: bytesToRead = remainingBytes self.position = [bufferIndex, bufferOffset + bytesToRead] @@ -91,20 +106,33 @@ class BufferedStream: bytesToRead = len(bufferedData) - bufferOffset self.position = [bufferIndex, len(bufferedData)] bufferIndex += 1 - data = rv.append(bufferedData[bufferOffset: - bufferOffset + bytesToRead]) + rv.append(bufferedData[bufferOffset:bufferOffset + bytesToRead]) remainingBytes -= bytesToRead bufferOffset = 0 if remainingBytes: rv.append(self._readStream(remainingBytes)) - - return "".join(rv) - + + return b"".join(rv) -class HTMLInputStream: +def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True): + if hasattr(source, "read"): + isUnicode = isinstance(source.read(0), text_type) + else: + isUnicode = isinstance(source, text_type) + + if isUnicode: + if encoding is not None: + raise TypeError("Cannot explicitly set an encoding with a unicode string") + + return HTMLUnicodeInputStream(source) + else: + return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) + + +class HTMLUnicodeInputStream(object): """Provides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing @@ -114,7 +142,7 @@ class HTMLInputStream: _defaultChunkSize = 10240 - def __init__(self, source, encoding=None, parseMeta=True, chardet=True): + def __init__(self, source): """Initialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source @@ -126,49 +154,29 @@ class HTMLInputStream: the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) - + parseMeta - Look for a <meta> element containing encoding information """ - #Craziness - if len(u"\U0010FFFF") == 1: + # Craziness + if len("\U0010FFFF") == 1: self.reportCharacterErrors = self.characterErrorsUCS4 - self.replaceCharactersRegexp = re.compile(u"[\uD800-\uDFFF]") + self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]") else: self.reportCharacterErrors = self.characterErrorsUCS2 - self.replaceCharactersRegexp = re.compile(u"([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])") + self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])") # List of where new lines occur self.newLines = [0] - self.charEncoding = (codecName(encoding), "certain") - - # Raw Stream - for unicode objects this will encode to utf-8 and set - # self.charEncoding as appropriate - self.rawStream = self.openStream(source) - - # Encoding Information - #Number of bytes to use when looking for a meta element with - #encoding information - self.numBytesMeta = 512 - #Number of bytes to use when using detecting encoding using chardet - self.numBytesChardet = 100 - #Encoding to use if no other information can be found - self.defaultEncoding = "windows-1252" - - #Detect encoding iff no explicit "transport level" encoding is supplied - if (self.charEncoding[0] is None): - self.charEncoding = self.detectEncoding(parseMeta, chardet) - + self.charEncoding = ("utf-8", "certain") + self.dataStream = self.openStream(source) self.reset() def reset(self): - self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream, - 'replace') - - self.chunk = u"" + self.chunk = "" self.chunkSize = 0 self.chunkOffset = 0 self.errors = [] @@ -177,8 +185,8 @@ class HTMLInputStream: self.prevNumLines = 0 # number of columns in the last line of the previous chunk self.prevNumCols = 0 - - #Deal with CR LF and surrogates split over chunk boundaries + + # Deal with CR LF and surrogates split over chunk boundaries self._bufferedCharacter = None def openStream(self, source): @@ -191,126 +199,15 @@ class HTMLInputStream: if hasattr(source, 'read'): stream = source else: - # Otherwise treat source as a string and convert to a file object - if isinstance(source, unicode): - source = source.encode('utf-8') - self.charEncoding = ("utf-8", "certain") - try: - from io import BytesIO - except: - # 2to3 converts this line to: from io import StringIO - from cStringIO import StringIO as BytesIO - stream = BytesIO(source) - - if (not(hasattr(stream, "tell") and hasattr(stream, "seek")) or - stream is sys.stdin): - stream = BufferedStream(stream) + stream = StringIO(source) return stream - def detectEncoding(self, parseMeta=True, chardet=True): - #First look for a BOM - #This will also read past the BOM if present - encoding = self.detectBOM() - confidence = "certain" - #If there is no BOM need to look for meta elements with encoding - #information - if encoding is None and parseMeta: - encoding = self.detectEncodingMeta() - confidence = "tentative" - #Guess with chardet, if avaliable - if encoding is None and chardet: - confidence = "tentative" - try: - from chardet.universaldetector import UniversalDetector - buffers = [] - detector = UniversalDetector() - while not detector.done: - buffer = self.rawStream.read(self.numBytesChardet) - if not buffer: - break - buffers.append(buffer) - detector.feed(buffer) - detector.close() - encoding = detector.result['encoding'] - self.rawStream.seek(0) - except ImportError: - pass - # If all else fails use the default encoding - if encoding is None: - confidence="tentative" - encoding = self.defaultEncoding - - #Substitute for equivalent encodings: - encodingSub = {"iso-8859-1":"windows-1252"} - - if encoding.lower() in encodingSub: - encoding = encodingSub[encoding.lower()] - - return encoding, confidence - - def changeEncoding(self, newEncoding): - newEncoding = codecName(newEncoding) - if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"): - newEncoding = "utf-8" - if newEncoding is None: - return - elif newEncoding == self.charEncoding[0]: - self.charEncoding = (self.charEncoding[0], "certain") - else: - self.rawStream.seek(0) - self.reset() - self.charEncoding = (newEncoding, "certain") - raise ReparseException, "Encoding changed from %s to %s"%(self.charEncoding[0], newEncoding) - - def detectBOM(self): - """Attempts to detect at BOM at the start of the stream. If - an encoding can be determined from the BOM return the name of the - encoding otherwise return None""" - bomDict = { - codecs.BOM_UTF8: 'utf-8', - codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be', - codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be' - } - - # Go to beginning of file and read in 4 bytes - string = self.rawStream.read(4) - - # Try detecting the BOM using bytes from the string - encoding = bomDict.get(string[:3]) # UTF-8 - seek = 3 - if not encoding: - # Need to detect UTF-32 before UTF-16 - encoding = bomDict.get(string) # UTF-32 - seek = 4 - if not encoding: - encoding = bomDict.get(string[:2]) # UTF-16 - seek = 2 - - # Set the read position past the BOM if one was found, otherwise - # set it to the start of the stream - self.rawStream.seek(encoding and seek or 0) - - return encoding - - def detectEncodingMeta(self): - """Report the encoding declared by the meta element - """ - buffer = self.rawStream.read(self.numBytesMeta) - parser = EncodingParser(buffer) - self.rawStream.seek(0) - encoding = parser.getEncoding() - - if encoding in ("utf-16", "utf-16-be", "utf-16-le"): - encoding = "utf-8" - - return encoding - def _position(self, offset): chunk = self.chunk - nLines = chunk.count(u'\n', 0, offset) + nLines = chunk.count('\n', 0, offset) positionLine = self.prevNumLines + nLines - lastLinePos = chunk.rfind(u'\n', 0, offset) + lastLinePos = chunk.rfind('\n', 0, offset) if lastLinePos == -1: positionColumn = self.prevNumCols + offset else: @@ -320,7 +217,7 @@ class HTMLInputStream: def position(self): """Returns (line, col) of the current position in the stream.""" line, col = self._position(self.chunkOffset) - return (line+1, col) + return (line + 1, col) def char(self): """ Read one character from the stream or queue if available. Return @@ -343,34 +240,34 @@ class HTMLInputStream: self.prevNumLines, self.prevNumCols = self._position(self.chunkSize) - self.chunk = u"" + self.chunk = "" self.chunkSize = 0 self.chunkOffset = 0 data = self.dataStream.read(chunkSize) - - #Deal with CR LF and surrogates broken across chunks + + # Deal with CR LF and surrogates broken across chunks if self._bufferedCharacter: data = self._bufferedCharacter + data self._bufferedCharacter = None elif not data: # We have no more data, bye-bye stream return False - + if len(data) > 1: lastv = ord(data[-1]) if lastv == 0x0D or 0xD800 <= lastv <= 0xDBFF: self._bufferedCharacter = data[-1] data = data[:-1] - + self.reportCharacterErrors(data) - + # Replace invalid characters # Note U+0000 is dealt with in the tokenizer - data = self.replaceCharactersRegexp.sub(u"\ufffd", data) - - data = data.replace(u"\r\n", u"\n") - data = data.replace(u"\r", u"\n") + data = self.replaceCharactersRegexp.sub("\ufffd", data) + + data = data.replace("\r\n", "\n") + data = data.replace("\r", "\n") self.chunk = data self.chunkSize = len(data) @@ -378,23 +275,22 @@ class HTMLInputStream: return True def characterErrorsUCS4(self, data): - for i in xrange(len(invalid_unicode_re.findall(data))): + for i in range(len(invalid_unicode_re.findall(data))): self.errors.append("invalid-codepoint") def characterErrorsUCS2(self, data): - #Someone picked the wrong compile option - #You lose + # Someone picked the wrong compile option + # You lose skip = False - import sys for match in invalid_unicode_re.finditer(data): if skip: continue codepoint = ord(match.group()) pos = match.start() - #Pretty sure there should be endianness issues here - if utils.isSurrogatePair(data[pos:pos+2]): - #We have a surrogate pair! - char_val = utils.surrogatePairToCodepoint(data[pos:pos+2]) + # Pretty sure there should be endianness issues here + if utils.isSurrogatePair(data[pos:pos + 2]): + # We have a surrogate pair! + char_val = utils.surrogatePairToCodepoint(data[pos:pos + 2]) if char_val in non_bmp_invalid_codepoints: self.errors.append("invalid-codepoint") skip = True @@ -405,7 +301,7 @@ class HTMLInputStream: skip = False self.errors.append("invalid-codepoint") - def charsUntil(self, characters, opposite = False): + def charsUntil(self, characters, opposite=False): """ Returns a string of characters from the stream up to but not including any character in 'characters' or EOF. 'characters' must be a container that supports the 'in' method and iteration over its @@ -417,12 +313,12 @@ class HTMLInputStream: chars = charsUntilRegEx[(characters, opposite)] except KeyError: if __debug__: - for c in characters: + for c in characters: assert(ord(c) < 128) - regex = u"".join([u"\\x%02x" % ord(c) for c in characters]) + regex = "".join(["\\x%02x" % ord(c) for c in characters]) if not opposite: - regex = u"^%s" % regex - chars = charsUntilRegEx[(characters, opposite)] = re.compile(u"[%s]+" % regex) + regex = "^%s" % regex + chars = charsUntilRegEx[(characters, opposite)] = re.compile("[%s]+" % regex) rv = [] @@ -449,7 +345,7 @@ class HTMLInputStream: # Reached EOF break - r = u"".join(rv) + r = "".join(rv) return r def unget(self, char): @@ -468,26 +364,210 @@ class HTMLInputStream: self.chunkOffset -= 1 assert self.chunk[self.chunkOffset] == char -class EncodingBytes(str): + +class HTMLBinaryInputStream(HTMLUnicodeInputStream): + """Provides a unicode stream of characters to the HTMLTokenizer. + + This class takes care of character encoding and removing or replacing + incorrect byte-sequences and also provides column and line tracking. + + """ + + def __init__(self, source, encoding=None, parseMeta=True, chardet=True): + """Initialises the HTMLInputStream. + + HTMLInputStream(source, [encoding]) -> Normalized stream from source + for use by html5lib. + + source can be either a file-object, local filename or a string. + + The optional encoding parameter must be a string that indicates + the encoding. If specified, that encoding will be used, + regardless of any BOM or later declaration (such as in a meta + element) + + parseMeta - Look for a <meta> element containing encoding information + + """ + # Raw Stream - for unicode objects this will encode to utf-8 and set + # self.charEncoding as appropriate + self.rawStream = self.openStream(source) + + HTMLUnicodeInputStream.__init__(self, self.rawStream) + + self.charEncoding = (codecName(encoding), "certain") + + # Encoding Information + # Number of bytes to use when looking for a meta element with + # encoding information + self.numBytesMeta = 512 + # Number of bytes to use when using detecting encoding using chardet + self.numBytesChardet = 100 + # Encoding to use if no other information can be found + self.defaultEncoding = "windows-1252" + + # Detect encoding iff no explicit "transport level" encoding is supplied + if (self.charEncoding[0] is None): + self.charEncoding = self.detectEncoding(parseMeta, chardet) + + # Call superclass + self.reset() + + def reset(self): + self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream, + 'replace') + HTMLUnicodeInputStream.reset(self) + + def openStream(self, source): + """Produces a file object from source. + + source can be either a file object, local filename or a string. + + """ + # Already a file object + if hasattr(source, 'read'): + stream = source + else: + stream = BytesIO(source) + + try: + stream.seek(stream.tell()) + except: + stream = BufferedStream(stream) + + return stream + + def detectEncoding(self, parseMeta=True, chardet=True): + # First look for a BOM + # This will also read past the BOM if present + encoding = self.detectBOM() + confidence = "certain" + # If there is no BOM need to look for meta elements with encoding + # information + if encoding is None and parseMeta: + encoding = self.detectEncodingMeta() + confidence = "tentative" + # Guess with chardet, if avaliable + if encoding is None and chardet: + confidence = "tentative" + try: + try: + from charade.universaldetector import UniversalDetector + except ImportError: + from chardet.universaldetector import UniversalDetector + buffers = [] + detector = UniversalDetector() + while not detector.done: + buffer = self.rawStream.read(self.numBytesChardet) + assert isinstance(buffer, bytes) + if not buffer: + break + buffers.append(buffer) + detector.feed(buffer) + detector.close() + encoding = detector.result['encoding'] + self.rawStream.seek(0) + except ImportError: + pass + # If all else fails use the default encoding + if encoding is None: + confidence = "tentative" + encoding = self.defaultEncoding + + # Substitute for equivalent encodings: + encodingSub = {"iso-8859-1": "windows-1252"} + + if encoding.lower() in encodingSub: + encoding = encodingSub[encoding.lower()] + + return encoding, confidence + + def changeEncoding(self, newEncoding): + assert self.charEncoding[1] != "certain" + newEncoding = codecName(newEncoding) + if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"): + newEncoding = "utf-8" + if newEncoding is None: + return + elif newEncoding == self.charEncoding[0]: + self.charEncoding = (self.charEncoding[0], "certain") + else: + self.rawStream.seek(0) + self.reset() + self.charEncoding = (newEncoding, "certain") + raise ReparseException("Encoding changed from %s to %s" % (self.charEncoding[0], newEncoding)) + + def detectBOM(self): + """Attempts to detect at BOM at the start of the stream. If + an encoding can be determined from the BOM return the name of the + encoding otherwise return None""" + bomDict = { + codecs.BOM_UTF8: 'utf-8', + codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be', + codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be' + } + + # Go to beginning of file and read in 4 bytes + string = self.rawStream.read(4) + assert isinstance(string, bytes) + + # Try detecting the BOM using bytes from the string + encoding = bomDict.get(string[:3]) # UTF-8 + seek = 3 + if not encoding: + # Need to detect UTF-32 before UTF-16 + encoding = bomDict.get(string) # UTF-32 + seek = 4 + if not encoding: + encoding = bomDict.get(string[:2]) # UTF-16 + seek = 2 + + # Set the read position past the BOM if one was found, otherwise + # set it to the start of the stream + self.rawStream.seek(encoding and seek or 0) + + return encoding + + def detectEncodingMeta(self): + """Report the encoding declared by the meta element + """ + buffer = self.rawStream.read(self.numBytesMeta) + assert isinstance(buffer, bytes) + parser = EncodingParser(buffer) + self.rawStream.seek(0) + encoding = parser.getEncoding() + + if encoding in ("utf-16", "utf-16-be", "utf-16-le"): + encoding = "utf-8" + + return encoding + + +class EncodingBytes(bytes): """String-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raised""" def __new__(self, value): - return str.__new__(self, value.lower()) + assert isinstance(value, bytes) + return bytes.__new__(self, value.lower()) def __init__(self, value): - self._position=-1 - + self._position = -1 + def __iter__(self): return self - - def next(self): + + def __next__(self): p = self._position = self._position + 1 if p >= len(self): raise StopIteration elif p < 0: raise TypeError - return self[p] + return self[p:p + 1] + + def next(self): + # Py2 compat + return self.__next__() def previous(self): p = self._position @@ -496,13 +576,13 @@ class EncodingBytes(str): elif p < 0: raise TypeError self._position = p = p - 1 - return self[p] - + return self[p:p + 1] + def setPosition(self, position): if self._position >= len(self): raise StopIteration self._position = position - + def getPosition(self): if self._position >= len(self): raise StopIteration @@ -510,19 +590,19 @@ class EncodingBytes(str): return self._position else: return None - + position = property(getPosition, setPosition) def getCurrentByte(self): - return self[self.position] - + return self[self.position:self.position + 1] + currentByte = property(getCurrentByte) def skip(self, chars=spaceCharactersBytes): """Skip past a list of characters""" p = self.position # use property for the error-checking while p < len(self): - c = self[p] + c = self[p:p + 1] if c not in chars: self._position = p return c @@ -533,7 +613,7 @@ class EncodingBytes(str): def skipUntil(self, chars): p = self.position while p < len(self): - c = self[p] + c = self[p:p + 1] if c in chars: self._position = p return c @@ -542,16 +622,16 @@ class EncodingBytes(str): return None def matchBytes(self, bytes): - """Look for a sequence of bytes at the start of a string. If the bytes - are found return True and advance the position to the byte after the + """Look for a sequence of bytes at the start of a string. If the bytes + are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone""" p = self.position - data = self[p:p+len(bytes)] + data = self[p:p + len(bytes)] rv = data.startswith(bytes) if rv: self.position += len(bytes) return rv - + def jumpTo(self, bytes): """Look for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the match""" @@ -560,11 +640,12 @@ class EncodingBytes(str): # XXX: This is ugly, but I can't see a nicer way to fix this. if self._position == -1: self._position = 0 - self._position += (newPosition + len(bytes)-1) + self._position += (newPosition + len(bytes) - 1) return True else: raise StopIteration + class EncodingParser(object): """Mini parser for detecting character encoding from meta elements""" @@ -575,147 +656,158 @@ class EncodingParser(object): def getEncoding(self): methodDispatch = ( - ("<!--",self.handleComment), - ("<meta",self.handleMeta), - ("</",self.handlePossibleEndTag), - ("<!",self.handleOther), - ("<?",self.handleOther), - ("<",self.handlePossibleStartTag)) + (b"<!--", self.handleComment), + (b"<meta", self.handleMeta), + (b"</", self.handlePossibleEndTag), + (b"<!", self.handleOther), + (b"<?", self.handleOther), + (b"<", self.handlePossibleStartTag)) for byte in self.data: keepParsing = True for key, method in methodDispatch: if self.data.matchBytes(key): try: - keepParsing = method() + keepParsing = method() break except StopIteration: - keepParsing=False + keepParsing = False break if not keepParsing: break - + return self.encoding def handleComment(self): """Skip over comments""" - return self.data.jumpTo("-->") + return self.data.jumpTo(b"-->") def handleMeta(self): if self.data.currentByte not in spaceCharactersBytes: - #if we have <meta not followed by a space so just keep going + # if we have <meta not followed by a space so just keep going return True - #We have a valid meta element we want to search for attributes + # We have a valid meta element we want to search for attributes + hasPragma = False + pendingEncoding = None while True: - #Try to find the next attribute after the current position + # Try to find the next attribute after the current position attr = self.getAttribute() if attr is None: return True else: - if attr[0] == "charset": + if attr[0] == b"http-equiv": + hasPragma = attr[1] == b"content-type" + if hasPragma and pendingEncoding is not None: + self.encoding = pendingEncoding + return False + elif attr[0] == b"charset": tentativeEncoding = attr[1] codec = codecName(tentativeEncoding) if codec is not None: self.encoding = codec return False - elif attr[0] == "content": + elif attr[0] == b"content": contentParser = ContentAttrParser(EncodingBytes(attr[1])) tentativeEncoding = contentParser.parse() - codec = codecName(tentativeEncoding) - if codec is not None: - self.encoding = codec - return False + if tentativeEncoding is not None: + codec = codecName(tentativeEncoding) + if codec is not None: + if hasPragma: + self.encoding = codec + return False + else: + pendingEncoding = codec def handlePossibleStartTag(self): return self.handlePossibleTag(False) def handlePossibleEndTag(self): - self.data.next() + next(self.data) return self.handlePossibleTag(True) def handlePossibleTag(self, endTag): data = self.data if data.currentByte not in asciiLettersBytes: - #If the next byte is not an ascii letter either ignore this - #fragment (possible start tag case) or treat it according to - #handleOther + # If the next byte is not an ascii letter either ignore this + # fragment (possible start tag case) or treat it according to + # handleOther if endTag: data.previous() self.handleOther() return True - + c = data.skipUntil(spacesAngleBrackets) - if c == "<": - #return to the first step in the overall "two step" algorithm - #reprocessing the < byte + if c == b"<": + # return to the first step in the overall "two step" algorithm + # reprocessing the < byte data.previous() else: - #Read all attributes + # Read all attributes attr = self.getAttribute() while attr is not None: attr = self.getAttribute() return True def handleOther(self): - return self.data.jumpTo(">") + return self.data.jumpTo(b">") def getAttribute(self): - """Return a name,value pair for the next attribute in the stream, + """Return a name,value pair for the next attribute in the stream, if one is found, or None""" data = self.data # Step 1 (skip chars) - c = data.skip(spaceCharactersBytes | frozenset("/")) + c = data.skip(spaceCharactersBytes | frozenset([b"/"])) + assert c is None or len(c) == 1 # Step 2 - if c in (">", None): + if c in (b">", None): return None # Step 3 attrName = [] attrValue = [] - #Step 4 attribute name + # Step 4 attribute name while True: - if c == "=" and attrName: + if c == b"=" and attrName: break elif c in spaceCharactersBytes: - #Step 6! + # Step 6! c = data.skip() - c = data.next() break - elif c in ("/", ">"): - return "".join(attrName), "" + elif c in (b"/", b">"): + return b"".join(attrName), b"" elif c in asciiUppercaseBytes: attrName.append(c.lower()) - elif c == None: + elif c is None: return None else: attrName.append(c) - #Step 5 - c = data.next() - #Step 7 - if c != "=": + # Step 5 + c = next(data) + # Step 7 + if c != b"=": data.previous() - return "".join(attrName), "" - #Step 8 - data.next() - #Step 9 + return b"".join(attrName), b"" + # Step 8 + next(data) + # Step 9 c = data.skip() - #Step 10 - if c in ("'", '"'): - #10.1 + # Step 10 + if c in (b"'", b'"'): + # 10.1 quoteChar = c while True: - #10.2 - c = data.next() - #10.3 + # 10.2 + c = next(data) + # 10.3 if c == quoteChar: - data.next() - return "".join(attrName), "".join(attrValue) - #10.4 + next(data) + return b"".join(attrName), b"".join(attrValue) + # 10.4 elif c in asciiUppercaseBytes: attrValue.append(c.lower()) - #10.5 + # 10.5 else: attrValue.append(c) - elif c == ">": - return "".join(attrName), "" + elif c == b">": + return b"".join(attrName), b"" elif c in asciiUppercaseBytes: attrValue.append(c.lower()) elif c is None: @@ -724,9 +816,9 @@ class EncodingParser(object): attrValue.append(c) # Step 11 while True: - c = data.next() + c = next(data) if c in spacesAngleBrackets: - return "".join(attrName), "".join(attrValue) + return b"".join(attrName), b"".join(attrValue) elif c in asciiUppercaseBytes: attrValue.append(c.lower()) elif c is None: @@ -737,21 +829,23 @@ class EncodingParser(object): class ContentAttrParser(object): def __init__(self, data): + assert isinstance(data, bytes) self.data = data + def parse(self): try: - #Check if the attr name is charset - #otherwise return - self.data.jumpTo("charset") + # Check if the attr name is charset + # otherwise return + self.data.jumpTo(b"charset") self.data.position += 1 self.data.skip() - if not self.data.currentByte == "=": - #If there is no = sign keep looking for attrs + if not self.data.currentByte == b"=": + # If there is no = sign keep looking for attrs return None self.data.position += 1 self.data.skip() - #Look for an encoding between matching quote marks - if self.data.currentByte in ('"', "'"): + # Look for an encoding between matching quote marks + if self.data.currentByte in (b'"', b"'"): quoteMark = self.data.currentByte self.data.position += 1 oldPosition = self.data.position @@ -760,13 +854,13 @@ class ContentAttrParser(object): else: return None else: - #Unquoted value + # Unquoted value oldPosition = self.data.position try: self.data.skipUntil(spaceCharactersBytes) return self.data[oldPosition:self.data.position] except StopIteration: - #Return the whole remaining value + # Return the whole remaining value return self.data[oldPosition:] except StopIteration: return None @@ -775,7 +869,12 @@ class ContentAttrParser(object): def codecName(encoding): """Return the python codec name corresponding to an encoding or None if the string doesn't correspond to a valid encoding.""" - if (encoding is not None and type(encoding) in types.StringTypes): + if isinstance(encoding, bytes): + try: + encoding = encoding.decode("ascii") + except UnicodeDecodeError: + return None + if encoding: canonicalName = ascii_punctuation_re.sub("", encoding).lower() return encodings.get(canonicalName, None) else: diff --git a/libs/html5lib/sanitizer.py b/libs/html5lib/sanitizer.py index ae4c7d83..71dc5212 100644 --- a/libs/html5lib/sanitizer.py +++ b/libs/html5lib/sanitizer.py @@ -1,142 +1,145 @@ +from __future__ import absolute_import, division, unicode_literals + import re from xml.sax.saxutils import escape, unescape -from tokenizer import HTMLTokenizer -from constants import tokenTypes +from .tokenizer import HTMLTokenizer +from .constants import tokenTypes + class HTMLSanitizerMixin(object): """ sanitization of XHTML+MathML+SVG and of inline style attributes.""" acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', - 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', - 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', - 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', - 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', - 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', - 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', - 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', - 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', - 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', - 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', - 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] - + 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', + 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', + 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', + 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', + 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', + 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', + 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', + 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', + 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', + 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', + 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', + 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] + mathml_elements = ['maction', 'math', 'merror', 'mfrac', 'mi', - 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', - 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', - 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', - 'munderover', 'none'] - + 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', + 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', + 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', + 'munderover', 'none'] + svg_elements = ['a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', - 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', - 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] - + 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', + 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', + 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', + 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', + 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] + acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', - 'background', 'balance', 'bgcolor', 'bgproperties', 'border', - 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', - 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', - 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', - 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', - 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', - 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', - 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', - 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', - 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', - 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', - 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', - 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', - 'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg', - 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', - 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', - 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', - 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', - 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', - 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', - 'width', 'wrap', 'xml:lang'] + 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', + 'background', 'balance', 'bgcolor', 'bgproperties', 'border', + 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', + 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', + 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', + 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', + 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', + 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', + 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', + 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', + 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', + 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', + 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', + 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', + 'optimum', 'pattern', 'ping', 'point-size', 'poster', 'pqg', 'preload', + 'prompt', 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', + 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', + 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', + 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', + 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', + 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', + 'width', 'wrap', 'xml:lang'] mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign', - 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', - 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', - 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', - 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', - 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', - 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', - 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', - 'xlink:type', 'xmlns', 'xmlns:xlink'] - - svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', - 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', - 'fill-opacity', 'fill-rule', 'font-family', 'font-size', - 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', - 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', - 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', - 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', - 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', - 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', - 'opacity', 'orient', 'origin', 'overline-position', - 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', - 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', - 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', - 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', - 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', - 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', - 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', - 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', - 'transform', 'type', 'u1', 'u2', 'underline-position', - 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', - 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', - 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', - 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', - 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', - 'y1', 'y2', 'zoomAndPan'] + 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', + 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', + 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', + 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', + 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', + 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', + 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', + 'xlink:type', 'xmlns', 'xmlns:xlink'] - attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', - 'xlink:href', 'xml:base'] + svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', + 'arabic-form', 'ascent', 'attributeName', 'attributeType', + 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', + 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', + 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', + 'fill-opacity', 'fill-rule', 'font-family', 'font-size', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', + 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', + 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', + 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', + 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', + 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', + 'opacity', 'orient', 'origin', 'overline-position', + 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', + 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', + 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', + 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', + 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', + 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', + 'transform', 'type', 'u1', 'u2', 'underline-position', + 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', + 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', + 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', + 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', + 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', + 'y1', 'y2', 'zoomAndPan'] + + attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', + 'xlink:href', 'xml:base'] svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill', - 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', - 'mask', 'stroke'] + 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', + 'mask', 'stroke'] svg_allow_local_href = ['altGlyph', 'animate', 'animateColor', - 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', - 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', - 'set', 'use'] - + 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', + 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', + 'set', 'use'] + acceptable_css_properties = ['azimuth', 'background-color', - 'border-bottom-color', 'border-collapse', 'border-color', - 'border-left-color', 'border-right-color', 'border-top-color', 'clear', - 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', - 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', - 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', - 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', - 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', - 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', - 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', - 'white-space', 'width'] - + 'border-bottom-color', 'border-collapse', 'border-color', + 'border-left-color', 'border-right-color', 'border-top-color', 'clear', + 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', + 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', + 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', + 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', + 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', + 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', + 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', + 'white-space', 'width'] + acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue', - 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', - 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', - 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', - 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', - 'transparent', 'underline', 'white', 'yellow'] - - acceptable_svg_properties = [ 'fill', 'fill-opacity', 'fill-rule', - 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', - 'stroke-opacity'] - - acceptable_protocols = [ 'ed2k', 'ftp', 'http', 'https', 'irc', - 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', - 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', - 'ssh', 'sftp', 'rtsp', 'afs' ] - + 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', + 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', + 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', + 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', + 'transparent', 'underline', 'white', 'yellow'] + + acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule', + 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', + 'stroke-opacity'] + + acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc', + 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', + 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', + 'ssh', 'sftp', 'rtsp', 'afs'] + # subclasses may define their own versions of these constants allowed_elements = acceptable_elements + mathml_elements + svg_elements allowed_attributes = acceptable_attributes + mathml_attributes + svg_attributes @@ -160,94 +163,104 @@ class HTMLSanitizerMixin(object): # accommodate filters which use token_type differently token_type = token["type"] - if token_type in tokenTypes.keys(): - token_type = tokenTypes[token_type] + if token_type in list(tokenTypes.keys()): + token_type = tokenTypes[token_type] - if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], - tokenTypes["EmptyTag"]): + if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], + tokenTypes["EmptyTag"]): if token["name"] in self.allowed_elements: - if token.has_key("data"): - attrs = dict([(name,val) for name,val in - token["data"][::-1] - if name in self.allowed_attributes]) - for attr in self.attr_val_is_uri: - if not attrs.has_key(attr): - continue - val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', - unescape(attrs[attr])).lower() - #remove replacement characters from unescaped characters - val_unescaped = val_unescaped.replace(u"\ufffd", "") - if (re.match("^[a-z0-9][-+.a-z0-9]*:",val_unescaped) and - (val_unescaped.split(':')[0] not in - self.allowed_protocols)): - del attrs[attr] - for attr in self.svg_attr_val_allows_ref: - if attr in attrs: - attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', - ' ', - unescape(attrs[attr])) - if (token["name"] in self.svg_allow_local_href and - 'xlink:href' in attrs and re.search('^\s*[^#\s].*', - attrs['xlink:href'])): - del attrs['xlink:href'] - if attrs.has_key('style'): - attrs['style'] = self.sanitize_css(attrs['style']) - token["data"] = [[name,val] for name,val in attrs.items()] - return token + return self.allowed_token(token, token_type) else: - if token_type == tokenTypes["EndTag"]: - token["data"] = "</%s>" % token["name"] - elif token["data"]: - attrs = ''.join([' %s="%s"' % (k,escape(v)) for k,v in token["data"]]) - token["data"] = "<%s%s>" % (token["name"],attrs) - else: - token["data"] = "<%s>" % token["name"] - if token.get("selfClosing"): - token["data"]=token["data"][:-1] + "/>" - - if token["type"] in tokenTypes.keys(): - token["type"] = "Characters" - else: - token["type"] = tokenTypes["Characters"] - - del token["name"] - return token + return self.disallowed_token(token, token_type) elif token_type == tokenTypes["Comment"]: pass else: return token + def allowed_token(self, token, token_type): + if "data" in token: + attrs = dict([(name, val) for name, val in + token["data"][::-1] + if name in self.allowed_attributes]) + for attr in self.attr_val_is_uri: + if attr not in attrs: + continue + val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', + unescape(attrs[attr])).lower() + # remove replacement characters from unescaped characters + val_unescaped = val_unescaped.replace("\ufffd", "") + if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and + (val_unescaped.split(':')[0] not in + self.allowed_protocols)): + del attrs[attr] + for attr in self.svg_attr_val_allows_ref: + if attr in attrs: + attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', + ' ', + unescape(attrs[attr])) + if (token["name"] in self.svg_allow_local_href and + 'xlink:href' in attrs and re.search('^\s*[^#\s].*', + attrs['xlink:href'])): + del attrs['xlink:href'] + if 'style' in attrs: + attrs['style'] = self.sanitize_css(attrs['style']) + token["data"] = [[name, val] for name, val in list(attrs.items())] + return token + + def disallowed_token(self, token, token_type): + if token_type == tokenTypes["EndTag"]: + token["data"] = "</%s>" % token["name"] + elif token["data"]: + attrs = ''.join([' %s="%s"' % (k, escape(v)) for k, v in token["data"]]) + token["data"] = "<%s%s>" % (token["name"], attrs) + else: + token["data"] = "<%s>" % token["name"] + if token.get("selfClosing"): + token["data"] = token["data"][:-1] + "/>" + + if token["type"] in list(tokenTypes.keys()): + token["type"] = "Characters" + else: + token["type"] = tokenTypes["Characters"] + + del token["name"] + return token + def sanitize_css(self, style): # disallow urls - style=re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ',style) + style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style) # gauntlet - if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): return '' - if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): return '' + if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): + return '' + if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): + return '' clean = [] - for prop,value in re.findall("([-\w]+)\s*:\s*([^:;]*)",style): - if not value: continue - if prop.lower() in self.allowed_css_properties: - clean.append(prop + ': ' + value + ';') - elif prop.split('-')[0].lower() in ['background','border','margin', - 'padding']: - for keyword in value.split(): - if not keyword in self.acceptable_css_keywords and \ - not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$",keyword): - break - else: - clean.append(prop + ': ' + value + ';') - elif prop.lower() in self.allowed_svg_properties: - clean.append(prop + ': ' + value + ';') + for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style): + if not value: + continue + if prop.lower() in self.allowed_css_properties: + clean.append(prop + ': ' + value + ';') + elif prop.split('-')[0].lower() in ['background', 'border', 'margin', + 'padding']: + for keyword in value.split(): + if not keyword in self.acceptable_css_keywords and \ + not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword): + break + else: + clean.append(prop + ': ' + value + ';') + elif prop.lower() in self.allowed_svg_properties: + clean.append(prop + ': ' + value + ';') return ' '.join(clean) + class HTMLSanitizer(HTMLTokenizer, HTMLSanitizerMixin): def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True, lowercaseElementName=False, lowercaseAttrName=False, parser=None): - #Change case matching defaults as we only output lowercase html anyway - #This solution doesn't seem ideal... + # Change case matching defaults as we only output lowercase html anyway + # This solution doesn't seem ideal... HTMLTokenizer.__init__(self, stream, encoding, parseMeta, useChardet, lowercaseElementName, lowercaseAttrName, parser=parser) diff --git a/libs/html5lib/serializer/__init__.py b/libs/html5lib/serializer/__init__.py index 1b746655..8380839a 100644 --- a/libs/html5lib/serializer/__init__.py +++ b/libs/html5lib/serializer/__init__.py @@ -1,17 +1,16 @@ +from __future__ import absolute_import, division, unicode_literals -from html5lib import treewalkers +from .. import treewalkers -from htmlserializer import HTMLSerializer -from xhtmlserializer import XHTMLSerializer +from .htmlserializer import HTMLSerializer -def serialize(input, tree="simpletree", format="html", encoding=None, + +def serialize(input, tree="etree", format="html", encoding=None, **serializer_opts): # XXX: Should we cache this? - walker = treewalkers.getTreeWalker(tree) + walker = treewalkers.getTreeWalker(tree) if format == "html": s = HTMLSerializer(**serializer_opts) - elif format == "xhtml": - s = XHTMLSerializer(**serializer_opts) else: - raise ValueError, "type must be either html or xhtml" + raise ValueError("type must be html") return s.render(walker(input), encoding) diff --git a/libs/html5lib/serializer/htmlserializer.py b/libs/html5lib/serializer/htmlserializer.py index 8dd0a815..412a5a22 100644 --- a/libs/html5lib/serializer/htmlserializer.py +++ b/libs/html5lib/serializer/htmlserializer.py @@ -1,18 +1,20 @@ -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import ImmutableSet as frozenset +from __future__ import absolute_import, division, unicode_literals +from six import text_type import gettext _ = gettext.gettext -from html5lib.constants import voidElements, booleanAttributes, spaceCharacters -from html5lib.constants import rcdataElements, entities, xmlEntities -from html5lib import utils +try: + from functools import reduce +except ImportError: + pass + +from ..constants import voidElements, booleanAttributes, spaceCharacters +from ..constants import rcdataElements, entities, xmlEntities +from .. import utils from xml.sax.saxutils import escape -spaceCharacters = u"".join(spaceCharacters) +spaceCharacters = "".join(spaceCharacters) try: from codecs import register_error, xmlcharrefreplace_errors @@ -21,24 +23,18 @@ except ImportError: else: unicode_encode_errors = "htmlentityreplace" - from html5lib.constants import entities - encode_entity_map = {} - is_ucs4 = len(u"\U0010FFFF") == 1 - for k, v in entities.items(): - #skip multi-character entities + is_ucs4 = len("\U0010FFFF") == 1 + for k, v in list(entities.items()): + # skip multi-character entities if ((is_ucs4 and len(v) > 1) or - (not is_ucs4 and len(v) > 2)): + (not is_ucs4 and len(v) > 2)): continue if v != "&": if len(v) == 2: v = utils.surrogatePairToCodepoint(v) else: - try: - v = ord(v) - except: - print v - raise + v = ord(v) if not v in encode_entity_map or k.islower(): # prefer < over < and similarly for &, >, etc. encode_entity_map[v] = k @@ -53,8 +49,8 @@ else: skip = False continue index = i + exc.start - if utils.isSurrogatePair(exc.object[index:min([exc.end, index+2])]): - codepoint = utils.surrogatePairToCodepoint(exc.object[index:index+2]) + if utils.isSurrogatePair(exc.object[index:min([exc.end, index + 2])]): + codepoint = utils.surrogatePairToCodepoint(exc.object[index:index + 2]) skip = True else: codepoint = ord(c) @@ -67,8 +63,8 @@ else: if not e.endswith(";"): res.append(";") else: - res.append("&#x%s;"%(hex(cp)[2:])) - return (u"".join(res), exc.end) + res.append("&#x%s;" % (hex(cp)[2:])) + return ("".join(res), exc.end) else: return xmlcharrefreplace_errors(exc) @@ -81,7 +77,7 @@ class HTMLSerializer(object): # attribute quoting options quote_attr_values = False - quote_char = u'"' + quote_char = '"' use_best_quote_char = True # tag syntax options @@ -96,15 +92,17 @@ class HTMLSerializer(object): resolve_entities = True # miscellaneous options + alphabetical_attributes = False inject_meta_charset = True strip_whitespace = False sanitize = False options = ("quote_attr_values", "quote_char", "use_best_quote_char", - "minimize_boolean_attributes", "use_trailing_solidus", - "space_before_trailing_solidus", "omit_optional_tags", - "strip_whitespace", "inject_meta_charset", "escape_lt_in_attrs", - "escape_rcdata", "resolve_entities", "sanitize") + "omit_optional_tags", "minimize_boolean_attributes", + "use_trailing_solidus", "space_before_trailing_solidus", + "escape_lt_in_attrs", "escape_rcdata", "resolve_entities", + "alphabetical_attributes", "inject_meta_charset", + "strip_whitespace", "sanitize") def __init__(self, **kwargs): """Initialize HTMLSerializer. @@ -147,10 +145,12 @@ class HTMLSerializer(object): See `html5lib user documentation`_ omit_optional_tags=True|False Omit start/end tags that are optional. + alphabetical_attributes=False|True + Reorder attributes to be in alphabetical order. .. _html5lib user documentation: http://code.google.com/p/html5lib/wiki/UserDocumentation """ - if kwargs.has_key('quote_char'): + if 'quote_char' in kwargs: self.use_best_quote_char = False for attr in self.options: setattr(self, attr, kwargs.get(attr, getattr(self, attr))) @@ -158,14 +158,14 @@ class HTMLSerializer(object): self.strict = False def encode(self, string): - assert(isinstance(string, unicode)) + assert(isinstance(string, text_type)) if self.encoding: return string.encode(self.encoding, unicode_encode_errors) else: return string def encodeStrict(self, string): - assert(isinstance(string, unicode)) + assert(isinstance(string, text_type)) if self.encoding: return string.encode(self.encoding, "strict") else: @@ -175,39 +175,46 @@ class HTMLSerializer(object): self.encoding = encoding in_cdata = False self.errors = [] + if encoding and self.inject_meta_charset: - from html5lib.filters.inject_meta_charset import Filter + from ..filters.inject_meta_charset import Filter treewalker = Filter(treewalker, encoding) - # XXX: WhitespaceFilter should be used before OptionalTagFilter + # WhitespaceFilter should be used before OptionalTagFilter # for maximum efficiently of this latter filter if self.strip_whitespace: - from html5lib.filters.whitespace import Filter + from ..filters.whitespace import Filter treewalker = Filter(treewalker) if self.sanitize: - from html5lib.filters.sanitizer import Filter + from ..filters.sanitizer import Filter treewalker = Filter(treewalker) if self.omit_optional_tags: - from html5lib.filters.optionaltags import Filter + from ..filters.optionaltags import Filter treewalker = Filter(treewalker) + # Alphabetical attributes must be last, as other filters + # could add attributes and alter the order + if self.alphabetical_attributes: + from ..filters.alphabeticalattributes import Filter + treewalker = Filter(treewalker) + for token in treewalker: type = token["type"] if type == "Doctype": - doctype = u"<!DOCTYPE %s" % token["name"] - + doctype = "<!DOCTYPE %s" % token["name"] + if token["publicId"]: - doctype += u' PUBLIC "%s"' % token["publicId"] + doctype += ' PUBLIC "%s"' % token["publicId"] elif token["systemId"]: - doctype += u" SYSTEM" - if token["systemId"]: - if token["systemId"].find(u'"') >= 0: - if token["systemId"].find(u"'") >= 0: + doctype += " SYSTEM" + if token["systemId"]: + if token["systemId"].find('"') >= 0: + if token["systemId"].find("'") >= 0: self.serializeError(_("System identifer contains both single and double quote characters")) - quote_char = u"'" + quote_char = "'" else: - quote_char = u'"' - doctype += u" %s%s%s" % (quote_char, token["systemId"], quote_char) - - doctype += u">" + quote_char = '"' + doctype += " %s%s%s" % (quote_char, token["systemId"], quote_char) + + doctype += ">" yield self.encodeStrict(doctype) elif type in ("Characters", "SpaceCharacters"): @@ -220,41 +227,41 @@ class HTMLSerializer(object): elif type in ("StartTag", "EmptyTag"): name = token["name"] - yield self.encodeStrict(u"<%s" % name) + yield self.encodeStrict("<%s" % name) if name in rcdataElements and not self.escape_rcdata: in_cdata = True elif in_cdata: self.serializeError(_("Unexpected child element of a CDATA element")) - attributes = [] - for (attr_namespace,attr_name),attr_value in sorted(token["data"].items()): - #TODO: Add namespace support here + for (attr_namespace, attr_name), attr_value in token["data"].items(): + # TODO: Add namespace support here k = attr_name v = attr_value - yield self.encodeStrict(u' ') + yield self.encodeStrict(' ') yield self.encodeStrict(k) if not self.minimize_boolean_attributes or \ - (k not in booleanAttributes.get(name, tuple()) \ - and k not in booleanAttributes.get("", tuple())): - yield self.encodeStrict(u"=") + (k not in booleanAttributes.get(name, tuple()) + and k not in booleanAttributes.get("", tuple())): + yield self.encodeStrict("=") if self.quote_attr_values or not v: quote_attr = True else: - quote_attr = reduce(lambda x,y: x or (y in v), - spaceCharacters + u">\"'=", False) - v = v.replace(u"&", u"&") - if self.escape_lt_in_attrs: v = v.replace(u"<", u"<") + quote_attr = reduce(lambda x, y: x or (y in v), + spaceCharacters + ">\"'=", False) + v = v.replace("&", "&") + if self.escape_lt_in_attrs: + v = v.replace("<", "<") if quote_attr: quote_char = self.quote_char if self.use_best_quote_char: - if u"'" in v and u'"' not in v: - quote_char = u'"' - elif u'"' in v and u"'" not in v: - quote_char = u"'" - if quote_char == u"'": - v = v.replace(u"'", u"'") + if "'" in v and '"' not in v: + quote_char = '"' + elif '"' in v and "'" not in v: + quote_char = "'" + if quote_char == "'": + v = v.replace("'", "'") else: - v = v.replace(u'"', u""") + v = v.replace('"', """) yield self.encodeStrict(quote_char) yield self.encode(v) yield self.encodeStrict(quote_char) @@ -262,10 +269,10 @@ class HTMLSerializer(object): yield self.encode(v) if name in voidElements and self.use_trailing_solidus: if self.space_before_trailing_solidus: - yield self.encodeStrict(u" /") + yield self.encodeStrict(" /") else: - yield self.encodeStrict(u"/") - yield self.encode(u">") + yield self.encodeStrict("/") + yield self.encode(">") elif type == "EndTag": name = token["name"] @@ -273,13 +280,13 @@ class HTMLSerializer(object): in_cdata = False elif in_cdata: self.serializeError(_("Unexpected child element of a CDATA element")) - yield self.encodeStrict(u"</%s>" % name) + yield self.encodeStrict("</%s>" % name) elif type == "Comment": data = token["data"] if data.find("--") >= 0: self.serializeError(_("Comment contains --")) - yield self.encodeStrict(u"<!--%s-->" % token["data"]) + yield self.encodeStrict("<!--%s-->" % token["data"]) elif type == "Entity": name = token["name"] @@ -289,7 +296,7 @@ class HTMLSerializer(object): if self.resolve_entities and key not in xmlEntities: data = entities[key] else: - data = u"&%s;" % name + data = "&%s;" % name yield self.encodeStrict(data) else: @@ -297,9 +304,9 @@ class HTMLSerializer(object): def render(self, treewalker, encoding=None): if encoding: - return "".join(list(self.serialize(treewalker, encoding))) + return b"".join(list(self.serialize(treewalker, encoding))) else: - return u"".join(list(self.serialize(treewalker))) + return "".join(list(self.serialize(treewalker))) def serializeError(self, data="XXX ERROR MESSAGE NEEDED"): # XXX The idea is to make data mandatory. @@ -307,6 +314,7 @@ class HTMLSerializer(object): if self.strict: raise SerializeError + def SerializeError(Exception): """Error in serialized tree""" pass diff --git a/libs/html5lib/serializer/xhtmlserializer.py b/libs/html5lib/serializer/xhtmlserializer.py deleted file mode 100644 index 7fdce47b..00000000 --- a/libs/html5lib/serializer/xhtmlserializer.py +++ /dev/null @@ -1,9 +0,0 @@ -from htmlserializer import HTMLSerializer - -class XHTMLSerializer(HTMLSerializer): - quote_attr_values = True - minimize_boolean_attributes = False - use_trailing_solidus = True - escape_lt_in_attrs = True - omit_optional_tags = False - escape_rcdata = True diff --git a/libs/html5lib/tokenizer.py b/libs/html5lib/tokenizer.py index 7e9eca88..79774578 100644 --- a/libs/html5lib/tokenizer.py +++ b/libs/html5lib/tokenizer.py @@ -1,27 +1,25 @@ +from __future__ import absolute_import, division, unicode_literals + try: - frozenset + chr = unichr # flake8: noqa except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset -try: - from collections import deque -except ImportError: - from utils import deque - -from constants import spaceCharacters -from constants import entitiesWindows1252, entities -from constants import asciiLowercase, asciiLetters, asciiUpper2Lower -from constants import digits, hexDigits, EOF -from constants import tokenTypes, tagTokenTypes -from constants import replacementCharacters + pass -from inputstream import HTMLInputStream +from collections import deque + +from .constants import spaceCharacters +from .constants import entities +from .constants import asciiLetters, asciiUpper2Lower +from .constants import digits, hexDigits, EOF +from .constants import tokenTypes, tagTokenTypes +from .constants import replacementCharacters + +from .inputstream import HTMLInputStream + +from .trie import Trie + +entitiesTrie = Trie(entities) -# Group entities by their first character, for faster lookups -entitiesByFirstChar = {} -for e in entities: - entitiesByFirstChar.setdefault(e[0], []).append(e) class HTMLTokenizer(object): """ This class takes care of tokenizing HTML. @@ -42,10 +40,10 @@ class HTMLTokenizer(object): self.stream = HTMLInputStream(stream, encoding, parseMeta, useChardet) self.parser = parser - #Perform case conversions? + # Perform case conversions? self.lowercaseElementName = lowercaseElementName self.lowercaseAttrName = lowercaseAttrName - + # Setup the initial tokenizer state self.escapeFlag = False self.lastFourChars = [] @@ -100,78 +98,79 @@ class HTMLTokenizer(object): if charAsInt in replacementCharacters: char = replacementCharacters[charAsInt] self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", - "datavars": {"charAsInt": charAsInt}}) - elif ((0xD800 <= charAsInt <= 0xDFFF) or + "illegal-codepoint-for-numeric-entity", + "datavars": {"charAsInt": charAsInt}}) + elif ((0xD800 <= charAsInt <= 0xDFFF) or (charAsInt > 0x10FFFF)): - char = u"\uFFFD" + char = "\uFFFD" self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", - "datavars": {"charAsInt": charAsInt}}) + "illegal-codepoint-for-numeric-entity", + "datavars": {"charAsInt": charAsInt}}) else: - #Should speed up this check somehow (e.g. move the set to a constant) - if ((0x0001 <= charAsInt <= 0x0008) or - (0x000E <= charAsInt <= 0x001F) or - (0x007F <= charAsInt <= 0x009F) or - (0xFDD0 <= charAsInt <= 0xFDEF) or - charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE, + # Should speed up this check somehow (e.g. move the set to a constant) + if ((0x0001 <= charAsInt <= 0x0008) or + (0x000E <= charAsInt <= 0x001F) or + (0x007F <= charAsInt <= 0x009F) or + (0xFDD0 <= charAsInt <= 0xFDEF) or + charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, - 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, + 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, - 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE, - 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, - 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, + 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE, + 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, + 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF])): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", + "illegal-codepoint-for-numeric-entity", "datavars": {"charAsInt": charAsInt}}) try: # Try/except needed as UCS-2 Python builds' unichar only works # within the BMP. - char = unichr(charAsInt) + char = chr(charAsInt) except ValueError: - char = eval("u'\\U%08x'" % charAsInt) + v = charAsInt - 0x10000 + char = chr(0xD800 | (v >> 10)) + chr(0xDC00 | (v & 0x3FF)) # Discard the ; if present. Otherwise, put it back on the queue and # invoke parseError on parser. - if c != u";": + if c != ";": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "numeric-entity-without-semicolon"}) + "numeric-entity-without-semicolon"}) self.stream.unget(c) return char def consumeEntity(self, allowedChar=None, fromAttribute=False): # Initialise to the default output for when no entity is matched - output = u"&" + output = "&" charStack = [self.stream.char()] - if (charStack[0] in spaceCharacters or charStack[0] in (EOF, u"<", u"&") - or (allowedChar is not None and allowedChar == charStack[0])): + if (charStack[0] in spaceCharacters or charStack[0] in (EOF, "<", "&") + or (allowedChar is not None and allowedChar == charStack[0])): self.stream.unget(charStack[0]) - elif charStack[0] == u"#": + elif charStack[0] == "#": # Read the next character to see if it's hex or decimal hex = False charStack.append(self.stream.char()) - if charStack[-1] in (u"x", u"X"): + if charStack[-1] in ("x", "X"): hex = True charStack.append(self.stream.char()) # charStack[-1] should be the first digit if (hex and charStack[-1] in hexDigits) \ - or (not hex and charStack[-1] in digits): + or (not hex and charStack[-1] in digits): # At least one digit found, so consume the whole number self.stream.unget(charStack[-1]) output = self.consumeNumberEntity(hex) else: # No digits found self.tokenQueue.append({"type": tokenTypes["ParseError"], - "data": "expected-numeric-entity"}) + "data": "expected-numeric-entity"}) self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) else: # At this point in the process might have named entity. Entities @@ -179,46 +178,40 @@ class HTMLTokenizer(object): # # Consume characters and compare to these to a substring of the # entity names in the list until the substring no longer matches. - filteredEntityList = entitiesByFirstChar.get(charStack[0], []) - - def entitiesStartingWith(name): - return [e for e in filteredEntityList if e.startswith(name)] - - while (charStack[-1] is not EOF and - entitiesStartingWith("".join(charStack))): + while (charStack[-1] is not EOF): + if not entitiesTrie.has_keys_with_prefix("".join(charStack)): + break charStack.append(self.stream.char()) # At this point we have a string that starts with some characters # that may match an entity - entityName = None - # Try to find the longest entity the string will match to take care # of ¬i for instance. - for entityLength in xrange(len(charStack)-1, 1, -1): - possibleEntityName = "".join(charStack[:entityLength]) - if possibleEntityName in entities: - entityName = possibleEntityName - break + try: + entityName = entitiesTrie.longest_prefix("".join(charStack[:-1])) + entityLength = len(entityName) + except KeyError: + entityName = None if entityName is not None: if entityName[-1] != ";": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "named-entity-without-semicolon"}) + "named-entity-without-semicolon"}) if (entityName[-1] != ";" and fromAttribute and (charStack[entityLength] in asciiLetters or charStack[entityLength] in digits or - charStack[entityLength] == "=")): + charStack[entityLength] == "=")): self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) else: output = entities[entityName] self.stream.unget(charStack.pop()) - output += u"".join(charStack[entityLength:]) + output += "".join(charStack[entityLength:]) else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-named-entity"}) + "expected-named-entity"}) self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) if fromAttribute: self.currentToken["data"][-1][1] += output @@ -246,28 +239,26 @@ class HTMLTokenizer(object): token["name"] = token["name"].translate(asciiUpper2Lower) if token["type"] == tokenTypes["EndTag"]: if token["data"]: - self.tokenQueue.append({"type":tokenTypes["ParseError"], - "data":"attributes-in-end-tag"}) + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "attributes-in-end-tag"}) if token["selfClosing"]: - self.tokenQueue.append({"type":tokenTypes["ParseError"], - "data":"self-closing-flag-on-end-tag"}) + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "self-closing-flag-on-end-tag"}) self.tokenQueue.append(token) self.state = self.dataState - # Below are the various tokenizer states worked out. - def dataState(self): data = self.stream.char() if data == "&": self.state = self.entityDataState elif data == "<": self.state = self.tagOpenState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], - "data":"invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\u0000"}) + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "invalid-codepoint"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\u0000"}) elif data is EOF: # Tokenization ends. return False @@ -276,21 +267,21 @@ class HTMLTokenizer(object): # state". At that point spaceCharacters are important so they are # emitted separately. self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data": - data + self.stream.charsUntil(spaceCharacters, True)}) + data + self.stream.charsUntil(spaceCharacters, True)}) # No need to update lastFourChars here, since the first space will # have already been appended to lastFourChars and will have broken # any <!-- or --> sequences else: - chars = self.stream.charsUntil((u"&", u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("&", "<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True def entityDataState(self): self.consumeEntity() self.state = self.dataState return True - + def rcdataState(self): data = self.stream.char() if data == "&": @@ -300,113 +291,113 @@ class HTMLTokenizer(object): elif data == EOF: # Tokenization ends. return False - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data in spaceCharacters: # Directly after emitting a token you switch back to the "data # state". At that point spaceCharacters are important so they are # emitted separately. self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data": - data + self.stream.charsUntil(spaceCharacters, True)}) + data + self.stream.charsUntil(spaceCharacters, True)}) # No need to update lastFourChars here, since the first space will # have already been appended to lastFourChars and will have broken # any <!-- or --> sequences else: - chars = self.stream.charsUntil((u"&", u"<")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("&", "<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True def characterReferenceInRcdata(self): self.consumeEntity() self.state = self.rcdataState return True - + def rawtextState(self): data = self.stream.char() if data == "<": self.state = self.rawtextLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: # Tokenization ends. return False else: - chars = self.stream.charsUntil((u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def scriptDataState(self): data = self.stream.char() if data == "<": self.state = self.scriptDataLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: # Tokenization ends. return False else: - chars = self.stream.charsUntil((u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def plaintextState(self): data = self.stream.char() if data == EOF: # Tokenization ends. return False - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + self.stream.charsUntil(u"\u0000")}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + self.stream.charsUntil("\u0000")}) return True def tagOpenState(self): data = self.stream.char() - if data == u"!": + if data == "!": self.state = self.markupDeclarationOpenState - elif data == u"/": + elif data == "/": self.state = self.closeTagOpenState elif data in asciiLetters: - self.currentToken = {"type": tokenTypes["StartTag"], + self.currentToken = {"type": tokenTypes["StartTag"], "name": data, "data": [], "selfClosing": False, "selfClosingAcknowledged": False} self.state = self.tagNameState - elif data == u">": + elif data == ">": # XXX In theory it could be something besides a tag name. But # do we really care? self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name-but-got-right-bracket"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<>"}) + "expected-tag-name-but-got-right-bracket"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<>"}) self.state = self.dataState - elif data == u"?": + elif data == "?": # XXX In theory it could be something besides a tag name. But # do we really care? self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name-but-got-question-mark"}) + "expected-tag-name-but-got-question-mark"}) self.stream.unget(data) self.state = self.bogusCommentState else: # XXX self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + "expected-tag-name"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.dataState return True @@ -415,22 +406,22 @@ class HTMLTokenizer(object): data = self.stream.char() if data in asciiLetters: self.currentToken = {"type": tokenTypes["EndTag"], "name": data, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.tagNameState - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-right-bracket"}) + "expected-closing-tag-but-got-right-bracket"}) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-eof"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + "expected-closing-tag-but-got-eof"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.state = self.dataState else: # XXX data can be _'_... self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-char", - "datavars": {"data": data}}) + "expected-closing-tag-but-got-char", + "datavars": {"data": data}}) self.stream.unget(data) self.state = self.bogusCommentState return True @@ -439,229 +430,229 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-tag-name"}) + "eof-in-tag-name"}) self.state = self.dataState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] += u"\uFFFD" + self.currentToken["name"] += "\uFFFD" else: self.currentToken["name"] += data # (Don't use charsUntil here, because tag names are # very short and it's faster to not do anything fancy) return True - + def rcdataLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.rcdataEndTagOpenState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.rcdataState return True - + def rcdataEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.rcdataEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.rcdataState return True - + def rcdataEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.rcdataState return True - + def rawtextLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.rawtextEndTagOpenState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.rawtextState return True - + def rawtextEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.rawtextEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.rawtextState return True - + def rawtextEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.rawtextState return True - + def scriptDataLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.scriptDataEndTagOpenState elif data == "!": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<!"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<!"}) self.state = self.scriptDataEscapeStartState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.scriptDataEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapeStartState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapeStartDashState else: self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapeStartDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashDashState else: self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapedState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashState elif data == "<": self.state = self.scriptDataEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: self.state = self.dataState else: - chars = self.stream.charsUntil((u"<", u"-", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "-", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def scriptDataEscapedDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashDashState elif data == "<": self.state = self.scriptDataEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataEscapedState elif data == EOF: self.state = self.dataState @@ -669,21 +660,21 @@ class HTMLTokenizer(object): self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedDashDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) elif data == "<": self.state = self.scriptDataEscapedLessThanSignState elif data == ">": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u">"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"}) self.state = self.scriptDataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataEscapedState elif data == EOF: self.state = self.dataState @@ -691,61 +682,61 @@ class HTMLTokenizer(object): self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.scriptDataEscapedEndTagOpenState elif data in asciiLetters: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<" + data}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<" + data}) self.temporaryBuffer = data self.state = self.scriptDataDoubleEscapeStartState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer = data self.state = self.scriptDataEscapedEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataDoubleEscapeStartState(self): data = self.stream.char() if data in (spaceCharacters | frozenset(("/", ">"))): @@ -761,87 +752,87 @@ class HTMLTokenizer(object): self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataDoubleEscapedState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataDoubleEscapedDashState elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) return True - + def scriptDataDoubleEscapedDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataDoubleEscapedDashDashState elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataDoubleEscapedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataDoubleEscapedState return True - - def scriptDataDoubleEscapedDashState(self): + + def scriptDataDoubleEscapedDashDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState elif data == ">": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u">"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"}) self.state = self.scriptDataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataDoubleEscapedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataDoubleEscapedState return True - + def scriptDataDoubleEscapedLessThanSignState(self): data = self.stream.char() if data == "/": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"/"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "/"}) self.temporaryBuffer = "" self.state = self.scriptDataDoubleEscapeEndState else: self.stream.unget(data) self.state = self.scriptDataDoubleEscapedState return True - + def scriptDataDoubleEscapeEndState(self): data = self.stream.char() if data in (spaceCharacters | frozenset(("/", ">"))): @@ -865,23 +856,23 @@ class HTMLTokenizer(object): elif data in asciiLetters: self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data in (u"'", u'"', u"=", u"<"): + elif data in ("'", '"', "=", "<"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-in-attribute-name"}) + "invalid-character-in-attribute-name"}) self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"].append([u"\uFFFD", ""]) + self.currentToken["data"].append(["\uFFFD", ""]) self.state = self.attributeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-name-but-got-eof"}) + "expected-attribute-name-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"].append([data, ""]) @@ -892,34 +883,34 @@ class HTMLTokenizer(object): data = self.stream.char() leavingThisState = True emitToken = False - if data == u"=": + if data == "=": self.state = self.beforeAttributeValueState elif data in asciiLetters: self.currentToken["data"][-1][0] += data +\ - self.stream.charsUntil(asciiLetters, True) + self.stream.charsUntil(asciiLetters, True) leavingThisState = False - elif data == u">": + elif data == ">": # XXX If we emit here the attributes are converted to a dict # without being checked and when the code below runs we error # because data is a dict not a list emitToken = True elif data in spaceCharacters: self.state = self.afterAttributeNameState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][0] += u"\uFFFD" + self.currentToken["data"][-1][0] += "\uFFFD" leavingThisState = False - elif data in (u"'", u'"', u"<"): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data in ("'", '"', "<"): + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-in-attribute-name"}) + "invalid-character-in-attribute-name"}) self.currentToken["data"][-1][0] += data leavingThisState = False elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "eof-in-attribute-name"}) self.state = self.dataState else: @@ -936,7 +927,7 @@ class HTMLTokenizer(object): for name, value in self.currentToken["data"][:-1]: if self.currentToken["data"][-1][0] == name: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "duplicate-attribute"}) + "duplicate-attribute"}) break # XXX Fix for above XXX if emitToken: @@ -947,28 +938,28 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.stream.charsUntil(spaceCharacters, True) - elif data == u"=": + elif data == "=": self.state = self.beforeAttributeValueState - elif data == u">": + elif data == ">": self.emitCurrentToken() elif data in asciiLetters: self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"].append([u"\uFFFD", ""]) + self.currentToken["data"].append(["\uFFFD", ""]) self.state = self.attributeNameState - elif data in (u"'", u'"', u"<"): + elif data in ("'", '"', "<"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-after-attribute-name"}) + "invalid-character-after-attribute-name"}) self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-end-of-tag-but-got-eof"}) + "expected-end-of-tag-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"].append([data, ""]) @@ -979,30 +970,30 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.stream.charsUntil(spaceCharacters, True) - elif data == u"\"": + elif data == "\"": self.state = self.attributeValueDoubleQuotedState - elif data == u"&": + elif data == "&": self.state = self.attributeValueUnQuotedState - self.stream.unget(data); - elif data == u"'": + self.stream.unget(data) + elif data == "'": self.state = self.attributeValueSingleQuotedState - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-value-but-got-right-bracket"}) + "expected-attribute-value-but-got-right-bracket"}) self.emitCurrentToken() - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" self.state = self.attributeValueUnQuotedState - elif data in (u"=", u"<", u"`"): + elif data in ("=", "<", "`"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "equals-in-unquoted-attribute-value"}) + "equals-in-unquoted-attribute-value"}) self.currentToken["data"][-1][1] += data self.state = self.attributeValueUnQuotedState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-value-but-got-eof"}) + "expected-attribute-value-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data @@ -1013,81 +1004,81 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterAttributeValueState - elif data == u"&": - self.processEntityInAttribute(u'"') - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "&": + self.processEntityInAttribute('"') + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-double-quote"}) + "eof-in-attribute-value-double-quote"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data +\ - self.stream.charsUntil(("\"", u"&")) + self.stream.charsUntil(("\"", "&", "\u0000")) return True def attributeValueSingleQuotedState(self): data = self.stream.char() if data == "'": self.state = self.afterAttributeValueState - elif data == u"&": - self.processEntityInAttribute(u"'") - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "&": + self.processEntityInAttribute("'") + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-single-quote"}) + "eof-in-attribute-value-single-quote"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data +\ - self.stream.charsUntil(("'", u"&")) + self.stream.charsUntil(("'", "&", "\u0000")) return True def attributeValueUnQuotedState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u"&": + elif data == "&": self.processEntityInAttribute(">") - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data in (u'"', u"'", u"=", u"<", u"`"): + elif data in ('"', "'", "=", "<", "`"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-in-unquoted-attribute-value"}) + "unexpected-character-in-unquoted-attribute-value"}) self.currentToken["data"][-1][1] += data - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-no-quotes"}) + "eof-in-attribute-value-no-quotes"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data + self.stream.charsUntil( - frozenset((u"&", u">", u'"', u"'", u"=", u"<", u"`")) | spaceCharacters) + frozenset(("&", ">", '"', "'", "=", "<", "`", "\u0000")) | spaceCharacters) return True def afterAttributeValueState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-EOF-after-attribute-value"}) + "unexpected-EOF-after-attribute-value"}) self.stream.unget(data) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-after-attribute-value"}) + "unexpected-character-after-attribute-value"}) self.stream.unget(data) self.state = self.beforeAttributeNameState return True @@ -1098,14 +1089,14 @@ class HTMLTokenizer(object): self.currentToken["selfClosing"] = True self.emitCurrentToken() elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-EOF-after-solidus-in-tag"}) + "unexpected-EOF-after-solidus-in-tag"}) self.stream.unget(data) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-after-soldius-in-tag"}) + "unexpected-character-after-solidus-in-tag"}) self.stream.unget(data) self.state = self.beforeAttributeNameState return True @@ -1114,10 +1105,10 @@ class HTMLTokenizer(object): # Make a new comment token and give it as value all the characters # until the first > or EOF (charsUntil checks for EOF automatically) # and emit it. - data = self.stream.charsUntil(u">") - data = data.replace(u"\u0000", u"\uFFFD") + data = self.stream.charsUntil(">") + data = data.replace("\u0000", "\uFFFD") self.tokenQueue.append( - {"type": tokenTypes["Comment"], "data": data}) + {"type": tokenTypes["Comment"], "data": data}) # Eat the character directly after the bogus comment which is either a # ">" or an EOF. @@ -1127,28 +1118,28 @@ class HTMLTokenizer(object): def markupDeclarationOpenState(self): charStack = [self.stream.char()] - if charStack[-1] == u"-": + if charStack[-1] == "-": charStack.append(self.stream.char()) - if charStack[-1] == u"-": - self.currentToken = {"type": tokenTypes["Comment"], "data": u""} + if charStack[-1] == "-": + self.currentToken = {"type": tokenTypes["Comment"], "data": ""} self.state = self.commentStartState return True - elif charStack[-1] in (u'd', u'D'): + elif charStack[-1] in ('d', 'D'): matched = True - for expected in ((u'o', u'O'), (u'c', u'C'), (u't', u'T'), - (u'y', u'Y'), (u'p', u'P'), (u'e', u'E')): + for expected in (('o', 'O'), ('c', 'C'), ('t', 'T'), + ('y', 'Y'), ('p', 'P'), ('e', 'E')): charStack.append(self.stream.char()) if charStack[-1] not in expected: matched = False break if matched: self.currentToken = {"type": tokenTypes["Doctype"], - "name": u"", - "publicId": None, "systemId": None, + "name": "", + "publicId": None, "systemId": None, "correct": True} self.state = self.doctypeState return True - elif (charStack[-1] == "[" and + elif (charStack[-1] == "[" and self.parser is not None and self.parser.tree.openElements and self.parser.tree.openElements[-1].namespace != self.parser.tree.defaultNamespace): @@ -1163,7 +1154,7 @@ class HTMLTokenizer(object): return True self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-dashes-or-doctype"}) + "expected-dashes-or-doctype"}) while charStack: self.stream.unget(charStack.pop()) @@ -1174,41 +1165,41 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "-": self.state = self.commentStartDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"\uFFFD" + self.currentToken["data"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "incorrect-comment"}) + "incorrect-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment"}) + "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.currentToken["data"] += data self.state = self.commentState return True - + def commentStartDashState(self): data = self.stream.char() if data == "-": self.state = self.commentEndState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"-\uFFFD" + self.currentToken["data"] += "-\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "incorrect-comment"}) + "incorrect-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment"}) + "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: @@ -1216,95 +1207,94 @@ class HTMLTokenizer(object): self.state = self.commentState return True - def commentState(self): data = self.stream.char() - if data == u"-": + if data == "-": self.state = self.commentEndDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"\uFFFD" + self.currentToken["data"] += "\uFFFD" elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.currentToken["data"] += data + \ - self.stream.charsUntil((u"-", u"\u0000")) + self.stream.charsUntil(("-", "\u0000")) return True def commentEndDashState(self): data = self.stream.char() - if data == u"-": + if data == "-": self.state = self.commentEndState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"-\uFFFD" + self.currentToken["data"] += "-\uFFFD" self.state = self.commentState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-end-dash"}) + "eof-in-comment-end-dash"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - self.currentToken["data"] += u"-" + data + self.currentToken["data"] += "-" + data self.state = self.commentState return True def commentEndState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"--\uFFFD" + self.currentToken["data"] += "--\uFFFD" self.state = self.commentState elif data == "!": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-bang-after-double-dash-in-comment"}) + "unexpected-bang-after-double-dash-in-comment"}) self.state = self.commentEndBangState - elif data == u"-": + elif data == "-": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-dash-after-double-dash-in-comment"}) + "unexpected-dash-after-double-dash-in-comment"}) self.currentToken["data"] += data elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-double-dash"}) + "eof-in-comment-double-dash"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: # XXX self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-comment"}) - self.currentToken["data"] += u"--" + data + "unexpected-char-in-comment"}) + self.currentToken["data"] += "--" + data self.state = self.commentState return True def commentEndBangState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"-": + elif data == "-": self.currentToken["data"] += "--!" self.state = self.commentEndDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"--!\uFFFD" + self.currentToken["data"] += "--!\uFFFD" self.state = self.commentState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-end-bang-state"}) + "eof-in-comment-end-bang-state"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - self.currentToken["data"] += u"--!" + data + self.currentToken["data"] += "--!" + data self.state = self.commentState return True @@ -1314,13 +1304,13 @@ class HTMLTokenizer(object): self.state = self.beforeDoctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-eof"}) + "expected-doctype-name-but-got-eof"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "need-space-after-doctype"}) + "need-space-after-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypeNameState return True @@ -1329,20 +1319,20 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: pass - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-right-bracket"}) + "expected-doctype-name-but-got-right-bracket"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] = u"\uFFFD" + self.currentToken["name"] = "\uFFFD" self.state = self.doctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-eof"}) + "expected-doctype-name-but-got-eof"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1356,18 +1346,18 @@ class HTMLTokenizer(object): if data in spaceCharacters: self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.state = self.afterDoctypeNameState - elif data == u">": + elif data == ">": self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] += u"\uFFFD" + self.currentToken["name"] += "\uFFFD" self.state = self.doctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype-name"}) + "eof-in-doctype-name"}) self.currentToken["correct"] = False self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.tokenQueue.append(self.currentToken) @@ -1380,21 +1370,21 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: pass - elif data == u">": + elif data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.currentToken["correct"] = False self.stream.unget(data) self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - if data in (u"p", u"P"): + if data in ("p", "P"): matched = True - for expected in ((u"u", u"U"), (u"b", u"B"), (u"l", u"L"), - (u"i", u"I"), (u"c", u"C")): + for expected in (("u", "U"), ("b", "B"), ("l", "L"), + ("i", "I"), ("c", "C")): data = self.stream.char() if data not in expected: matched = False @@ -1402,10 +1392,10 @@ class HTMLTokenizer(object): if matched: self.state = self.afterDoctypePublicKeywordState return True - elif data in (u"s", u"S"): + elif data in ("s", "S"): matched = True - for expected in ((u"y", u"Y"), (u"s", u"S"), (u"t", u"T"), - (u"e", u"E"), (u"m", u"M")): + for expected in (("y", "Y"), ("s", "S"), ("t", "T"), + ("e", "E"), ("m", "M")): data = self.stream.char() if data not in expected: matched = False @@ -1420,25 +1410,25 @@ class HTMLTokenizer(object): # and needs to be ungetted self.stream.unget(data) self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-space-or-right-bracket-in-doctype", "datavars": - {"data": data}}) + "expected-space-or-right-bracket-in-doctype", "datavars": + {"data": data}}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def afterDoctypePublicKeywordState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeDoctypePublicIdentifierState elif data in ("'", '"'): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypePublicIdentifierState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1452,26 +1442,26 @@ class HTMLTokenizer(object): if data in spaceCharacters: pass elif data == "\"": - self.currentToken["publicId"] = u"" + self.currentToken["publicId"] = "" self.state = self.doctypePublicIdentifierDoubleQuotedState elif data == "'": - self.currentToken["publicId"] = u"" + self.currentToken["publicId"] = "" self.state = self.doctypePublicIdentifierSingleQuotedState elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True @@ -1480,19 +1470,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterDoctypePublicIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["publicId"] += u"\uFFFD" + self.currentToken["publicId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1504,19 +1494,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "'": self.state = self.afterDoctypePublicIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["publicId"] += u"\uFFFD" + self.currentToken["publicId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1533,27 +1523,27 @@ class HTMLTokenizer(object): self.state = self.dataState elif data == '"': self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) - self.currentToken["systemId"] = u"" + "unexpected-char-in-doctype"}) + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) - self.currentToken["systemId"] = u"" + "unexpected-char-in-doctype"}) + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def betweenDoctypePublicAndSystemIdentifiersState(self): data = self.stream.char() if data in spaceCharacters: @@ -1562,36 +1552,36 @@ class HTMLTokenizer(object): self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data == '"': - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def afterDoctypeSystemKeywordState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeDoctypeSystemIdentifierState elif data in ("'", '"'): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypeSystemIdentifierState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1599,32 +1589,32 @@ class HTMLTokenizer(object): self.stream.unget(data) self.state = self.beforeDoctypeSystemIdentifierState return True - + def beforeDoctypeSystemIdentifierState(self): data = self.stream.char() if data in spaceCharacters: pass elif data == "\"": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True @@ -1633,19 +1623,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterDoctypeSystemIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["systemId"] += u"\uFFFD" + self.currentToken["systemId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1657,19 +1647,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "'": self.state = self.afterDoctypeSystemIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["systemId"] += u"\uFFFD" + self.currentToken["systemId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1686,19 +1676,19 @@ class HTMLTokenizer(object): self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.state = self.bogusDoctypeState return True def bogusDoctypeState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: @@ -1713,32 +1703,29 @@ class HTMLTokenizer(object): def cdataSectionState(self): data = [] while True: - data.append(self.stream.charsUntil(u"]")) - charStack = [] - - for expected in ["]", "]", ">"]: - charStack.append(self.stream.char()) - matched = True - if charStack[-1] == EOF: - data.extend(charStack[:-1]) - break - elif charStack[-1] != expected: - matched = False - data.extend(charStack) - break - - if matched: + data.append(self.stream.charsUntil("]")) + data.append(self.stream.charsUntil(">")) + char = self.stream.char() + if char == EOF: break + else: + assert char == ">" + if data[-1][-2:] == "]]": + data[-1] = data[-1][:-2] + break + else: + data.append(char) + data = "".join(data) - #Deal with null here rather than in the parser - nullCount = data.count(u"\u0000") + # Deal with null here rather than in the parser + nullCount = data.count("\u0000") if nullCount > 0: - for i in xrange(nullCount): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + for i in range(nullCount): + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - data = data.replace(u"\u0000", u"\uFFFD") + data = data.replace("\u0000", "\uFFFD") if data: - self.tokenQueue.append({"type": tokenTypes["Characters"], + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.dataState return True diff --git a/libs/html5lib/treeadapters/__init__.py b/libs/html5lib/treeadapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/html5lib/treeadapters/sax.py b/libs/html5lib/treeadapters/sax.py new file mode 100644 index 00000000..ad47df95 --- /dev/null +++ b/libs/html5lib/treeadapters/sax.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import, division, unicode_literals + +from xml.sax.xmlreader import AttributesNSImpl + +from ..constants import adjustForeignAttributes, unadjustForeignAttributes + +prefix_mapping = {} +for prefix, localName, namespace in adjustForeignAttributes.values(): + if prefix is not None: + prefix_mapping[prefix] = namespace + + +def to_sax(walker, handler): + """Call SAX-like content handler based on treewalker walker""" + handler.startDocument() + for prefix, namespace in prefix_mapping.items(): + handler.startPrefixMapping(prefix, namespace) + + for token in walker: + type = token["type"] + if type == "Doctype": + continue + elif type in ("StartTag", "EmptyTag"): + attrs = AttributesNSImpl(token["data"], + unadjustForeignAttributes) + handler.startElementNS((token["namespace"], token["name"]), + token["name"], + attrs) + if type == "EmptyTag": + handler.endElementNS((token["namespace"], token["name"]), + token["name"]) + elif type == "EndTag": + handler.endElementNS((token["namespace"], token["name"]), + token["name"]) + elif type in ("Characters", "SpaceCharacters"): + handler.characters(token["data"]) + elif type == "Comment": + pass + else: + assert False, "Unknown token type" + + for prefix, namespace in prefix_mapping.items(): + handler.endPrefixMapping(prefix) + handler.endDocument() diff --git a/libs/html5lib/treebuilders/__init__.py b/libs/html5lib/treebuilders/__init__.py index 14f66d40..6a6b2a4c 100755 --- a/libs/html5lib/treebuilders/__init__.py +++ b/libs/html5lib/treebuilders/__init__.py @@ -7,7 +7,7 @@ implement several things: 1) A set of classes for various types of elements: Document, Doctype, Comment, Element. These must implement the interface of _base.treebuilders.Node (although comment nodes have a different -signature for their constructor, see treebuilders.simpletree.Comment) +signature for their constructor, see treebuilders.etree.Comment) Textual content may also be implemented as another node type, or not, as your tree implementation requires. @@ -24,73 +24,53 @@ getDocument - Returns the root node of the complete document tree testSerializer method on your treebuilder which accepts a node and returns a string containing Node and its children serialized according to the format used in the unittests - -The supplied simpletree module provides a python-only implementation -of a full treebuilder and is a useful reference for the semantics of -the various methods. """ +from __future__ import absolute_import, division, unicode_literals + +from ..utils import default_etree + treeBuilderCache = {} -import sys def getTreeBuilder(treeType, implementation=None, **kwargs): """Get a TreeBuilder class for various types of tree with built-in support - + treeType - the name of the tree type required (case-insensitive). Supported - values are "simpletree", "dom", "etree" and "beautifulsoup" - - "simpletree" - a built-in DOM-ish tree type with support for some - more pythonic idioms. - "dom" - A generic builder for DOM implementations, defaulting to - a xml.dom.minidom based implementation for the sake of - backwards compatibility (as releases up until 0.10 had a - builder called "dom" that was a minidom implemenation). - "etree" - A generic builder for tree implementations exposing an - elementtree-like interface (known to work with - ElementTree, cElementTree and lxml.etree). - "beautifulsoup" - Beautiful soup (if installed) - + values are: + + "dom" - A generic builder for DOM implementations, defaulting to + a xml.dom.minidom based implementation. + "etree" - A generic builder for tree implementations exposing an + ElementTree-like interface, defaulting to + xml.etree.cElementTree if available and + xml.etree.ElementTree if not. + "lxml" - A etree-based builder for lxml.etree, handling + limitations of lxml's implementation. + implementation - (Currently applies to the "etree" and "dom" tree types). A module implementing the tree type e.g. - xml.etree.ElementTree or lxml.etree.""" - + xml.etree.ElementTree or xml.etree.cElementTree.""" + treeType = treeType.lower() if treeType not in treeBuilderCache: if treeType == "dom": - import dom - # XXX: Keep backwards compatibility by using minidom if no implementation is given - if implementation == None: + from . import dom + # Come up with a sane default (pref. from the stdlib) + if implementation is None: from xml.dom import minidom implementation = minidom - # XXX: NEVER cache here, caching is done in the dom submodule + # NEVER cache here, caching is done in the dom submodule return dom.getDomModule(implementation, **kwargs).TreeBuilder - elif treeType == "simpletree": - import simpletree - treeBuilderCache[treeType] = simpletree.TreeBuilder - elif treeType == "beautifulsoup": - import soup - treeBuilderCache[treeType] = soup.TreeBuilder elif treeType == "lxml": - import etree_lxml + from . import etree_lxml treeBuilderCache[treeType] = etree_lxml.TreeBuilder elif treeType == "etree": - # Come up with a sane default - if implementation == None: - try: - import xml.etree.cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except ImportError: - try: - import cElementTree as ET - except ImportError: - import elementtree.ElementTree as ET - implementation = ET - import etree + from . import etree + if implementation is None: + implementation = default_etree # NEVER cache here, caching is done in the etree submodule return etree.getETreeModule(implementation, **kwargs).TreeBuilder else: - raise ValueError("""Unrecognised treebuilder "%s" """%treeType) + raise ValueError("""Unrecognised treebuilder "%s" """ % treeType) return treeBuilderCache.get(treeType) diff --git a/libs/html5lib/treebuilders/_base.py b/libs/html5lib/treebuilders/_base.py index f3782d28..8b97cc11 100755 --- a/libs/html5lib/treebuilders/_base.py +++ b/libs/html5lib/treebuilders/_base.py @@ -1,25 +1,34 @@ -from html5lib.constants import scopingElements, tableInsertModeElements, namespaces -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from ..constants import scopingElements, tableInsertModeElements, namespaces # The scope markers are inserted when entering object elements, # marquees, table cells, and table captions, and are used to prevent formatting # from "leaking" into tables, object elements, and marquees. Marker = None +listElementsMap = { + None: (frozenset(scopingElements), False), + "button": (frozenset(scopingElements | set([(namespaces["html"], "button")])), False), + "list": (frozenset(scopingElements | set([(namespaces["html"], "ol"), + (namespaces["html"], "ul")])), False), + "table": (frozenset([(namespaces["html"], "html"), + (namespaces["html"], "table")]), False), + "select": (frozenset([(namespaces["html"], "optgroup"), + (namespaces["html"], "option")]), True) +} + + class Node(object): def __init__(self, name): """Node representing an item in the tree. name - The tag name associated with the node parent - The parent of the current node (or None for the document node) - value - The value of the current node (applies to text nodes and + value - The value of the current node (applies to text nodes and comments attributes - a dict holding name, value pairs for attributes of the node - childNodes - a list of child nodes of the current node. This must + childNodes - a list of child nodes of the current node. This must include all elements but not necessarily other node types _flags - A list of miscellaneous flags that can be set on the node """ @@ -30,14 +39,14 @@ class Node(object): self.childNodes = [] self._flags = [] - def __unicode__(self): - attributesStr = " ".join(["%s=\"%s\""%(name, value) - for name, value in - self.attributes.iteritems()]) + def __str__(self): + attributesStr = " ".join(["%s=\"%s\"" % (name, value) + for name, value in + self.attributes.items()]) if attributesStr: - return "<%s %s>"%(self.name,attributesStr) + return "<%s %s>" % (self.name, attributesStr) else: - return "<%s>"%(self.name) + return "<%s>" % (self.name) def __repr__(self): return "<%s>" % (self.name) @@ -48,14 +57,14 @@ class Node(object): raise NotImplementedError def insertText(self, data, insertBefore=None): - """Insert data as text in the current node, positioned before the + """Insert data as text in the current node, positioned before the start of node insertBefore or to the end of the node's text. """ raise NotImplementedError def insertBefore(self, node, refNode): - """Insert node as a child of the current node, before refNode in the - list of child nodes. Raises ValueError if refNode is not a child of + """Insert node as a child of the current node, before refNode in the + list of child nodes. Raises ValueError if refNode is not a child of the current node""" raise NotImplementedError @@ -65,11 +74,11 @@ class Node(object): raise NotImplementedError def reparentChildren(self, newParent): - """Move all the children of the current node to newParent. - This is needed so that trees that don't store text as nodes move the + """Move all the children of the current node to newParent. + This is needed so that trees that don't store text as nodes move the text in the correct way """ - #XXX - should this method be made more general? + # XXX - should this method be made more general? for child in self.childNodes: newParent.appendChild(child) self.childNodes = [] @@ -80,12 +89,12 @@ class Node(object): """ raise NotImplementedError - def hasContent(self): """Return true if the node has children or text, false otherwise """ raise NotImplementedError + class ActiveFormattingElements(list): def append(self, node): equalCount = 0 @@ -103,12 +112,13 @@ class ActiveFormattingElements(list): def nodesEqual(self, node1, node2): if not node1.nameTuple == node2.nameTuple: return False - + if not node1.attributes == node2.attributes: return False - + return True + class TreeBuilder(object): """Base treebuilder implementation documentClass - the class to use for the bottommost node of a document @@ -117,19 +127,19 @@ class TreeBuilder(object): doctypeClass - the class to use for doctypes """ - #Document class + # Document class documentClass = None - #The class to use for creating a node + # The class to use for creating a node elementClass = None - #The class to use for creating comments + # The class to use for creating comments commentClass = None - #The class to use for creating doctypes + # The class to use for creating doctypes doctypeClass = None - - #Fragment class + + # Fragment class fragmentClass = None def __init__(self, namespaceHTMLElements): @@ -138,12 +148,12 @@ class TreeBuilder(object): else: self.defaultNamespace = None self.reset() - + def reset(self): self.openElements = [] self.activeFormattingElements = ActiveFormattingElements() - #XXX - rename these to headElement, formElement + # XXX - rename these to headElement, formElement self.headPointer = None self.formPointer = None @@ -153,30 +163,20 @@ class TreeBuilder(object): def elementInScope(self, target, variant=None): - #If we pass a node in we match that. if we pass a string - #match any node with that name + # If we pass a node in we match that. if we pass a string + # match any node with that name exactNode = hasattr(target, "nameTuple") - listElementsMap = { - None:(scopingElements, False), - "button":(scopingElements | set([(namespaces["html"], "button")]), False), - "list":(scopingElements | set([(namespaces["html"], "ol"), - (namespaces["html"], "ul")]), False), - "table":(set([(namespaces["html"], "html"), - (namespaces["html"], "table")]), False), - "select":(set([(namespaces["html"], "optgroup"), - (namespaces["html"], "option")]), True) - } listElements, invert = listElementsMap[variant] for node in reversed(self.openElements): if (node.name == target and not exactNode or - node == target and exactNode): + node == target and exactNode): return True - elif (invert ^ (node.nameTuple in listElements)): + elif (invert ^ (node.nameTuple in listElements)): return False - assert False # We should never reach this point + assert False # We should never reach this point def reconstructActiveFormattingElements(self): # Within this algorithm the order of steps described in the @@ -196,7 +196,7 @@ class TreeBuilder(object): # Step 6 while entry != Marker and entry not in self.openElements: if i == 0: - #This will be reset to 0 below + # This will be reset to 0 below i = -1 break i -= 1 @@ -209,13 +209,13 @@ class TreeBuilder(object): # Step 8 entry = self.activeFormattingElements[i] - clone = entry.cloneNode() #Mainly to get a new copy of the attributes + clone = entry.cloneNode() # Mainly to get a new copy of the attributes # Step 9 - element = self.insertElement({"type":"StartTag", - "name":clone.name, - "namespace":clone.namespace, - "data":clone.attributes}) + element = self.insertElement({"type": "StartTag", + "name": clone.name, + "namespace": clone.namespace, + "data": clone.attributes}) # Step 10 self.activeFormattingElements[i] = element @@ -260,7 +260,7 @@ class TreeBuilder(object): if parent is None: parent = self.openElements[-1] parent.appendChild(self.commentClass(token["data"])) - + def createElement(self, token): """Create an element but don't insert it anywhere""" name = token["name"] @@ -282,10 +282,10 @@ class TreeBuilder(object): self.insertElement = self.insertElementNormal insertFromTable = property(_getInsertFromTable, _setInsertFromTable) - + def insertElementNormal(self, token): name = token["name"] - assert type(name) == unicode, "Element %s not unicode"%name + assert isinstance(name, text_type), "Element %s not unicode" % name namespace = token.get("namespace", self.defaultNamespace) element = self.elementClass(name, namespace) element.attributes = token["data"] @@ -294,13 +294,13 @@ class TreeBuilder(object): return element def insertElementTable(self, token): - """Create an element and insert it into the tree""" + """Create an element and insert it into the tree""" element = self.createElement(token) if self.openElements[-1].name not in tableInsertModeElements: return self.insertElementNormal(token) else: - #We should be in the InTable mode. This means we want to do - #special magic element rearranging + # We should be in the InTable mode. This means we want to do + # special magic element rearranging parent, insertBefore = self.getTableMisnestedNodePosition() if insertBefore is None: parent.appendChild(element) @@ -315,7 +315,7 @@ class TreeBuilder(object): parent = self.openElements[-1] if (not self.insertFromTable or (self.insertFromTable and - self.openElements[-1].name + self.openElements[-1].name not in tableInsertModeElements)): parent.insertText(data) else: @@ -323,14 +323,14 @@ class TreeBuilder(object): # special magic element rearranging parent, insertBefore = self.getTableMisnestedNodePosition() parent.insertText(data, insertBefore) - + def getTableMisnestedNodePosition(self): """Get the foster parent element, and sibling to insert before (or None) when inserting a misnested table node""" # The foster parent element is the one which comes before the most # recently opened table element # XXX - this is really inelegant - lastTable=None + lastTable = None fosterParent = None insertBefore = None for elm in self.openElements[::-1]: @@ -354,7 +354,7 @@ class TreeBuilder(object): name = self.openElements[-1].name # XXX td, th and tr are not actually needed if (name in frozenset(("dd", "dt", "li", "option", "optgroup", "p", "rp", "rt")) - and name != exclude): + and name != exclude): self.openElements.pop() # XXX This is not entirely what the specification says. We should # investigate it more closely. @@ -363,10 +363,10 @@ class TreeBuilder(object): def getDocument(self): "Return the final tree" return self.document - + def getFragment(self): "Return the final fragment" - #assert self.innerHTML + # assert self.innerHTML fragment = self.fragmentClass() self.openElements[0].reparentChildren(fragment) return fragment diff --git a/libs/html5lib/treebuilders/dom.py b/libs/html5lib/treebuilders/dom.py index 9578da2b..61e5ed79 100644 --- a/libs/html5lib/treebuilders/dom.py +++ b/libs/html5lib/treebuilders/dom.py @@ -1,45 +1,38 @@ +from __future__ import absolute_import, division, unicode_literals -from xml.dom import minidom, Node, XML_NAMESPACE, XMLNS_NAMESPACE -try: - from types import ModuleType -except: - from new import module as ModuleType -import re + +from xml.dom import minidom, Node import weakref -import _base -from html5lib import constants, ihatexml -from html5lib.constants import namespaces +from . import _base +from .. import constants +from ..constants import namespaces +from ..utils import moduleFactoryFactory -moduleCache = {} - -def getDomModule(DomImplementation): - name = "_" + DomImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType(name) - objs = getDomBuilder(DomImplementation) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getDomBuilder(DomImplementation): Dom = DomImplementation + class AttrList(object): def __init__(self, element): self.element = element + def __iter__(self): - return self.element.attributes.items().__iter__() + return list(self.element.attributes.items()).__iter__() + def __setitem__(self, name, value): self.element.setAttribute(name, value) + def __len__(self): - return len(self.element.attributes.items()) + return len(list(self.element.attributes.items())) + def items(self): return [(item[0], item[1]) for item in - self.element.attributes.items()] + list(self.element.attributes.items())] + def keys(self): - return self.element.attributes.keys() + return list(self.element.attributes.keys()) + def __getitem__(self, name): return self.element.getAttribute(name) @@ -48,68 +41,68 @@ def getDomBuilder(DomImplementation): raise NotImplementedError else: return self.element.hasAttribute(name) - + class NodeBuilder(_base.Node): def __init__(self, element): _base.Node.__init__(self, element.nodeName) self.element = element - namespace = property(lambda self:hasattr(self.element, "namespaceURI") + namespace = property(lambda self: hasattr(self.element, "namespaceURI") and self.element.namespaceURI or None) def appendChild(self, node): node.parent = self self.element.appendChild(node.element) - + def insertText(self, data, insertBefore=None): text = self.element.ownerDocument.createTextNode(data) if insertBefore: self.element.insertBefore(text, insertBefore.element) else: self.element.appendChild(text) - + def insertBefore(self, node, refNode): self.element.insertBefore(node.element, refNode.element) node.parent = self - + def removeChild(self, node): if node.element.parentNode == self.element: self.element.removeChild(node.element) node.parent = None - + def reparentChildren(self, newParent): while self.element.hasChildNodes(): child = self.element.firstChild self.element.removeChild(child) newParent.element.appendChild(child) self.childNodes = [] - + def getAttributes(self): return AttrList(self.element) - + def setAttributes(self, attributes): if attributes: - for name, value in attributes.items(): + for name, value in list(attributes.items()): if isinstance(name, tuple): if name[0] is not None: qualifiedName = (name[0] + ":" + name[1]) else: qualifiedName = name[1] - self.element.setAttributeNS(name[2], qualifiedName, + self.element.setAttributeNS(name[2], qualifiedName, value) else: self.element.setAttribute( name, value) attributes = property(getAttributes, setAttributes) - + def cloneNode(self): return NodeBuilder(self.element.cloneNode(False)) - + def hasContent(self): return self.element.hasChildNodes() def getNameTuple(self): - if self.namespace == None: + if self.namespace is None: return namespaces["html"], self.name else: return self.namespace, self.name @@ -118,9 +111,9 @@ def getDomBuilder(DomImplementation): class TreeBuilder(_base.TreeBuilder): def documentClass(self): - self.dom = Dom.getDOMImplementation().createDocument(None,None,None) + self.dom = Dom.getDOMImplementation().createDocument(None, None, None) return weakref.proxy(self) - + def insertDoctype(self, token): name = token["name"] publicId = token["publicId"] @@ -131,7 +124,7 @@ def getDomBuilder(DomImplementation): self.document.appendChild(NodeBuilder(doctype)) if Dom == minidom: doctype.ownerDocument = self.dom - + def elementClass(self, name, namespace=None): if namespace is None and self.defaultNamespace is None: node = self.dom.createElement(name) @@ -139,70 +132,72 @@ def getDomBuilder(DomImplementation): node = self.dom.createElementNS(namespace, name) return NodeBuilder(node) - + def commentClass(self, data): return NodeBuilder(self.dom.createComment(data)) - + def fragmentClass(self): return NodeBuilder(self.dom.createDocumentFragment()) - + def appendChild(self, node): self.dom.appendChild(node.element) - + def testSerializer(self, element): return testSerializer(element) - + def getDocument(self): return self.dom - + def getFragment(self): return _base.TreeBuilder.getFragment(self).element - + def insertText(self, data, parent=None): - data=data - if parent <> self: + data = data + if parent != self: _base.TreeBuilder.insertText(self, data, parent) else: # HACK: allow text nodes as children of the document node if hasattr(self.dom, '_child_node_types'): if not Node.TEXT_NODE in self.dom._child_node_types: - self.dom._child_node_types=list(self.dom._child_node_types) + self.dom._child_node_types = list(self.dom._child_node_types) self.dom._child_node_types.append(Node.TEXT_NODE) self.dom.appendChild(self.dom.createTextNode(data)) - + + implementation = DomImplementation name = None - + def testSerializer(element): element.normalize() rv = [] + def serializeElement(element, indent=0): if element.nodeType == Node.DOCUMENT_TYPE_NODE: if element.name: if element.publicId or element.systemId: publicId = element.publicId or "" systemId = element.systemId or "" - rv.append( """|%s<!DOCTYPE %s "%s" "%s">"""%( - ' '*indent, element.name, publicId, systemId)) + rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" % + (' ' * indent, element.name, publicId, systemId)) else: - rv.append("|%s<!DOCTYPE %s>"%(' '*indent, element.name)) + rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, element.name)) else: - rv.append("|%s<!DOCTYPE >"%(' '*indent,)) + rv.append("|%s<!DOCTYPE >" % (' ' * indent,)) elif element.nodeType == Node.DOCUMENT_NODE: rv.append("#document") elif element.nodeType == Node.DOCUMENT_FRAGMENT_NODE: rv.append("#document-fragment") elif element.nodeType == Node.COMMENT_NODE: - rv.append("|%s<!-- %s -->"%(' '*indent, element.nodeValue)) + rv.append("|%s<!-- %s -->" % (' ' * indent, element.nodeValue)) elif element.nodeType == Node.TEXT_NODE: - rv.append("|%s\"%s\"" %(' '*indent, element.nodeValue)) + rv.append("|%s\"%s\"" % (' ' * indent, element.nodeValue)) else: if (hasattr(element, "namespaceURI") and - element.namespaceURI != None): - name = "%s %s"%(constants.prefixes[element.namespaceURI], - element.nodeName) + element.namespaceURI is not None): + name = "%s %s" % (constants.prefixes[element.namespaceURI], + element.nodeName) else: name = element.nodeName - rv.append("|%s<%s>"%(' '*indent, name)) + rv.append("|%s<%s>" % (' ' * indent, name)) if element.hasAttributes(): attributes = [] for i in range(len(element.attributes)): @@ -211,81 +206,22 @@ def getDomBuilder(DomImplementation): value = attr.value ns = attr.namespaceURI if ns: - name = "%s %s"%(constants.prefixes[ns], attr.localName) + name = "%s %s" % (constants.prefixes[ns], attr.localName) else: name = attr.nodeName attributes.append((name, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) indent += 2 for child in element.childNodes: serializeElement(child, indent) serializeElement(element, 0) - + return "\n".join(rv) - - def dom2sax(node, handler, nsmap={'xml':XML_NAMESPACE}): - if node.nodeType == Node.ELEMENT_NODE: - if not nsmap: - handler.startElement(node.nodeName, node.attributes) - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endElement(node.nodeName) - else: - attributes = dict(node.attributes.itemsNS()) - - # gather namespace declarations - prefixes = [] - for attrname in node.attributes.keys(): - attr = node.getAttributeNode(attrname) - if (attr.namespaceURI == XMLNS_NAMESPACE or - (attr.namespaceURI == None and attr.nodeName.startswith('xmlns'))): - prefix = (attr.nodeName != 'xmlns' and attr.nodeName or None) - handler.startPrefixMapping(prefix, attr.nodeValue) - prefixes.append(prefix) - nsmap = nsmap.copy() - nsmap[prefix] = attr.nodeValue - del attributes[(attr.namespaceURI, attr.nodeName)] - - # apply namespace declarations - for attrname in node.attributes.keys(): - attr = node.getAttributeNode(attrname) - if attr.namespaceURI == None and ':' in attr.nodeName: - prefix = attr.nodeName.split(':')[0] - if nsmap.has_key(prefix): - del attributes[(attr.namespaceURI, attr.nodeName)] - attributes[(nsmap[prefix],attr.nodeName)]=attr.nodeValue - - # SAX events - ns = node.namespaceURI or nsmap.get(None,None) - handler.startElementNS((ns,node.nodeName), node.nodeName, attributes) - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endElementNS((ns, node.nodeName), node.nodeName) - for prefix in prefixes: handler.endPrefixMapping(prefix) - - elif node.nodeType in [Node.TEXT_NODE, Node.CDATA_SECTION_NODE]: - handler.characters(node.nodeValue) - - elif node.nodeType == Node.DOCUMENT_NODE: - handler.startDocument() - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endDocument() - - elif node.nodeType == Node.DOCUMENT_FRAGMENT_NODE: - for child in node.childNodes: dom2sax(child, handler, nsmap) - - else: - # ATTRIBUTE_NODE - # ENTITY_NODE - # PROCESSING_INSTRUCTION_NODE - # COMMENT_NODE - # DOCUMENT_TYPE_NODE - # NOTATION_NODE - pass - + return locals() -# Keep backwards compatibility with things that directly load -# classes/functions from this module -for key, value in getDomModule(minidom).__dict__.items(): - globals()[key] = value + +# The actual means to get a module! +getDomModule = moduleFactoryFactory(getDomBuilder) diff --git a/libs/html5lib/treebuilders/etree.py b/libs/html5lib/treebuilders/etree.py index 95be4755..2c8ed19f 100755 --- a/libs/html5lib/treebuilders/etree.py +++ b/libs/html5lib/treebuilders/etree.py @@ -1,32 +1,21 @@ -try: - from types import ModuleType -except: - from new import module as ModuleType -import re -import types +from __future__ import absolute_import, division, unicode_literals +from six import text_type -import _base -from html5lib import ihatexml -from html5lib import constants -from html5lib.constants import namespaces +import re + +from . import _base +from .. import ihatexml +from .. import constants +from ..constants import namespaces +from ..utils import moduleFactoryFactory tag_regexp = re.compile("{([^}]*)}(.*)") -moduleCache = {} - -def getETreeModule(ElementTreeImplementation, fullTree=False): - name = "_" + ElementTreeImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder") - objs = getETreeBuilder(ElementTreeImplementation, fullTree) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getETreeBuilder(ElementTreeImplementation, fullTree=False): ElementTree = ElementTreeImplementation + ElementTreeCommentType = ElementTree.Comment("asd").tag + class Element(_base.Node): def __init__(self, name, namespace=None): self._name = name @@ -45,16 +34,16 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): if namespace is None: etree_tag = name else: - etree_tag = "{%s}%s"%(namespace, name) + etree_tag = "{%s}%s" % (namespace, name) return etree_tag - + def _setName(self, name): self._name = name self._element.tag = self._getETreeTag(self._name, self._namespace) - + def _getName(self): return self._name - + name = property(_getName, _setName) def _setNamespace(self, namespace): @@ -65,81 +54,82 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): return self._namespace namespace = property(_getNamespace, _setNamespace) - + def _getAttributes(self): return self._element.attrib - + def _setAttributes(self, attributes): - #Delete existing attributes first - #XXX - there may be a better way to do this... - for key in self._element.attrib.keys(): + # Delete existing attributes first + # XXX - there may be a better way to do this... + for key in list(self._element.attrib.keys()): del self._element.attrib[key] - for key, value in attributes.iteritems(): + for key, value in attributes.items(): if isinstance(key, tuple): - name = "{%s}%s"%(key[2], key[1]) + name = "{%s}%s" % (key[2], key[1]) else: name = key self._element.set(name, value) - + attributes = property(_getAttributes, _setAttributes) - + def _getChildNodes(self): - return self._childNodes + return self._childNodes + def _setChildNodes(self, value): del self._element[:] self._childNodes = [] for element in value: self.insertChild(element) - + childNodes = property(_getChildNodes, _setChildNodes) - + def hasContent(self): """Return true if the node has children or text""" return bool(self._element.text or len(self._element)) - + def appendChild(self, node): self._childNodes.append(node) self._element.append(node._element) node.parent = self - + def insertBefore(self, node, refNode): index = list(self._element).index(refNode._element) self._element.insert(index, node._element) node.parent = self - + def removeChild(self, node): self._element.remove(node._element) - node.parent=None - + node.parent = None + def insertText(self, data, insertBefore=None): if not(len(self._element)): if not self._element.text: self._element.text = "" self._element.text += data elif insertBefore is None: - #Insert the text as the tail of the last child element + # Insert the text as the tail of the last child element if not self._element[-1].tail: self._element[-1].tail = "" self._element[-1].tail += data else: - #Insert the text before the specified node + # Insert the text before the specified node children = list(self._element) index = children.index(insertBefore._element) if index > 0: - if not self._element[index-1].tail: - self._element[index-1].tail = "" - self._element[index-1].tail += data + if not self._element[index - 1].tail: + self._element[index - 1].tail = "" + self._element[index - 1].tail += data else: if not self._element.text: self._element.text = "" self._element.text += data - + def cloneNode(self): element = type(self)(self.name, self.namespace) - for name, value in self.attributes.iteritems(): + for name, value in self.attributes.items(): element.attributes[name] = value return element - + def reparentChildren(self, newParent): if newParent.childNodes: newParent.childNodes[-1]._element.tail += self._element.text @@ -150,60 +140,60 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): newParent._element.text += self._element.text self._element.text = "" _base.Node.reparentChildren(self, newParent) - + class Comment(Element): def __init__(self, data): - #Use the superclass constructor to set all properties on the - #wrapper element + # Use the superclass constructor to set all properties on the + # wrapper element self._element = ElementTree.Comment(data) self.parent = None self._childNodes = [] self._flags = [] - + def _getData(self): return self._element.text - + def _setData(self, value): self._element.text = value - + data = property(_getData, _setData) - + class DocumentType(Element): def __init__(self, name, publicId, systemId): - Element.__init__(self, "<!DOCTYPE>") + Element.__init__(self, "<!DOCTYPE>") self._element.text = name self.publicId = publicId self.systemId = systemId def _getPublicId(self): - return self._element.get(u"publicId", "") + return self._element.get("publicId", "") def _setPublicId(self, value): if value is not None: - self._element.set(u"publicId", value) + self._element.set("publicId", value) publicId = property(_getPublicId, _setPublicId) - + def _getSystemId(self): - return self._element.get(u"systemId", "") + return self._element.get("systemId", "") def _setSystemId(self, value): if value is not None: - self._element.set(u"systemId", value) + self._element.set("systemId", value) systemId = property(_getSystemId, _setSystemId) - + class Document(Element): def __init__(self): - Element.__init__(self, "<DOCUMENT_ROOT>") - + Element.__init__(self, "DOCUMENT_ROOT") + class DocumentFragment(Element): def __init__(self): - Element.__init__(self, "<DOCUMENT_FRAGMENT>") - + Element.__init__(self, "DOCUMENT_FRAGMENT") + def testSerializer(element): rv = [] - finalText = None + def serializeElement(element, indent=0): if not(hasattr(element, "tag")): element = element.getroot() @@ -211,20 +201,23 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): if element.get("publicId") or element.get("systemId"): publicId = element.get("publicId") or "" systemId = element.get("systemId") or "" - rv.append( """<!DOCTYPE %s "%s" "%s">"""%( - element.text, publicId, systemId)) - else: - rv.append("<!DOCTYPE %s>"%(element.text,)) - elif element.tag == "<DOCUMENT_ROOT>": + rv.append("""<!DOCTYPE %s "%s" "%s">""" % + (element.text, publicId, systemId)) + else: + rv.append("<!DOCTYPE %s>" % (element.text,)) + elif element.tag == "DOCUMENT_ROOT": rv.append("#document") - if element.text: - rv.append("|%s\"%s\""%(' '*(indent+2), element.text)) - if element.tail: - finalText = element.tail - elif element.tag == ElementTree.Comment: - rv.append("|%s<!-- %s -->"%(' '*indent, element.text)) + if element.text is not None: + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) + if element.tail is not None: + raise TypeError("Document node cannot have tail") + if hasattr(element, "attrib") and len(element.attrib): + raise TypeError("Document node cannot have attributes") + elif element.tag == ElementTreeCommentType: + rv.append("|%s<!-- %s -->" % (' ' * indent, element.text)) else: - assert type(element.tag) in types.StringTypes, "Expected unicode, got %s"%type(element.tag) + assert isinstance(element.tag, text_type), \ + "Expected unicode, got %s, %s" % (type(element.tag), element.tag) nsmatch = tag_regexp.match(element.tag) if nsmatch is None: @@ -232,113 +225,113 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): else: ns, name = nsmatch.groups() prefix = constants.prefixes[ns] - name = "%s %s"%(prefix, name) - rv.append("|%s<%s>"%(' '*indent, name)) + name = "%s %s" % (prefix, name) + rv.append("|%s<%s>" % (' ' * indent, name)) if hasattr(element, "attrib"): attributes = [] - for name, value in element.attrib.iteritems(): + for name, value in element.attrib.items(): nsmatch = tag_regexp.match(name) if nsmatch is not None: ns, name = nsmatch.groups() prefix = constants.prefixes[ns] - attr_string = "%s %s"%(prefix, name) + attr_string = "%s %s" % (prefix, name) else: attr_string = name attributes.append((attr_string, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) if element.text: - rv.append("|%s\"%s\"" %(' '*(indent+2), element.text)) + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) indent += 2 for child in element: serializeElement(child, indent) if element.tail: - rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail)) + rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail)) serializeElement(element, 0) - - if finalText is not None: - rv.append("|%s\"%s\""%(' '*2, finalText)) - + return "\n".join(rv) - + def tostring(element): """Serialize an element and its child nodes to a string""" rv = [] - finalText = None filter = ihatexml.InfosetFilter() + def serializeElement(element): - if type(element) == type(ElementTree.ElementTree): + if isinstance(element, ElementTree.ElementTree): element = element.getroot() - + if element.tag == "<!DOCTYPE>": if element.get("publicId") or element.get("systemId"): publicId = element.get("publicId") or "" systemId = element.get("systemId") or "" - rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%( - element.text, publicId, systemId)) - else: - rv.append("<!DOCTYPE %s>"%(element.text,)) - elif element.tag == "<DOCUMENT_ROOT>": - if element.text: - rv.append(element.text) - if element.tail: - finalText = element.tail - - for child in element: - serializeElement(child) - - elif type(element.tag) == type(ElementTree.Comment): - rv.append("<!--%s-->"%(element.text,)) - else: - #This is assumed to be an ordinary element - if not element.attrib: - rv.append("<%s>"%(filter.fromXmlName(element.tag),)) + rv.append("""<!DOCTYPE %s PUBLIC "%s" "%s">""" % + (element.text, publicId, systemId)) else: - attr = " ".join(["%s=\"%s\""%( - filter.fromXmlName(name), value) - for name, value in element.attrib.iteritems()]) - rv.append("<%s %s>"%(element.tag, attr)) - if element.text: + rv.append("<!DOCTYPE %s>" % (element.text,)) + elif element.tag == "DOCUMENT_ROOT": + if element.text is not None: rv.append(element.text) - + if element.tail is not None: + raise TypeError("Document node cannot have tail") + if hasattr(element, "attrib") and len(element.attrib): + raise TypeError("Document node cannot have attributes") + for child in element: serializeElement(child) - - rv.append("</%s>"%(element.tag,)) - + + elif element.tag == ElementTreeCommentType: + rv.append("<!--%s-->" % (element.text,)) + else: + # This is assumed to be an ordinary element + if not element.attrib: + rv.append("<%s>" % (filter.fromXmlName(element.tag),)) + else: + attr = " ".join(["%s=\"%s\"" % ( + filter.fromXmlName(name), value) + for name, value in element.attrib.items()]) + rv.append("<%s %s>" % (element.tag, attr)) + if element.text: + rv.append(element.text) + + for child in element: + serializeElement(child) + + rv.append("</%s>" % (element.tag,)) + if element.tail: rv.append(element.tail) - + serializeElement(element) - - if finalText is not None: - rv.append("%s\""%(' '*2, finalText)) - + return "".join(rv) - + class TreeBuilder(_base.TreeBuilder): documentClass = Document doctypeClass = DocumentType elementClass = Element commentClass = Comment fragmentClass = DocumentFragment - + implementation = ElementTreeImplementation + def testSerializer(self, element): return testSerializer(element) - + def getDocument(self): if fullTree: return self.document._element else: if self.defaultNamespace is not None: return self.document._element.find( - "{%s}html"%self.defaultNamespace) + "{%s}html" % self.defaultNamespace) else: return self.document._element.find("html") - + def getFragment(self): return _base.TreeBuilder.getFragment(self)._element - + return locals() + + +getETreeModule = moduleFactoryFactory(getETreeBuilder) diff --git a/libs/html5lib/treebuilders/etree_lxml.py b/libs/html5lib/treebuilders/etree_lxml.py index eee1e3b2..35d08efa 100644 --- a/libs/html5lib/treebuilders/etree_lxml.py +++ b/libs/html5lib/treebuilders/etree_lxml.py @@ -1,20 +1,3 @@ -import warnings -import re - -import _base -from html5lib.constants import DataLossWarning -import html5lib.constants as constants -import etree as etree_builders -from html5lib import ihatexml - -try: - import lxml.etree as etree -except ImportError: - pass - -fullTree = True -tag_regexp = re.compile("{([^}]*)}(.*)") - """Module for supporting the lxml.etree library. The idea here is to use as much of the native library as possible, without using fragile hacks like custom element names that break between releases. The downside of this is that we cannot represent @@ -26,12 +9,34 @@ Docypes with no name When any of these things occur, we emit a DataLossWarning """ +from __future__ import absolute_import, division, unicode_literals + +import warnings +import re +import sys + +from . import _base +from ..constants import DataLossWarning +from .. import constants +from . import etree as etree_builders +from .. import ihatexml + +import lxml.etree as etree + + +fullTree = True +tag_regexp = re.compile("{([^}]*)}(.*)") + +comment_type = etree.Comment("asd").tag + + class DocumentType(object): def __init__(self, name, publicId, systemId): - self.name = name + self.name = name self.publicId = publicId self.systemId = systemId + class Document(object): def __init__(self): self._elementTree = None @@ -42,118 +47,126 @@ class Document(object): def _getChildNodes(self): return self._childNodes - + childNodes = property(_getChildNodes) + def testSerializer(element): rv = [] finalText = None - filter = ihatexml.InfosetFilter() + infosetFilter = ihatexml.InfosetFilter() + def serializeElement(element, indent=0): if not hasattr(element, "tag"): - if hasattr(element, "getroot"): - #Full tree case + if hasattr(element, "getroot"): + # Full tree case rv.append("#document") if element.docinfo.internalDTD: - if not (element.docinfo.public_id or + if not (element.docinfo.public_id or element.docinfo.system_url): - dtd_str = "<!DOCTYPE %s>"%element.docinfo.root_name + dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name else: - dtd_str = """<!DOCTYPE %s "%s" "%s">"""%( - element.docinfo.root_name, + dtd_str = """<!DOCTYPE %s "%s" "%s">""" % ( + element.docinfo.root_name, element.docinfo.public_id, element.docinfo.system_url) - rv.append("|%s%s"%(' '*(indent+2), dtd_str)) + rv.append("|%s%s" % (' ' * (indent + 2), dtd_str)) next_element = element.getroot() while next_element.getprevious() is not None: next_element = next_element.getprevious() while next_element is not None: - serializeElement(next_element, indent+2) + serializeElement(next_element, indent + 2) next_element = next_element.getnext() - elif isinstance(element, basestring): - #Text in a fragment - rv.append("|%s\"%s\""%(' '*indent, element)) + elif isinstance(element, str) or isinstance(element, bytes): + # Text in a fragment + assert isinstance(element, str) or sys.version_info.major == 2 + rv.append("|%s\"%s\"" % (' ' * indent, element)) else: - #Fragment case + # Fragment case rv.append("#document-fragment") for next_element in element: - serializeElement(next_element, indent+2) - elif type(element.tag) == type(etree.Comment): - rv.append("|%s<!-- %s -->"%(' '*indent, element.text)) + serializeElement(next_element, indent + 2) + elif element.tag == comment_type: + rv.append("|%s<!-- %s -->" % (' ' * indent, element.text)) + if hasattr(element, "tail") and element.tail: + rv.append("|%s\"%s\"" % (' ' * indent, element.tail)) else: + assert isinstance(element, etree._Element) nsmatch = etree_builders.tag_regexp.match(element.tag) if nsmatch is not None: ns = nsmatch.group(1) tag = nsmatch.group(2) prefix = constants.prefixes[ns] - rv.append("|%s<%s %s>"%(' '*indent, prefix, - filter.fromXmlName(tag))) + rv.append("|%s<%s %s>" % (' ' * indent, prefix, + infosetFilter.fromXmlName(tag))) else: - rv.append("|%s<%s>"%(' '*indent, - filter.fromXmlName(element.tag))) + rv.append("|%s<%s>" % (' ' * indent, + infosetFilter.fromXmlName(element.tag))) if hasattr(element, "attrib"): attributes = [] - for name, value in element.attrib.iteritems(): + for name, value in element.attrib.items(): nsmatch = tag_regexp.match(name) if nsmatch is not None: ns, name = nsmatch.groups() - name = filter.fromXmlName(name) + name = infosetFilter.fromXmlName(name) prefix = constants.prefixes[ns] - attr_string = "%s %s"%(prefix, name) + attr_string = "%s %s" % (prefix, name) else: - attr_string = filter.fromXmlName(name) + attr_string = infosetFilter.fromXmlName(name) attributes.append((attr_string, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) if element.text: - rv.append("|%s\"%s\"" %(' '*(indent+2), element.text)) + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) indent += 2 - for child in element.getchildren(): + for child in element: serializeElement(child, indent) - if hasattr(element, "tail") and element.tail: - rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail)) + if hasattr(element, "tail") and element.tail: + rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail)) serializeElement(element, 0) if finalText is not None: - rv.append("|%s\"%s\""%(' '*2, finalText)) + rv.append("|%s\"%s\"" % (' ' * 2, finalText)) return "\n".join(rv) + def tostring(element): """Serialize an element and its child nodes to a string""" rv = [] finalText = None + def serializeElement(element): if not hasattr(element, "tag"): if element.docinfo.internalDTD: if element.docinfo.doctype: dtd_str = element.docinfo.doctype else: - dtd_str = "<!DOCTYPE %s>"%element.docinfo.root_name + dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name rv.append(dtd_str) serializeElement(element.getroot()) - - elif type(element.tag) == type(etree.Comment): - rv.append("<!--%s-->"%(element.text,)) - + + elif element.tag == comment_type: + rv.append("<!--%s-->" % (element.text,)) + else: - #This is assumed to be an ordinary element + # This is assumed to be an ordinary element if not element.attrib: - rv.append("<%s>"%(element.tag,)) + rv.append("<%s>" % (element.tag,)) else: - attr = " ".join(["%s=\"%s\""%(name, value) - for name, value in element.attrib.iteritems()]) - rv.append("<%s %s>"%(element.tag, attr)) + attr = " ".join(["%s=\"%s\"" % (name, value) + for name, value in element.attrib.items()]) + rv.append("<%s %s>" % (element.tag, attr)) if element.text: rv.append(element.text) - for child in element.getchildren(): + for child in element: serializeElement(child) - rv.append("</%s>"%(element.tag,)) + rv.append("</%s>" % (element.tag,)) if hasattr(element, "tail") and element.tail: rv.append(element.tail) @@ -161,56 +174,57 @@ def tostring(element): serializeElement(element) if finalText is not None: - rv.append("%s\""%(' '*2, finalText)) + rv.append("%s\"" % (' ' * 2, finalText)) return "".join(rv) - + class TreeBuilder(_base.TreeBuilder): documentClass = Document doctypeClass = DocumentType elementClass = None commentClass = None - fragmentClass = Document + fragmentClass = Document + implementation = etree - def __init__(self, namespaceHTMLElements, fullTree = False): + def __init__(self, namespaceHTMLElements, fullTree=False): builder = etree_builders.getETreeModule(etree, fullTree=fullTree) - filter = self.filter = ihatexml.InfosetFilter() + infosetFilter = self.infosetFilter = ihatexml.InfosetFilter() self.namespaceHTMLElements = namespaceHTMLElements class Attributes(dict): def __init__(self, element, value={}): self._element = element dict.__init__(self, value) - for key, value in self.iteritems(): + for key, value in self.items(): if isinstance(key, tuple): - name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1])) + name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1])) else: - name = filter.coerceAttribute(key) + name = infosetFilter.coerceAttribute(key) self._element._element.attrib[name] = value def __setitem__(self, key, value): dict.__setitem__(self, key, value) if isinstance(key, tuple): - name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1])) + name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1])) else: - name = filter.coerceAttribute(key) + name = infosetFilter.coerceAttribute(key) self._element._element.attrib[name] = value class Element(builder.Element): def __init__(self, name, namespace): - name = filter.coerceElement(name) + name = infosetFilter.coerceElement(name) builder.Element.__init__(self, name, namespace=namespace) self._attributes = Attributes(self) def _setName(self, name): - self._name = filter.coerceElement(name) + self._name = infosetFilter.coerceElement(name) self._element.tag = self._getETreeTag( self._name, self._namespace) - + def _getName(self): - return filter.fromXmlName(self._name) - + return infosetFilter.fromXmlName(self._name) + name = property(_getName, _setName) def _getAttributes(self): @@ -218,24 +232,23 @@ class TreeBuilder(_base.TreeBuilder): def _setAttributes(self, attributes): self._attributes = Attributes(self, attributes) - + attributes = property(_getAttributes, _setAttributes) def insertText(self, data, insertBefore=None): - data = filter.coerceCharacters(data) + data = infosetFilter.coerceCharacters(data) builder.Element.insertText(self, data, insertBefore) def appendChild(self, child): builder.Element.appendChild(self, child) - class Comment(builder.Comment): def __init__(self, data): - data = filter.coerceComment(data) + data = infosetFilter.coerceComment(data) builder.Comment.__init__(self, data) def _setData(self, data): - data = filter.coerceComment(data) + data = infosetFilter.coerceComment(data) self._element.text = data def _getData(self): @@ -245,9 +258,9 @@ class TreeBuilder(_base.TreeBuilder): self.elementClass = Element self.commentClass = builder.Comment - #self.fragmentClass = builder.DocumentFragment + # self.fragmentClass = builder.DocumentFragment _base.TreeBuilder.__init__(self, namespaceHTMLElements) - + def reset(self): _base.TreeBuilder.reset(self) self.insertComment = self.insertCommentInitial @@ -262,13 +275,13 @@ class TreeBuilder(_base.TreeBuilder): return self.document._elementTree else: return self.document._elementTree.getroot() - + def getFragment(self): fragment = [] element = self.openElements[0]._element if element.text: fragment.append(element.text) - fragment.extend(element.getchildren()) + fragment.extend(list(element)) if element.tail: fragment.append(element.tail) return fragment @@ -278,59 +291,79 @@ class TreeBuilder(_base.TreeBuilder): publicId = token["publicId"] systemId = token["systemId"] - if not name or ihatexml.nonXmlNameBMPRegexp.search(name) or name[0] == '"': - warnings.warn("lxml cannot represent null or non-xml doctype", DataLossWarning) + if not name: + warnings.warn("lxml cannot represent empty doctype", DataLossWarning) + self.doctype = None + else: + coercedName = self.infosetFilter.coerceElement(name) + if coercedName != name: + warnings.warn("lxml cannot represent non-xml doctype", DataLossWarning) + + doctype = self.doctypeClass(coercedName, publicId, systemId) + self.doctype = doctype - doctype = self.doctypeClass(name, publicId, systemId) - self.doctype = doctype - def insertCommentInitial(self, data, parent=None): self.initial_comments.append(data) - + + def insertCommentMain(self, data, parent=None): + if (parent == self.document and + self.document._elementTree.getroot()[-1].tag == comment_type): + warnings.warn("lxml cannot represent adjacent comments beyond the root elements", DataLossWarning) + super(TreeBuilder, self).insertComment(data, parent) + def insertRoot(self, token): """Create the document root""" - #Because of the way libxml2 works, it doesn't seem to be possible to - #alter information like the doctype after the tree has been parsed. - #Therefore we need to use the built-in parser to create our iniial - #tree, after which we can add elements like normal + # Because of the way libxml2 works, it doesn't seem to be possible to + # alter information like the doctype after the tree has been parsed. + # Therefore we need to use the built-in parser to create our iniial + # tree, after which we can add elements like normal docStr = "" - if self.doctype and self.doctype.name and not self.doctype.name.startswith('"'): - docStr += "<!DOCTYPE %s"%self.doctype.name - if (self.doctype.publicId is not None or - self.doctype.systemId is not None): - docStr += ' PUBLIC "%s" "%s"'%(self.doctype.publicId or "", - self.doctype.systemId or "") + if self.doctype: + assert self.doctype.name + docStr += "<!DOCTYPE %s" % self.doctype.name + if (self.doctype.publicId is not None or + self.doctype.systemId is not None): + docStr += (' PUBLIC "%s" ' % + (self.infosetFilter.coercePubid(self.doctype.publicId or ""))) + if self.doctype.systemId: + sysid = self.doctype.systemId + if sysid.find("'") >= 0 and sysid.find('"') >= 0: + warnings.warn("DOCTYPE system cannot contain single and double quotes", DataLossWarning) + sysid = sysid.replace("'", 'U00027') + if sysid.find("'") >= 0: + docStr += '"%s"' % sysid + else: + docStr += "'%s'" % sysid + else: + docStr += "''" docStr += ">" + if self.doctype.name != token["name"]: + warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning) docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>" - - try: - root = etree.fromstring(docStr) - except etree.XMLSyntaxError: - print docStr - raise - - #Append the initial comments: + root = etree.fromstring(docStr) + + # Append the initial comments: for comment_token in self.initial_comments: root.addprevious(etree.Comment(comment_token["data"])) - - #Create the root document and add the ElementTree to it + + # Create the root document and add the ElementTree to it self.document = self.documentClass() self.document._elementTree = root.getroottree() - + # Give the root element the right name name = token["name"] namespace = token.get("namespace", self.defaultNamespace) if namespace is None: etree_tag = name else: - etree_tag = "{%s}%s"%(namespace, name) + etree_tag = "{%s}%s" % (namespace, name) root.tag = etree_tag - - #Add the root element to the internal child/open data structures + + # Add the root element to the internal child/open data structures root_element = self.elementClass(name, namespace) root_element._element = root self.document._childNodes.append(root_element) self.openElements.append(root_element) - - #Reset to the default insert comment function - self.insertComment = super(TreeBuilder, self).insertComment + + # Reset to the default insert comment function + self.insertComment = self.insertCommentMain diff --git a/libs/html5lib/treebuilders/simpletree.py b/libs/html5lib/treebuilders/simpletree.py deleted file mode 100755 index 67fe7583..00000000 --- a/libs/html5lib/treebuilders/simpletree.py +++ /dev/null @@ -1,256 +0,0 @@ -import _base -from html5lib.constants import voidElements, namespaces, prefixes -from xml.sax.saxutils import escape - -# Really crappy basic implementation of a DOM-core like thing -class Node(_base.Node): - type = -1 - def __init__(self, name): - self.name = name - self.parent = None - self.value = None - self.childNodes = [] - self._flags = [] - - def __iter__(self): - for node in self.childNodes: - yield node - for item in node: - yield item - - def __unicode__(self): - return self.name - - def toxml(self): - raise NotImplementedError - - def printTree(self, indent=0): - tree = '\n|%s%s' % (' '* indent, unicode(self)) - for child in self.childNodes: - tree += child.printTree(indent + 2) - return tree - - def appendChild(self, node): - assert isinstance(node, Node) - if (isinstance(node, TextNode) and self.childNodes and - isinstance(self.childNodes[-1], TextNode)): - self.childNodes[-1].value += node.value - else: - self.childNodes.append(node) - node.parent = self - - def insertText(self, data, insertBefore=None): - assert isinstance(data, unicode), "data %s is of type %s expected unicode"%(repr(data), type(data)) - if insertBefore is None: - self.appendChild(TextNode(data)) - else: - self.insertBefore(TextNode(data), insertBefore) - - def insertBefore(self, node, refNode): - index = self.childNodes.index(refNode) - if (isinstance(node, TextNode) and index > 0 and - isinstance(self.childNodes[index - 1], TextNode)): - self.childNodes[index - 1].value += node.value - else: - self.childNodes.insert(index, node) - node.parent = self - - def removeChild(self, node): - try: - self.childNodes.remove(node) - except: - # XXX - raise - node.parent = None - - def cloneNode(self): - raise NotImplementedError - - def hasContent(self): - """Return true if the node has children or text""" - return bool(self.childNodes) - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class Document(Node): - type = 1 - def __init__(self): - Node.__init__(self, None) - - def __str__(self): - return "#document" - - def __unicode__(self): - return str(self) - - def appendChild(self, child): - Node.appendChild(self, child) - - def toxml(self, encoding="utf=8"): - result = "" - for child in self.childNodes: - result += child.toxml() - return result.encode(encoding) - - def hilite(self, encoding="utf-8"): - result = "<pre>" - for child in self.childNodes: - result += child.hilite() - return result.encode(encoding) + "</pre>" - - def printTree(self): - tree = unicode(self) - for child in self.childNodes: - tree += child.printTree(2) - return tree - - def cloneNode(self): - return Document() - -class DocumentFragment(Document): - type = 2 - def __str__(self): - return "#document-fragment" - - def __unicode__(self): - return str(self) - - def cloneNode(self): - return DocumentFragment() - -class DocumentType(Node): - type = 3 - def __init__(self, name, publicId, systemId): - Node.__init__(self, name) - self.publicId = publicId - self.systemId = systemId - - def __unicode__(self): - if self.publicId or self.systemId: - publicId = self.publicId or "" - systemId = self.systemId or "" - return """<!DOCTYPE %s "%s" "%s">"""%( - self.name, publicId, systemId) - - else: - return u"<!DOCTYPE %s>" % self.name - - - toxml = __unicode__ - - def hilite(self): - return '<code class="markup doctype"><!DOCTYPE %s></code>' % self.name - - def cloneNode(self): - return DocumentType(self.name, self.publicId, self.systemId) - -class TextNode(Node): - type = 4 - def __init__(self, value): - Node.__init__(self, None) - self.value = value - - def __unicode__(self): - return u"\"%s\"" % self.value - - def toxml(self): - return escape(self.value) - - hilite = toxml - - def cloneNode(self): - return TextNode(self.value) - -class Element(Node): - type = 5 - def __init__(self, name, namespace=None): - Node.__init__(self, name) - self.namespace = namespace - self.attributes = {} - - def __unicode__(self): - if self.namespace == None: - return u"<%s>" % self.name - else: - return u"<%s %s>"%(prefixes[self.namespace], self.name) - - def toxml(self): - result = '<' + self.name - if self.attributes: - for name,value in self.attributes.iteritems(): - result += u' %s="%s"' % (name, escape(value,{'"':'"'})) - if self.childNodes: - result += '>' - for child in self.childNodes: - result += child.toxml() - result += u'</%s>' % self.name - else: - result += u'/>' - return result - - def hilite(self): - result = '<<code class="markup element-name">%s</code>' % self.name - if self.attributes: - for name, value in self.attributes.iteritems(): - result += ' <code class="markup attribute-name">%s</code>=<code class="markup attribute-value">"%s"</code>' % (name, escape(value, {'"':'"'})) - if self.childNodes: - result += ">" - for child in self.childNodes: - result += child.hilite() - elif self.name in voidElements: - return result + ">" - return result + '</<code class="markup element-name">%s</code>>' % self.name - - def printTree(self, indent): - tree = '\n|%s%s' % (' '*indent, unicode(self)) - indent += 2 - if self.attributes: - for name, value in sorted(self.attributes.iteritems()): - if isinstance(name, tuple): - name = "%s %s"%(name[0], name[1]) - tree += '\n|%s%s="%s"' % (' ' * indent, name, value) - for child in self.childNodes: - tree += child.printTree(indent) - return tree - - def cloneNode(self): - newNode = Element(self.name) - if hasattr(self, 'namespace'): - newNode.namespace = self.namespace - for attr, value in self.attributes.iteritems(): - newNode.attributes[attr] = value - return newNode - -class CommentNode(Node): - type = 6 - def __init__(self, data): - Node.__init__(self, None) - self.data = data - - def __unicode__(self): - return "<!-- %s -->" % self.data - - def toxml(self): - return "<!--%s-->" % self.data - - def hilite(self): - return '<code class="markup comment"><!--%s--></code>' % escape(self.data) - - def cloneNode(self): - return CommentNode(self.data) - -class TreeBuilder(_base.TreeBuilder): - documentClass = Document - doctypeClass = DocumentType - elementClass = Element - commentClass = CommentNode - fragmentClass = DocumentFragment - - def testSerializer(self, node): - return node.printTree() diff --git a/libs/html5lib/treebuilders/soup.py b/libs/html5lib/treebuilders/soup.py deleted file mode 100644 index 9bc5ff0e..00000000 --- a/libs/html5lib/treebuilders/soup.py +++ /dev/null @@ -1,236 +0,0 @@ -import warnings - -warnings.warn("BeautifulSoup 3.x (as of 3.1) is not fully compatible with html5lib and support will be removed in the future", DeprecationWarning) - -from BeautifulSoup import BeautifulSoup, Tag, NavigableString, Comment, Declaration - -import _base -from html5lib.constants import namespaces, DataLossWarning - -class AttrList(object): - def __init__(self, element): - self.element = element - self.attrs = dict(self.element.attrs) - def __iter__(self): - return self.attrs.items().__iter__() - def __setitem__(self, name, value): - "set attr", name, value - self.element[name] = value - def items(self): - return self.attrs.items() - def keys(self): - return self.attrs.keys() - def __getitem__(self, name): - return self.attrs[name] - def __contains__(self, name): - return name in self.attrs.keys() - def __eq__(self, other): - if len(self.keys()) != len(other.keys()): - return False - for item in self.keys(): - if item not in other: - return False - if self[item] != other[item]: - return False - return True - -class Element(_base.Node): - def __init__(self, element, soup, namespace): - _base.Node.__init__(self, element.name) - self.element = element - self.soup = soup - self.namespace = namespace - - def _nodeIndex(self, node, refNode): - # Finds a node by identity rather than equality - for index in range(len(self.element.contents)): - if id(self.element.contents[index]) == id(refNode.element): - return index - return None - - def appendChild(self, node): - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[-1].__class__ == NavigableString): - # Concatenate new text onto old text node - # (TODO: This has O(n^2) performance, for input like "a</a>a</a>a</a>...") - newStr = NavigableString(self.element.contents[-1]+node.element) - - # Remove the old text node - # (Can't simply use .extract() by itself, because it fails if - # an equal text node exists within the parent node) - oldElement = self.element.contents[-1] - del self.element.contents[-1] - oldElement.parent = None - oldElement.extract() - - self.element.insert(len(self.element.contents), newStr) - else: - self.element.insert(len(self.element.contents), node.element) - node.parent = self - - def getAttributes(self): - return AttrList(self.element) - - def setAttributes(self, attributes): - if attributes: - for name, value in attributes.items(): - self.element[name] = value - - attributes = property(getAttributes, setAttributes) - - def insertText(self, data, insertBefore=None): - text = TextNode(NavigableString(data), self.soup) - if insertBefore: - self.insertBefore(text, insertBefore) - else: - self.appendChild(text) - - def insertBefore(self, node, refNode): - index = self._nodeIndex(node, refNode) - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[index-1].__class__ == NavigableString): - # (See comments in appendChild) - newStr = NavigableString(self.element.contents[index-1]+node.element) - oldNode = self.element.contents[index-1] - del self.element.contents[index-1] - oldNode.parent = None - oldNode.extract() - - self.element.insert(index-1, newStr) - else: - self.element.insert(index, node.element) - node.parent = self - - def removeChild(self, node): - index = self._nodeIndex(node.parent, node) - del node.parent.element.contents[index] - node.element.parent = None - node.element.extract() - node.parent = None - - def reparentChildren(self, newParent): - while self.element.contents: - child = self.element.contents[0] - child.extract() - if isinstance(child, Tag): - newParent.appendChild(Element(child, self.soup, namespaces["html"])) - else: - newParent.appendChild(TextNode(child, self.soup)) - - def cloneNode(self): - node = Element(Tag(self.soup, self.element.name), self.soup, self.namespace) - for key,value in self.attributes: - node.attributes[key] = value - return node - - def hasContent(self): - return self.element.contents - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class TextNode(Element): - def __init__(self, element, soup): - _base.Node.__init__(self, None) - self.element = element - self.soup = soup - - def cloneNode(self): - raise NotImplementedError - -class TreeBuilder(_base.TreeBuilder): - def __init__(self, namespaceHTMLElements): - if namespaceHTMLElements: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - _base.TreeBuilder.__init__(self, namespaceHTMLElements) - - def documentClass(self): - self.soup = BeautifulSoup("") - return Element(self.soup, self.soup, None) - - def insertDoctype(self, token): - name = token["name"] - publicId = token["publicId"] - systemId = token["systemId"] - - if publicId: - self.soup.insert(0, Declaration("DOCTYPE %s PUBLIC \"%s\" \"%s\""%(name, publicId, systemId or ""))) - elif systemId: - self.soup.insert(0, Declaration("DOCTYPE %s SYSTEM \"%s\""% - (name, systemId))) - else: - self.soup.insert(0, Declaration("DOCTYPE %s"%name)) - - def elementClass(self, name, namespace): - if namespace is not None: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - return Element(Tag(self.soup, name), self.soup, namespace) - - def commentClass(self, data): - return TextNode(Comment(data), self.soup) - - def fragmentClass(self): - self.soup = BeautifulSoup("") - self.soup.name = "[document_fragment]" - return Element(self.soup, self.soup, None) - - def appendChild(self, node): - self.soup.insert(len(self.soup.contents), node.element) - - def testSerializer(self, element): - return testSerializer(element) - - def getDocument(self): - return self.soup - - def getFragment(self): - return _base.TreeBuilder.getFragment(self).element - -def testSerializer(element): - import re - rv = [] - def serializeElement(element, indent=0): - if isinstance(element, Declaration): - doctype_regexp = r'DOCTYPE\s+(?P<name>[^\s]*)( PUBLIC "(?P<publicId>.*)" "(?P<systemId1>.*)"| SYSTEM "(?P<systemId2>.*)")?' - m = re.compile(doctype_regexp).match(element.string) - assert m is not None, "DOCTYPE did not match expected format" - name = m.group('name') - publicId = m.group('publicId') - if publicId is not None: - systemId = m.group('systemId1') or "" - else: - systemId = m.group('systemId2') - - if publicId is not None or systemId is not None: - rv.append("""|%s<!DOCTYPE %s "%s" "%s">"""% - (' '*indent, name, publicId or "", systemId or "")) - else: - rv.append("|%s<!DOCTYPE %s>"%(' '*indent, name)) - - elif isinstance(element, BeautifulSoup): - if element.name == "[document_fragment]": - rv.append("#document-fragment") - else: - rv.append("#document") - - elif isinstance(element, Comment): - rv.append("|%s<!-- %s -->"%(' '*indent, element.string)) - elif isinstance(element, unicode): - rv.append("|%s\"%s\"" %(' '*indent, element)) - else: - rv.append("|%s<%s>"%(' '*indent, element.name)) - if element.attrs: - for name, value in sorted(element.attrs): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) - indent += 2 - if hasattr(element, "contents"): - for child in element.contents: - serializeElement(child, indent) - serializeElement(element, 0) - - return "\n".join(rv) diff --git a/libs/html5lib/treewalkers/__init__.py b/libs/html5lib/treewalkers/__init__.py index 3a606a8b..18124e75 100644 --- a/libs/html5lib/treewalkers/__init__.py +++ b/libs/html5lib/treewalkers/__init__.py @@ -8,23 +8,27 @@ implements a 'serialize' method taking a tree as sole argument and returning an iterator generating tokens. """ +from __future__ import absolute_import, division, unicode_literals + +import sys + +from ..utils import default_etree + treeWalkerCache = {} + def getTreeWalker(treeType, implementation=None, **kwargs): """Get a TreeWalker class for various types of tree with built-in support treeType - the name of the tree type required (case-insensitive). Supported - values are "simpletree", "dom", "etree" and "beautifulsoup" + values are: - "simpletree" - a built-in DOM-ish tree type with support for some - more pythonic idioms. "dom" - The xml.dom.minidom DOM implementation "pulldom" - The xml.dom.pulldom event stream "etree" - A generic walker for tree implementations exposing an elementtree-like interface (known to work with ElementTree, cElementTree and lxml.etree). "lxml" - Optimized walker for lxml.etree - "beautifulsoup" - Beautiful soup (if installed) "genshi" - a Genshi stream implementation - (Currently applies to the "etree" tree type only). A module @@ -33,20 +37,21 @@ def getTreeWalker(treeType, implementation=None, **kwargs): treeType = treeType.lower() if treeType not in treeWalkerCache: - if treeType in ("dom", "pulldom", "simpletree"): - mod = __import__(treeType, globals()) + if treeType in ("dom", "pulldom"): + name = "%s.%s" % (__name__, treeType) + __import__(name) + mod = sys.modules[name] treeWalkerCache[treeType] = mod.TreeWalker elif treeType == "genshi": - import genshistream + from . import genshistream treeWalkerCache[treeType] = genshistream.TreeWalker - elif treeType == "beautifulsoup": - import soup - treeWalkerCache[treeType] = soup.TreeWalker elif treeType == "lxml": - import lxmletree + from . import lxmletree treeWalkerCache[treeType] = lxmletree.TreeWalker elif treeType == "etree": - import etree + from . import etree + if implementation is None: + implementation = default_etree # XXX: NEVER cache here, caching is done in the etree submodule return etree.getETreeModule(implementation, **kwargs).TreeWalker return treeWalkerCache.get(treeType) diff --git a/libs/html5lib/treewalkers/_base.py b/libs/html5lib/treewalkers/_base.py index 5929ba05..34252e50 100644 --- a/libs/html5lib/treewalkers/_base.py +++ b/libs/html5lib/treewalkers/_base.py @@ -1,94 +1,9 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type, string_types + import gettext _ = gettext.gettext -from html5lib.constants import voidElements, spaceCharacters -spaceCharacters = u"".join(spaceCharacters) - -class TreeWalker(object): - def __init__(self, tree): - self.tree = tree - - def __iter__(self): - raise NotImplementedError - - def error(self, msg): - return {"type": "SerializeError", "data": msg} - - def normalizeAttrs(self, attrs): - newattrs = {} - if attrs: - #TODO: treewalkers should always have attrs - for (namespace,name),value in attrs.iteritems(): - namespace = unicode(namespace) if namespace else None - name = unicode(name) - value = unicode(value) - newattrs[(namespace,name)] = value - return newattrs - - def emptyTag(self, namespace, name, attrs, hasChildren=False): - yield {"type": "EmptyTag", "name": unicode(name), - "namespace":unicode(namespace), - "data": self.normalizeAttrs(attrs)} - if hasChildren: - yield self.error(_("Void element has children")) - - def startTag(self, namespace, name, attrs): - return {"type": "StartTag", - "name": unicode(name), - "namespace":unicode(namespace), - "data": self.normalizeAttrs(attrs)} - - def endTag(self, namespace, name): - return {"type": "EndTag", - "name": unicode(name), - "namespace":unicode(namespace), - "data": {}} - - def text(self, data): - data = unicode(data) - middle = data.lstrip(spaceCharacters) - left = data[:len(data)-len(middle)] - if left: - yield {"type": "SpaceCharacters", "data": left} - data = middle - middle = data.rstrip(spaceCharacters) - right = data[len(middle):] - if middle: - yield {"type": "Characters", "data": middle} - if right: - yield {"type": "SpaceCharacters", "data": right} - - def comment(self, data): - return {"type": "Comment", "data": unicode(data)} - - def doctype(self, name, publicId=None, systemId=None, correct=True): - return {"type": "Doctype", - "name": name is not None and unicode(name) or u"", - "publicId": publicId, - "systemId": systemId, - "correct": correct} - - def entity(self, name): - return {"type": "Entity", "name": unicode(name)} - - def unknown(self, nodeType): - return self.error(_("Unknown node type: ") + nodeType) - -class RecursiveTreeWalker(TreeWalker): - def walkChildren(self, node): - raise NodeImplementedError - - def element(self, node, namespace, name, attrs, hasChildren): - if name in voidElements: - for token in self.emptyTag(namespace, name, attrs, hasChildren): - yield token - else: - yield self.startTag(name, attrs) - if hasChildren: - for token in self.walkChildren(node): - yield token - yield self.endTag(name) - from xml.dom import Node DOCUMENT = Node.DOCUMENT_NODE @@ -99,16 +14,127 @@ COMMENT = Node.COMMENT_NODE ENTITY = Node.ENTITY_NODE UNKNOWN = "<#UNKNOWN#>" +from ..constants import voidElements, spaceCharacters +spaceCharacters = "".join(spaceCharacters) + + +def to_text(s, blank_if_none=True): + """Wrapper around six.text_type to convert None to empty string""" + if s is None: + if blank_if_none: + return "" + else: + return None + elif isinstance(s, text_type): + return s + else: + return text_type(s) + + +def is_text_or_none(string): + """Wrapper around isinstance(string_types) or is None""" + return string is None or isinstance(string, string_types) + + +class TreeWalker(object): + def __init__(self, tree): + self.tree = tree + + def __iter__(self): + raise NotImplementedError + + def error(self, msg): + return {"type": "SerializeError", "data": msg} + + def emptyTag(self, namespace, name, attrs, hasChildren=False): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(name) + assert all((namespace is None or isinstance(namespace, string_types)) and + isinstance(name, string_types) and + isinstance(value, string_types) + for (namespace, name), value in attrs.items()) + + yield {"type": "EmptyTag", "name": to_text(name, False), + "namespace": to_text(namespace), + "data": attrs} + if hasChildren: + yield self.error(_("Void element has children")) + + def startTag(self, namespace, name, attrs): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(name) + assert all((namespace is None or isinstance(namespace, string_types)) and + isinstance(name, string_types) and + isinstance(value, string_types) + for (namespace, name), value in attrs.items()) + + return {"type": "StartTag", + "name": text_type(name), + "namespace": to_text(namespace), + "data": dict(((to_text(namespace, False), to_text(name)), + to_text(value, False)) + for (namespace, name), value in attrs.items())} + + def endTag(self, namespace, name): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(namespace) + + return {"type": "EndTag", + "name": to_text(name, False), + "namespace": to_text(namespace), + "data": {}} + + def text(self, data): + assert isinstance(data, string_types), type(data) + + data = to_text(data) + middle = data.lstrip(spaceCharacters) + left = data[:len(data) - len(middle)] + if left: + yield {"type": "SpaceCharacters", "data": left} + data = middle + middle = data.rstrip(spaceCharacters) + right = data[len(middle):] + if middle: + yield {"type": "Characters", "data": middle} + if right: + yield {"type": "SpaceCharacters", "data": right} + + def comment(self, data): + assert isinstance(data, string_types), type(data) + + return {"type": "Comment", "data": text_type(data)} + + def doctype(self, name, publicId=None, systemId=None, correct=True): + assert is_text_or_none(name), type(name) + assert is_text_or_none(publicId), type(publicId) + assert is_text_or_none(systemId), type(systemId) + + return {"type": "Doctype", + "name": to_text(name), + "publicId": to_text(publicId), + "systemId": to_text(systemId), + "correct": to_text(correct)} + + def entity(self, name): + assert isinstance(name, string_types), type(name) + + return {"type": "Entity", "name": text_type(name)} + + def unknown(self, nodeType): + return self.error(_("Unknown node type: ") + nodeType) + + class NonRecursiveTreeWalker(TreeWalker): def getNodeDetails(self, node): raise NotImplementedError - + def getFirstChild(self, node): raise NotImplementedError - + def getNextSibling(self, node): raise NotImplementedError - + def getParentNode(self, node): raise NotImplementedError @@ -118,7 +144,6 @@ class NonRecursiveTreeWalker(TreeWalker): details = self.getNodeDetails(currentNode) type, details = details[0], details[1:] hasChildren = False - endTag = None if type == DOCTYPE: yield self.doctype(*details) @@ -130,12 +155,11 @@ class NonRecursiveTreeWalker(TreeWalker): elif type == ELEMENT: namespace, name, attributes, hasChildren = details if name in voidElements: - for token in self.emptyTag(namespace, name, attributes, + for token in self.emptyTag(namespace, name, attributes, hasChildren): yield token hasChildren = False else: - endTag = name yield self.startTag(namespace, name, attributes) elif type == COMMENT: @@ -149,12 +173,12 @@ class NonRecursiveTreeWalker(TreeWalker): else: yield self.unknown(details[0]) - + if hasChildren: firstChild = self.getFirstChild(currentNode) else: firstChild = None - + if firstChild is not None: currentNode = firstChild else: diff --git a/libs/html5lib/treewalkers/dom.py b/libs/html5lib/treewalkers/dom.py index 383b46cb..a01287a9 100644 --- a/libs/html5lib/treewalkers/dom.py +++ b/libs/html5lib/treewalkers/dom.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + from xml.dom import Node import gettext _ = gettext.gettext -import _base -from html5lib.constants import voidElements +from . import _base + class TreeWalker(_base.NonRecursiveTreeWalker): def getNodeDetails(self, node): @@ -16,10 +18,13 @@ class TreeWalker(_base.NonRecursiveTreeWalker): elif node.nodeType == Node.ELEMENT_NODE: attrs = {} - for attr in node.attributes.keys(): + for attr in list(node.attributes.keys()): attr = node.getAttributeNode(attr) - attrs[(attr.namespaceURI,attr.localName)] = attr.value - return (_base.ELEMENT, node.namespaceURI, node.nodeName, + if attr.namespaceURI: + attrs[(attr.namespaceURI, attr.localName)] = attr.value + else: + attrs[(None, attr.name)] = attr.value + return (_base.ELEMENT, node.namespaceURI, node.nodeName, attrs, node.hasChildNodes()) elif node.nodeType == Node.COMMENT_NODE: diff --git a/libs/html5lib/treewalkers/etree.py b/libs/html5lib/treewalkers/etree.py index 13b03194..fd8a9cc9 100644 --- a/libs/html5lib/treewalkers/etree.py +++ b/libs/html5lib/treewalkers/etree.py @@ -1,33 +1,28 @@ +from __future__ import absolute_import, division, unicode_literals + +try: + from collections import OrderedDict +except ImportError: + try: + from ordereddict import OrderedDict + except ImportError: + OrderedDict = dict import gettext _ = gettext.gettext -try: - from types import ModuleType -except: - from new import module as ModuleType -import copy import re -import _base -from html5lib.constants import voidElements +from six import text_type + +from . import _base +from ..utils import moduleFactoryFactory tag_regexp = re.compile("{([^}]*)}(.*)") -moduleCache = {} - -def getETreeModule(ElementTreeImplementation): - name = "_" + ElementTreeImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder") - objs = getETreeBuilder(ElementTreeImplementation) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getETreeBuilder(ElementTreeImplementation): ElementTree = ElementTreeImplementation + ElementTreeCommentType = ElementTree.Comment("asd").tag class TreeWalker(_base.NonRecursiveTreeWalker): """Given the particular ElementTree representation, this implementation, @@ -35,16 +30,16 @@ def getETreeBuilder(ElementTreeImplementation): content: 1. The current element - + 2. The index of the element relative to its parent - + 3. A stack of ancestor elements - + 4. A flag "text", "tail" or None to indicate if the current node is a text node; either the text or tail of the current element (1) """ def getNodeDetails(self, node): - if isinstance(node, tuple): # It might be the root Element + if isinstance(node, tuple): # It might be the root Element elt, key, parents, flag = node if flag in ("text", "tail"): return _base.TEXT, getattr(elt, flag) @@ -54,41 +49,41 @@ def getETreeBuilder(ElementTreeImplementation): if not(hasattr(node, "tag")): node = node.getroot() - if node.tag in ("<DOCUMENT_ROOT>", "<DOCUMENT_FRAGMENT>"): + if node.tag in ("DOCUMENT_ROOT", "DOCUMENT_FRAGMENT"): return (_base.DOCUMENT,) elif node.tag == "<!DOCTYPE>": - return (_base.DOCTYPE, node.text, + return (_base.DOCTYPE, node.text, node.get("publicId"), node.get("systemId")) - elif node.tag == ElementTree.Comment: + elif node.tag == ElementTreeCommentType: return _base.COMMENT, node.text else: - assert type(node.tag) in (str, unicode), type(node.tag) - #This is assumed to be an ordinary element + assert type(node.tag) == text_type, type(node.tag) + # This is assumed to be an ordinary element match = tag_regexp.match(node.tag) if match: namespace, tag = match.groups() else: namespace = None tag = node.tag - attrs = {} - for name, value in node.attrib.items(): + attrs = OrderedDict() + for name, value in list(node.attrib.items()): match = tag_regexp.match(name) if match: - attrs[(match.group(1),match.group(2))] = value + attrs[(match.group(1), match.group(2))] = value else: - attrs[(None,name)] = value - return (_base.ELEMENT, namespace, tag, + attrs[(None, name)] = value + return (_base.ELEMENT, namespace, tag, attrs, len(node) or node.text) - + def getFirstChild(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: element, key, parents, flag = node, None, [], None - + if flag in ("text", "tail"): return None else: @@ -99,13 +94,13 @@ def getETreeBuilder(ElementTreeImplementation): return element[0], 0, parents, None else: return None - + def getNextSibling(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: return None - + if flag == "text": if len(element): parents.append(element) @@ -116,16 +111,16 @@ def getETreeBuilder(ElementTreeImplementation): if element.tail and flag != "tail": return element, key, parents, "tail" elif key < len(parents[-1]) - 1: - return parents[-1][key+1], key+1, parents, None + return parents[-1][key + 1], key + 1, parents, None else: return None - + def getParentNode(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: return None - + if flag == "text": if not parents: return element @@ -139,3 +134,5 @@ def getETreeBuilder(ElementTreeImplementation): return parent, list(parents[-1]).index(parent), parents, None return locals() + +getETreeModule = moduleFactoryFactory(getETreeBuilder) diff --git a/libs/html5lib/treewalkers/genshistream.py b/libs/html5lib/treewalkers/genshistream.py index ef71a83e..f559c45d 100644 --- a/libs/html5lib/treewalkers/genshistream.py +++ b/libs/html5lib/treewalkers/genshistream.py @@ -1,50 +1,49 @@ +from __future__ import absolute_import, division, unicode_literals + +from genshi.core import QName from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT -from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT -from genshi.output import NamespaceFlattener +from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT -import _base +from . import _base + +from ..constants import voidElements, namespaces -from html5lib.constants import voidElements class TreeWalker(_base.TreeWalker): def __iter__(self): - depth = 0 - ignore_until = None + # Buffer the events so we can pass in the following one previous = None for event in self.tree: if previous is not None: - if previous[0] == START: - depth += 1 - if ignore_until <= depth: - ignore_until = None - if ignore_until is None: - for token in self.tokens(previous, event): - yield token - if token["type"] == "EmptyTag": - ignore_until = depth - if previous[0] == END: - depth -= 1 - previous = event - if previous is not None: - if ignore_until is None or ignore_until <= depth: - for token in self.tokens(previous, None): + for token in self.tokens(previous, event): yield token - elif ignore_until is not None: - raise ValueError("Illformed DOM event stream: void element without END_ELEMENT") + previous = event + + # Don't forget the final event! + if previous is not None: + for token in self.tokens(previous, None): + yield token def tokens(self, event, next): kind, data, pos = event if kind == START: - tag, attrib = data + tag, attribs = data name = tag.localname namespace = tag.namespace - if tag in voidElements: - for token in self.emptyTag(namespace, name, list(attrib), - not next or next[0] != END + converted_attribs = {} + for k, v in attribs: + if isinstance(k, QName): + converted_attribs[(k.namespace, k.localname)] = v + else: + converted_attribs[(None, k)] = v + + if namespace == namespaces["html"] and name in voidElements: + for token in self.emptyTag(namespace, name, converted_attribs, + not next or next[0] != END or next[1] != tag): yield token else: - yield self.startTag(namespace, name, list(attrib)) + yield self.startTag(namespace, name, converted_attribs) elif kind == END: name = data.localname @@ -62,8 +61,8 @@ class TreeWalker(_base.TreeWalker): elif kind == DOCTYPE: yield self.doctype(*data) - elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \ - START_CDATA, END_CDATA, PI): + elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, + START_CDATA, END_CDATA, PI): pass else: diff --git a/libs/html5lib/treewalkers/lxmletree.py b/libs/html5lib/treewalkers/lxmletree.py index 46f4908c..375cc2e8 100644 --- a/libs/html5lib/treewalkers/lxmletree.py +++ b/libs/html5lib/treewalkers/lxmletree.py @@ -1,186 +1,208 @@ -from lxml import etree -from html5lib.treebuilders.etree import tag_regexp - -from gettext import gettext -_ = gettext - -import _base - -from html5lib.constants import voidElements -from html5lib import ihatexml - -class Root(object): - def __init__(self, et): - self.elementtree = et - self.children = [] - if et.docinfo.internalDTD: - self.children.append(Doctype(self, et.docinfo.root_name, - et.docinfo.public_id, - et.docinfo.system_url)) - root = et.getroot() - node = root - - while node.getprevious() is not None: - node = node.getprevious() - while node is not None: - self.children.append(node) - node = node.getnext() - - self.text = None - self.tail = None - - def __getitem__(self, key): - return self.children[key] - - def getnext(self): - return None - - def __len__(self): - return 1 - -class Doctype(object): - def __init__(self, root_node, name, public_id, system_id): - self.root_node = root_node - self.name = name - self.public_id = public_id - self.system_id = system_id - - self.text = None - self.tail = None - - def getnext(self): - return self.root_node.children[1] - -class FragmentRoot(Root): - def __init__(self, children): - self.children = [FragmentWrapper(self, child) for child in children] - self.text = self.tail = None - - def getnext(self): - return None - -class FragmentWrapper(object): - def __init__(self, fragment_root, obj): - self.root_node = fragment_root - self.obj = obj - if hasattr(self.obj, 'text'): - self.text = self.obj.text - else: - self.text = None - if hasattr(self.obj, 'tail'): - self.tail = self.obj.tail - else: - self.tail = None - self.isstring = isinstance(obj, basestring) - - def __getattr__(self, name): - return getattr(self.obj, name) - - def getnext(self): - siblings = self.root_node.children - idx = siblings.index(self) - if idx < len(siblings) - 1: - return siblings[idx + 1] - else: - return None - - def __getitem__(self, key): - return self.obj[key] - - def __nonzero__(self): - return bool(self.obj) - - def getparent(self): - return None - - def __str__(self): - return str(self.obj) - - def __unicode__(self): - return unicode(self.obj) - - def __len__(self): - return len(self.obj) - - -class TreeWalker(_base.NonRecursiveTreeWalker): - def __init__(self, tree): - if hasattr(tree, "getroot"): - tree = Root(tree) - elif isinstance(tree, list): - tree = FragmentRoot(tree) - _base.NonRecursiveTreeWalker.__init__(self, tree) - self.filter = ihatexml.InfosetFilter() - def getNodeDetails(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - return _base.TEXT, getattr(node, key) - - elif isinstance(node, Root): - return (_base.DOCUMENT,) - - elif isinstance(node, Doctype): - return _base.DOCTYPE, node.name, node.public_id, node.system_id - - elif isinstance(node, FragmentWrapper) and node.isstring: - return _base.TEXT, node - - elif node.tag == etree.Comment: - return _base.COMMENT, node.text - - elif node.tag == etree.Entity: - return _base.ENTITY, node.text[1:-1] # strip &; - - else: - #This is assumed to be an ordinary element - match = tag_regexp.match(node.tag) - if match: - namespace, tag = match.groups() - else: - namespace = None - tag = node.tag - attrs = {} - for name, value in node.attrib.items(): - match = tag_regexp.match(name) - if match: - attrs[(match.group(1),match.group(2))] = value - else: - attrs[(None,name)] = value - return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), - attrs, len(node) > 0 or node.text) - - def getFirstChild(self, node): - assert not isinstance(node, tuple), _("Text nodes have no children") - - assert len(node) or node.text, "Node has no children" - if node.text: - return (node, "text") - else: - return node[0] - - def getNextSibling(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - if key == "text": - # XXX: we cannot use a "bool(node) and node[0] or None" construct here - # because node[0] might evaluate to False if it has no child element - if len(node): - return node[0] - else: - return None - else: # tail - return node.getnext() - - return node.tail and (node, "tail") or node.getnext() - - def getParentNode(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - if key == "text": - return node - # else: fallback to "normal" processing - - return node.getparent() +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from lxml import etree +from ..treebuilders.etree import tag_regexp + +from gettext import gettext +_ = gettext + +from . import _base + +from .. import ihatexml + + +def ensure_str(s): + if s is None: + return None + elif isinstance(s, text_type): + return s + else: + return s.decode("utf-8", "strict") + + +class Root(object): + def __init__(self, et): + self.elementtree = et + self.children = [] + if et.docinfo.internalDTD: + self.children.append(Doctype(self, + ensure_str(et.docinfo.root_name), + ensure_str(et.docinfo.public_id), + ensure_str(et.docinfo.system_url))) + root = et.getroot() + node = root + + while node.getprevious() is not None: + node = node.getprevious() + while node is not None: + self.children.append(node) + node = node.getnext() + + self.text = None + self.tail = None + + def __getitem__(self, key): + return self.children[key] + + def getnext(self): + return None + + def __len__(self): + return 1 + + +class Doctype(object): + def __init__(self, root_node, name, public_id, system_id): + self.root_node = root_node + self.name = name + self.public_id = public_id + self.system_id = system_id + + self.text = None + self.tail = None + + def getnext(self): + return self.root_node.children[1] + + +class FragmentRoot(Root): + def __init__(self, children): + self.children = [FragmentWrapper(self, child) for child in children] + self.text = self.tail = None + + def getnext(self): + return None + + +class FragmentWrapper(object): + def __init__(self, fragment_root, obj): + self.root_node = fragment_root + self.obj = obj + if hasattr(self.obj, 'text'): + self.text = ensure_str(self.obj.text) + else: + self.text = None + if hasattr(self.obj, 'tail'): + self.tail = ensure_str(self.obj.tail) + else: + self.tail = None + self.isstring = isinstance(obj, str) or isinstance(obj, bytes) + # Support for bytes here is Py2 + if self.isstring: + self.obj = ensure_str(self.obj) + + def __getattr__(self, name): + return getattr(self.obj, name) + + def getnext(self): + siblings = self.root_node.children + idx = siblings.index(self) + if idx < len(siblings) - 1: + return siblings[idx + 1] + else: + return None + + def __getitem__(self, key): + return self.obj[key] + + def __bool__(self): + return bool(self.obj) + + def getparent(self): + return None + + def __str__(self): + return str(self.obj) + + def __unicode__(self): + return str(self.obj) + + def __len__(self): + return len(self.obj) + + +class TreeWalker(_base.NonRecursiveTreeWalker): + def __init__(self, tree): + if hasattr(tree, "getroot"): + tree = Root(tree) + elif isinstance(tree, list): + tree = FragmentRoot(tree) + _base.NonRecursiveTreeWalker.__init__(self, tree) + self.filter = ihatexml.InfosetFilter() + + def getNodeDetails(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + return _base.TEXT, ensure_str(getattr(node, key)) + + elif isinstance(node, Root): + return (_base.DOCUMENT,) + + elif isinstance(node, Doctype): + return _base.DOCTYPE, node.name, node.public_id, node.system_id + + elif isinstance(node, FragmentWrapper) and node.isstring: + return _base.TEXT, node.obj + + elif node.tag == etree.Comment: + return _base.COMMENT, ensure_str(node.text) + + elif node.tag == etree.Entity: + return _base.ENTITY, ensure_str(node.text)[1:-1] # strip &; + + else: + # This is assumed to be an ordinary element + match = tag_regexp.match(ensure_str(node.tag)) + if match: + namespace, tag = match.groups() + else: + namespace = None + tag = ensure_str(node.tag) + attrs = {} + for name, value in list(node.attrib.items()): + name = ensure_str(name) + value = ensure_str(value) + match = tag_regexp.match(name) + if match: + attrs[(match.group(1), match.group(2))] = value + else: + attrs[(None, name)] = value + return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), + attrs, len(node) > 0 or node.text) + + def getFirstChild(self, node): + assert not isinstance(node, tuple), _("Text nodes have no children") + + assert len(node) or node.text, "Node has no children" + if node.text: + return (node, "text") + else: + return node[0] + + def getNextSibling(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + if key == "text": + # XXX: we cannot use a "bool(node) and node[0] or None" construct here + # because node[0] might evaluate to False if it has no child element + if len(node): + return node[0] + else: + return None + else: # tail + return node.getnext() + + return (node, "tail") if node.tail else node.getnext() + + def getParentNode(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + if key == "text": + return node + # else: fallback to "normal" processing + + return node.getparent() diff --git a/libs/html5lib/treewalkers/pulldom.py b/libs/html5lib/treewalkers/pulldom.py index 1f8b95b8..0b0f515f 100644 --- a/libs/html5lib/treewalkers/pulldom.py +++ b/libs/html5lib/treewalkers/pulldom.py @@ -1,9 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \ COMMENT, IGNORABLE_WHITESPACE, CHARACTERS -import _base +from . import _base + +from ..constants import voidElements -from html5lib.constants import voidElements class TreeWalker(_base.TreeWalker): def __iter__(self): @@ -11,7 +14,7 @@ class TreeWalker(_base.TreeWalker): previous = None for event in self.tree: if previous is not None and \ - (ignore_until is None or previous[1] is ignore_until): + (ignore_until is None or previous[1] is ignore_until): if previous[1] is ignore_until: ignore_until = None for token in self.tokens(previous, event): @@ -31,9 +34,9 @@ class TreeWalker(_base.TreeWalker): name = node.nodeName namespace = node.namespaceURI attrs = {} - for attr in node.attributes.keys(): + for attr in list(node.attributes.keys()): attr = node.getAttributeNode(attr) - attrs[(attr.namespaceURI,attr.localName)] = attr.value + attrs[(attr.namespaceURI, attr.localName)] = attr.value if name in voidElements: for token in self.emptyTag(namespace, name, diff --git a/libs/html5lib/treewalkers/simpletree.py b/libs/html5lib/treewalkers/simpletree.py deleted file mode 100644 index 9e6bd4c5..00000000 --- a/libs/html5lib/treewalkers/simpletree.py +++ /dev/null @@ -1,78 +0,0 @@ -import gettext -_ = gettext.gettext - -import _base - -class TreeWalker(_base.NonRecursiveTreeWalker): - """Given that simpletree has no performant way of getting a node's - next sibling, this implementation returns "nodes" as tuples with the - following content: - - 1. The parent Node (Element, Document or DocumentFragment) - - 2. The child index of the current node in its parent's children list - - 3. A list used as a stack of all ancestors. It is a pair tuple whose - first item is a parent Node and second item is a child index. - """ - - def getNodeDetails(self, node): - if isinstance(node, tuple): # It might be the root Node - parent, idx, parents = node - node = parent.childNodes[idx] - - # testing node.type allows us not to import treebuilders.simpletree - if node.type in (1, 2): # Document or DocumentFragment - return (_base.DOCUMENT,) - - elif node.type == 3: # DocumentType - return _base.DOCTYPE, node.name, node.publicId, node.systemId - - elif node.type == 4: # TextNode - return _base.TEXT, node.value - - elif node.type == 5: # Element - attrs = {} - for name, value in node.attributes.items(): - if isinstance(name, tuple): - attrs[(name[2],name[1])] = value - else: - attrs[(None,name)] = value - return (_base.ELEMENT, node.namespace, node.name, - attrs, node.hasContent()) - - elif node.type == 6: # CommentNode - return _base.COMMENT, node.data - - else: - return _node.UNKNOWN, node.type - - def getFirstChild(self, node): - if isinstance(node, tuple): # It might be the root Node - parent, idx, parents = node - parents.append((parent, idx)) - node = parent.childNodes[idx] - else: - parents = [] - - assert node.hasContent(), "Node has no children" - return (node, 0, parents) - - def getNextSibling(self, node): - assert isinstance(node, tuple), "Node is not a tuple: " + str(node) - parent, idx, parents = node - idx += 1 - if len(parent.childNodes) > idx: - return (parent, idx, parents) - else: - return None - - def getParentNode(self, node): - assert isinstance(node, tuple) - parent, idx, parents = node - if parents: - parent, idx = parents.pop() - return parent, idx, parents - else: - # HACK: We could return ``parent`` but None will stop the algorithm the same way - return None diff --git a/libs/html5lib/treewalkers/soup.py b/libs/html5lib/treewalkers/soup.py deleted file mode 100644 index fca65ecb..00000000 --- a/libs/html5lib/treewalkers/soup.py +++ /dev/null @@ -1,60 +0,0 @@ -import re -import gettext -_ = gettext.gettext - -from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag -from html5lib.constants import namespaces -import _base - -class TreeWalker(_base.NonRecursiveTreeWalker): - doctype_regexp = re.compile( - r'DOCTYPE\s+(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?') - def getNodeDetails(self, node): - if isinstance(node, BeautifulSoup): # Document or DocumentFragment - return (_base.DOCUMENT,) - - elif isinstance(node, Declaration): # DocumentType - string = unicode(node.string) - #Slice needed to remove markup added during unicode conversion, - #but only in some versions of BeautifulSoup/Python - if string.startswith('<!') and string.endswith('>'): - string = string[2:-1] - m = self.doctype_regexp.match(string) - #This regexp approach seems wrong and fragile - #but beautiful soup stores the doctype as a single thing and we want the seperate bits - #It should work as long as the tree is created by html5lib itself but may be wrong if it's - #been modified at all - #We could just feed to it a html5lib tokenizer, I guess... - assert m is not None, "DOCTYPE did not match expected format" - - name = m.group('name') - publicId = m.group('publicId') - if publicId is not None: - systemId = m.group('systemId1') - else: - systemId = m.group('systemId2') - return _base.DOCTYPE, name, publicId or "", systemId or "" - - elif isinstance(node, Comment): - string = unicode(node.string) - if string.startswith('<!--') and string.endswith('-->'): - string = string[4:-3] - return _base.COMMENT, string - - elif isinstance(node, unicode): # TextNode - return _base.TEXT, node - - elif isinstance(node, Tag): # Element - return (_base.ELEMENT, namespaces["html"], node.name, - dict(node.attrs).items(), node.contents) - else: - return _base.UNKNOWN, node.__class__.__name__ - - def getFirstChild(self, node): - return node.contents[0] - - def getNextSibling(self, node): - return node.nextSibling - - def getParentNode(self, node): - return node.parent diff --git a/libs/html5lib/trie/__init__.py b/libs/html5lib/trie/__init__.py new file mode 100644 index 00000000..a8cca8a9 --- /dev/null +++ b/libs/html5lib/trie/__init__.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + +from .py import Trie as PyTrie + +Trie = PyTrie + +try: + from .datrie import Trie as DATrie +except ImportError: + pass +else: + Trie = DATrie diff --git a/libs/html5lib/trie/_base.py b/libs/html5lib/trie/_base.py new file mode 100644 index 00000000..724486b1 --- /dev/null +++ b/libs/html5lib/trie/_base.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import, division, unicode_literals + +from collections import Mapping + + +class Trie(Mapping): + """Abstract base class for tries""" + + def keys(self, prefix=None): + keys = super().keys() + + if prefix is None: + return set(keys) + + # Python 2.6: no set comprehensions + return set([x for x in keys if x.startswith(prefix)]) + + def has_keys_with_prefix(self, prefix): + for key in self.keys(): + if key.startswith(prefix): + return True + + return False + + def longest_prefix(self, prefix): + if prefix in self: + return prefix + + for i in range(1, len(prefix) + 1): + if prefix[:-i] in self: + return prefix[:-i] + + raise KeyError(prefix) + + def longest_prefix_item(self, prefix): + lprefix = self.longest_prefix(prefix) + return (lprefix, self[lprefix]) diff --git a/libs/html5lib/trie/datrie.py b/libs/html5lib/trie/datrie.py new file mode 100644 index 00000000..51f3d046 --- /dev/null +++ b/libs/html5lib/trie/datrie.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import, division, unicode_literals + +from datrie import Trie as DATrie +from six import text_type + +from ._base import Trie as ABCTrie + + +class Trie(ABCTrie): + def __init__(self, data): + chars = set() + for key in data.keys(): + if not isinstance(key, text_type): + raise TypeError("All keys must be strings") + for char in key: + chars.add(char) + + self._data = DATrie("".join(chars)) + for key, value in data.items(): + self._data[key] = value + + def __contains__(self, key): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + raise NotImplementedError() + + def __getitem__(self, key): + return self._data[key] + + def keys(self, prefix=None): + return self._data.keys(prefix) + + def has_keys_with_prefix(self, prefix): + return self._data.has_keys_with_prefix(prefix) + + def longest_prefix(self, prefix): + return self._data.longest_prefix(prefix) + + def longest_prefix_item(self, prefix): + return self._data.longest_prefix_item(prefix) diff --git a/libs/html5lib/trie/py.py b/libs/html5lib/trie/py.py new file mode 100644 index 00000000..c2ba3da7 --- /dev/null +++ b/libs/html5lib/trie/py.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from bisect import bisect_left + +from ._base import Trie as ABCTrie + + +class Trie(ABCTrie): + def __init__(self, data): + if not all(isinstance(x, text_type) for x in data.keys()): + raise TypeError("All keys must be strings") + + self._data = data + self._keys = sorted(data.keys()) + self._cachestr = "" + self._cachepoints = (0, len(data)) + + def __contains__(self, key): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + return iter(self._data) + + def __getitem__(self, key): + return self._data[key] + + def keys(self, prefix=None): + if prefix is None or prefix == "" or not self._keys: + return set(self._keys) + + if prefix.startswith(self._cachestr): + lo, hi = self._cachepoints + start = i = bisect_left(self._keys, prefix, lo, hi) + else: + start = i = bisect_left(self._keys, prefix) + + keys = set() + if start == len(self._keys): + return keys + + while self._keys[i].startswith(prefix): + keys.add(self._keys[i]) + i += 1 + + self._cachestr = prefix + self._cachepoints = (start, i) + + return keys + + def has_keys_with_prefix(self, prefix): + if prefix in self._data: + return True + + if prefix.startswith(self._cachestr): + lo, hi = self._cachepoints + i = bisect_left(self._keys, prefix, lo, hi) + else: + i = bisect_left(self._keys, prefix) + + if i == len(self._keys): + return False + + return self._keys[i].startswith(prefix) diff --git a/libs/html5lib/utils.py b/libs/html5lib/utils.py index d53f6788..2f41f4df 100644 --- a/libs/html5lib/utils.py +++ b/libs/html5lib/utils.py @@ -1,9 +1,16 @@ +from __future__ import absolute_import, division, unicode_literals + +from types import ModuleType + try: - frozenset -except NameError: - #Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset + import xml.etree.cElementTree as default_etree +except ImportError: + import xml.etree.ElementTree as default_etree + + +__all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair", + "surrogatePairToCodepoint", "moduleFactoryFactory"] + class MethodDispatcher(dict): """Dict with 2 special properties: @@ -23,7 +30,7 @@ class MethodDispatcher(dict): # twice as fast. Please do careful performance testing before changing # anything here. _dictEntries = [] - for name,value in items: + for name, value in items: if type(name) in (list, tuple, frozenset, set): for item in name: _dictEntries.append((item, value)) @@ -35,141 +42,41 @@ class MethodDispatcher(dict): def __getitem__(self, key): return dict.get(self, key, self.default) -#Pure python implementation of deque taken from the ASPN Python Cookbook -#Original code by Raymond Hettinger -class deque(object): +# Some utility functions to dal with weirdness around UCS2 vs UCS4 +# python builds - def __init__(self, iterable=(), maxsize=-1): - if not hasattr(self, 'data'): - self.left = self.right = 0 - self.data = {} - self.maxsize = maxsize - self.extend(iterable) - - def append(self, x): - self.data[self.right] = x - self.right += 1 - if self.maxsize != -1 and len(self) > self.maxsize: - self.popleft() - - def appendleft(self, x): - self.left -= 1 - self.data[self.left] = x - if self.maxsize != -1 and len(self) > self.maxsize: - self.pop() - - def pop(self): - if self.left == self.right: - raise IndexError('cannot pop from empty deque') - self.right -= 1 - elem = self.data[self.right] - del self.data[self.right] - return elem - - def popleft(self): - if self.left == self.right: - raise IndexError('cannot pop from empty deque') - elem = self.data[self.left] - del self.data[self.left] - self.left += 1 - return elem - - def clear(self): - self.data.clear() - self.left = self.right = 0 - - def extend(self, iterable): - for elem in iterable: - self.append(elem) - - def extendleft(self, iterable): - for elem in iterable: - self.appendleft(elem) - - def rotate(self, n=1): - if self: - n %= len(self) - for i in xrange(n): - self.appendleft(self.pop()) - - def __getitem__(self, i): - if i < 0: - i += len(self) - try: - return self.data[i + self.left] - except KeyError: - raise IndexError - - def __setitem__(self, i, value): - if i < 0: - i += len(self) - try: - self.data[i + self.left] = value - except KeyError: - raise IndexError - - def __delitem__(self, i): - size = len(self) - if not (-size <= i < size): - raise IndexError - data = self.data - if i < 0: - i += size - for j in xrange(self.left+i, self.right-1): - data[j] = data[j+1] - self.pop() - - def __len__(self): - return self.right - self.left - - def __cmp__(self, other): - if type(self) != type(other): - return cmp(type(self), type(other)) - return cmp(list(self), list(other)) - - def __repr__(self, _track=[]): - if id(self) in _track: - return '...' - _track.append(id(self)) - r = 'deque(%r)' % (list(self),) - _track.remove(id(self)) - return r - - def __getstate__(self): - return (tuple(self),) - - def __setstate__(self, s): - self.__init__(s[0]) - - def __hash__(self): - raise TypeError - - def __copy__(self): - return self.__class__(self) - - def __deepcopy__(self, memo={}): - from copy import deepcopy - result = self.__class__() - memo[id(self)] = result - result.__init__(deepcopy(tuple(self), memo)) - return result - -#Some utility functions to dal with weirdness around UCS2 vs UCS4 -#python builds - -def encodingType(): - if len() == 2: - return "UCS2" - else: - return "UCS4" - -def isSurrogatePair(data): +def isSurrogatePair(data): return (len(data) == 2 and ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF) + def surrogatePairToCodepoint(data): - char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + + char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + (ord(data[1]) - 0xDC00)) return char_val + +# Module Factory Factory (no, this isn't Java, I know) +# Here to stop this being duplicated all over the place. + + +def moduleFactoryFactory(factory): + moduleCache = {} + + def moduleFactory(baseModule, *args, **kwargs): + if isinstance(ModuleType.__name__, type("")): + name = "_%s_factory" % baseModule.__name__ + else: + name = b"_%s_factory" % baseModule.__name__ + + if name in moduleCache: + return moduleCache[name] + else: + mod = ModuleType(name) + objs = factory(baseModule, *args, **kwargs) + mod.__dict__.update(objs) + moduleCache[name] = mod + return mod + + return moduleFactory diff --git a/libs/httplib2/__init__.py b/libs/httplib2/__init__.py index 01151f7f..9780d4e5 100644 --- a/libs/httplib2/__init__.py +++ b/libs/httplib2/__init__.py @@ -3,7 +3,7 @@ from __future__ import generators httplib2 A caching http interface that supports ETags and gzip -to conserve bandwidth. +to conserve bandwidth. Requires Python 2.3 or later @@ -15,17 +15,17 @@ Changelog: __author__ = "Joe Gregorio (joe@bitworking.org)" __copyright__ = "Copyright 2006, Joe Gregorio" __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", - "James Antill", - "Xavier Verges Farrero", - "Jonathan Feinberg", - "Blair Zajac", - "Sam Ruby", - "Louis Nyffenegger"] + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] __license__ = "MIT" -__version__ = "$Rev$" +__version__ = "0.8" -import re -import sys +import re +import sys import email import email.Utils import email.Message @@ -35,6 +35,7 @@ import gzip import zlib import httplib import urlparse +import urllib import base64 import os import copy @@ -42,10 +43,10 @@ import calendar import time import random import errno -# remove depracated warning in python2.6 try: from hashlib import sha1 as _sha, md5 as _md5 except ImportError: + # prior to Python 2.5, these were separate modules import sha import md5 _sha = sha.new @@ -54,21 +55,38 @@ import hmac from gettext import gettext as _ import socket -# Try using local version, followed by system, and none if neither are found try: - import lib.socks as socks + from httplib2 import socks except ImportError: try: - import socks as socks - except ImportError: + import socks + except (ImportError, AttributeError): socks = None # Build the appropriate socket wrapper for ssl try: import ssl # python 2.6 - _ssl_wrap_socket = ssl.wrap_socket -except ImportError: - def _ssl_wrap_socket(sock, key_file, cert_file): + ssl_SSLError = ssl.SSLError + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if disable_validation: + cert_reqs = ssl.CERT_NONE + else: + cert_reqs = ssl.CERT_REQUIRED + # We should be specifying SSL version 3 or TLS v1, but the ssl module + # doesn't expose the necessary knobs. So we need to go with the default + # of SSLv23. + return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, + cert_reqs=cert_reqs, ca_certs=ca_certs) +except (AttributeError, ImportError): + ssl_SSLError = None + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if not disable_validation: + raise CertificateValidationUnsupported( + "SSL certificate validation is not supported without " + "the ssl module installed. To avoid this error, install " + "the ssl module, or explicity disable validation.") ssl_sock = socket.ssl(sock, key_file, cert_file) return httplib.FakeSocket(sock, ssl_sock) @@ -84,15 +102,19 @@ def has_timeout(timeout): # python 2.6 return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) return (timeout is not None) -__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', - 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', - 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', - 'debuglevel'] +__all__ = [ + 'Http', 'Response', 'ProxyInfo', 'HttpLib2Error', 'RedirectMissingLocation', + 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', + 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel', 'ProxiesUnavailableError'] # The httplib debug level, set to a non-zero value to get debug output debuglevel = 0 +# A request will be tried 'RETRIES' times if it fails at the socket/connection level. +RETRIES = 2 # Python 2.3 support if sys.version_info < (2,4): @@ -113,8 +135,8 @@ if not hasattr(httplib.HTTPResponse, 'getheaders'): # All exceptions raised here derive from HttpLib2Error class HttpLib2Error(Exception): pass -# Some exceptions can be caught and optionally -# be turned back into responses. +# Some exceptions can be caught and optionally +# be turned back into responses. class HttpLib2ErrorWithResponse(HttpLib2Error): def __init__(self, desc, response, content): self.response = response @@ -127,8 +149,18 @@ class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class MalformedHeader(HttpLib2Error): pass class RelativeURIError(HttpLib2Error): pass class ServerNotFoundError(HttpLib2Error): pass +class ProxiesUnavailableError(HttpLib2Error): pass +class CertificateValidationUnsupported(HttpLib2Error): pass +class SSLHandshakeError(HttpLib2Error): pass +class NotSupportedOnThisPlatform(HttpLib2Error): pass +class CertificateHostnameMismatch(SSLHandshakeError): + def __init__(self, desc, host, cert): + HttpLib2Error.__init__(self, desc) + self.host = host + self.cert = cert # Open Items: # ----------- @@ -152,6 +184,16 @@ class ServerNotFoundError(HttpLib2Error): pass # requesting that URI again. DEFAULT_MAX_REDIRECTS = 5 +try: + # Users can optionally provide a module that tells us where the CA_CERTS + # are located. + import ca_certs_locater + CA_CERTS = ca_certs_locater.get() +except ImportError: + # Default CA certificates file bundled with httplib2. + CA_CERTS = os.path.join( + os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt") + # Which headers are hop-by-hop headers by default HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] @@ -176,7 +218,7 @@ def urlnorm(uri): raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) authority = authority.lower() scheme = scheme.lower() - if not path: + if not path: path = "/" # Could do syntax based normalization of the URI before # computing the digest. See Section 6.2.2 of Std 66. @@ -228,7 +270,7 @@ def _parse_cache_control(headers): parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] retval = dict(parts_with_args + parts_wo_args) - return retval + return retval # Whether to use a strict mode to parse WWW-Authenticate headers # Might lead to bad results in case of ill-formed header value, @@ -249,25 +291,30 @@ def _parse_www_authenticate(headers, headername='www-authenticate'): per auth_scheme.""" retval = {} if headers.has_key(headername): - authenticate = headers[headername].strip() - www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED - while authenticate: - # Break off the scheme at the beginning of the line - if headername == 'authentication-info': - (auth_scheme, the_rest) = ('digest', authenticate) - else: - (auth_scheme, the_rest) = authenticate.split(" ", 1) - # Now loop over all the key value pairs that come after the scheme, - # being careful not to roll into the next scheme - match = www_auth.search(the_rest) - auth_params = {} - while match: - if match and len(match.groups()) == 3: - (key, value, the_rest) = match.groups() - auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) + try: + + authenticate = headers[headername].strip() + www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED + while authenticate: + # Break off the scheme at the beginning of the line + if headername == 'authentication-info': + (auth_scheme, the_rest) = ('digest', authenticate) + else: + (auth_scheme, the_rest) = authenticate.split(" ", 1) + # Now loop over all the key value pairs that come after the scheme, + # being careful not to roll into the next scheme match = www_auth.search(the_rest) - retval[auth_scheme.lower()] = auth_params - authenticate = the_rest.strip() + auth_params = {} + while match: + if match and len(match.groups()) == 3: + (key, value, the_rest) = match.groups() + auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) + match = www_auth.search(the_rest) + retval[auth_scheme.lower()] = auth_params + authenticate = the_rest.strip() + + except ValueError: + raise MalformedHeader("WWW-Authenticate") return retval @@ -279,17 +326,17 @@ def _entry_disposition(response_headers, request_headers): 1. Cache-Control: max-stale 2. Age: headers are not used in the calculations. - Not that this algorithm is simpler than you might think + Not that this algorithm is simpler than you might think because we are operating as a private (non-shared) cache. This lets us ignore 's-maxage'. We can also ignore 'proxy-invalidate' since we aren't a proxy. - We will never return a stale document as - fresh as a design decision, and thus the non-implementation - of 'max-stale'. This also lets us safely ignore 'must-revalidate' + We will never return a stale document as + fresh as a design decision, and thus the non-implementation + of 'max-stale'. This also lets us safely ignore 'must-revalidate' since we operate as if every server has sent 'must-revalidate'. Since we are private we get to ignore both 'public' and 'private' parameters. We also ignore 'no-transform' since - we don't do any transformations. + we don't do any transformations. The 'no-store' parameter is handled at a higher level. So the only Cache-Control parameters we look at are: @@ -298,7 +345,7 @@ def _entry_disposition(response_headers, request_headers): max-age min-fresh """ - + retval = "STALE" cc = _parse_cache_control(request_headers) cc_response = _parse_cache_control(response_headers) @@ -340,10 +387,10 @@ def _entry_disposition(response_headers, request_headers): min_fresh = int(cc['min-fresh']) except ValueError: min_fresh = 0 - current_age += min_fresh + current_age += min_fresh if freshness_lifetime > current_age: retval = "FRESH" - return retval + return retval def _decompressContent(response, new_content): content = new_content @@ -391,7 +438,7 @@ def _updateCache(request_headers, response_headers, content, cache, cachekey): if status == 304: status = 200 - status_header = 'status: %d\r\n' % response_headers.status + status_header = 'status: %d\r\n' % status header_str = info.as_string() @@ -408,10 +455,10 @@ def _wsse_username_token(cnonce, iso_now, password): return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip() -# For credentials we need two things, first +# For credentials we need two things, first # a pool of credential to try (not necesarily tied to BAsic, Digest, etc.) # Then we also need a list of URIs that have already demanded authentication -# That list is tricky since sub-URIs can take the same auth, or the +# That list is tricky since sub-URIs can take the same auth, or the # auth scheme may change as you descend the tree. # So we also need each Auth instance to be able to tell us # how close to the 'top' it is. @@ -435,7 +482,7 @@ class Authentication(object): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate - Authorization header. Over-rise this in sub-classes.""" + Authorization header. Over-ride this in sub-classes.""" pass def response(self, response, content): @@ -443,7 +490,7 @@ class Authentication(object): or such returned from the last authorized response. Over-rise this in sub-classes if necessary. - Return TRUE is the request is to be retried, for + Return TRUE is the request is to be retried, for example Digest may return stale=true. """ return False @@ -461,7 +508,7 @@ class BasicAuthentication(Authentication): class DigestAuthentication(Authentication): - """Only do qop='auth' and MD5, since that + """Only do qop='auth' and MD5, since that is all Apache currently implements""" def __init__(self, credentials, host, request_uri, headers, response, content, http): Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) @@ -474,7 +521,7 @@ class DigestAuthentication(Authentication): self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper() if self.challenge['algorithm'] != 'MD5': raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) - self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) + self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) self.challenge['nc'] = 1 def request(self, method, request_uri, headers, content, cnonce = None): @@ -482,23 +529,24 @@ class DigestAuthentication(Authentication): H = lambda x: _md5(x).hexdigest() KD = lambda s, d: H("%s:%s" % (s, d)) A2 = "".join([method, ":", request_uri]) - self.challenge['cnonce'] = cnonce or _cnonce() - request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % (self.challenge['nonce'], - '%08x' % self.challenge['nc'], - self.challenge['cnonce'], - self.challenge['qop'], H(A2) - )) - headers['Authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( - self.credentials[0], + self.challenge['cnonce'] = cnonce or _cnonce() + request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % ( + self.challenge['nonce'], + '%08x' % self.challenge['nc'], + self.challenge['cnonce'], + self.challenge['qop'], H(A2))) + headers['authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( + self.credentials[0], self.challenge['realm'], self.challenge['nonce'], - request_uri, + request_uri, self.challenge['algorithm'], request_digest, self.challenge['qop'], self.challenge['nc'], - self.challenge['cnonce'], - ) + self.challenge['cnonce']) + if self.challenge.get('opaque'): + headers['authorization'] += ', opaque="%s"' % self.challenge['opaque'] self.challenge['nc'] += 1 def response(self, response, content): @@ -506,14 +554,14 @@ class DigestAuthentication(Authentication): challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {}) if 'true' == challenge.get('stale'): self.challenge['nonce'] = challenge['nonce'] - self.challenge['nc'] = 1 + self.challenge['nc'] = 1 return True else: updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {}) if updated_challenge.has_key('nextnonce'): self.challenge['nonce'] = updated_challenge['nextnonce'] - self.challenge['nc'] = 1 + self.challenge['nc'] = 1 return False @@ -547,9 +595,8 @@ class HmacDigestAuthentication(Authentication): else: self.pwhashmod = _sha self.key = "".join([self.credentials[0], ":", - self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), - ":", self.challenge['realm'] - ]) + self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), + ":", self.challenge['realm']]) self.key = self.pwhashmod.new(self.key).hexdigest().lower() def request(self, method, request_uri, headers, content): @@ -561,16 +608,15 @@ class HmacDigestAuthentication(Authentication): cnonce = _cnonce() request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val) request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() - headers['Authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( - self.credentials[0], + headers['authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( + self.credentials[0], self.challenge['realm'], self.challenge['snonce'], cnonce, - request_uri, + request_uri, created, request_digest, - keylist, - ) + keylist) def response(self, response, content): challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {}) @@ -583,7 +629,7 @@ class WsseAuthentication(Authentication): """This is thinly tested and should not be relied upon. At this time there isn't any third party server to test against. Blogger and TypePad implemented this algorithm at one point - but Blogger has since switched to Basic over HTTPS and + but Blogger has since switched to Basic over HTTPS and TypePad has implemented it wrong, by never issuing a 401 challenge but instead requiring your client to telepathically know that their endpoint is expecting WSSE profile="UsernameToken".""" @@ -593,7 +639,7 @@ class WsseAuthentication(Authentication): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers['Authorization'] = 'WSSE profile="UsernameToken"' + headers['authorization'] = 'WSSE profile="UsernameToken"' iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) @@ -629,7 +675,7 @@ class GoogleLoginAuthentication(Authentication): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth AUTH_SCHEME_CLASSES = { @@ -644,13 +690,13 @@ AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] class FileCache(object): """Uses a local directory as a store for cached files. - Not really safe to use if multiple threads or processes are going to + Not really safe to use if multiple threads or processes are going to be running on the same cache. """ def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior self.cache = cache self.safe = safe - if not os.path.exists(cache): + if not os.path.exists(cache): os.makedirs(self.cache) def get(self, key): @@ -660,7 +706,7 @@ class FileCache(object): f = file(cacheFullPath, "rb") retval = f.read() f.close() - except IOError, e: + except IOError: pass return retval @@ -688,34 +734,127 @@ class Credentials(object): def iter(self, domain): for (cdomain, name, password) in self.credentials: if cdomain == "" or domain == cdomain: - yield (name, password) + yield (name, password) class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" pass +class AllHosts(object): + pass class ProxyInfo(object): - """Collect information required to use a proxy.""" - def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): - """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX - constants. For example: + """Collect information required to use a proxy.""" + bypass_hosts = () -p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) - """ - self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass + def __init__(self, proxy_type, proxy_host, proxy_port, + proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: - def astuple(self): - return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, - self.proxy_user, self.proxy_pass) + p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, + proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type = proxy_type + self.proxy_host = proxy_host + self.proxy_port = proxy_port + self.proxy_rdns = proxy_rdns + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass - def isgood(self): - return socks and (self.proxy_host != None) and (self.proxy_port != None) + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, + self.proxy_rdns, self.proxy_user, self.proxy_pass) + + def isgood(self): + return (self.proxy_host != None) and (self.proxy_port != None) + + def applies_to(self, hostname): + return not self.bypass_host(hostname) + + def bypass_host(self, hostname): + """Has this host been excluded from the proxy config""" + if self.bypass_hosts is AllHosts: + return True + + bypass = False + for domain in self.bypass_hosts: + if hostname.endswith(domain): + bypass = True + + return bypass + + +def proxy_info_from_environment(method='http'): + """ + Read proxy info from the environment variables. + """ + if method not in ['http', 'https']: + return + + env_var = method + '_proxy' + url = os.environ.get(env_var, os.environ.get(env_var.upper())) + if not url: + return + pi = proxy_info_from_url(url, method) + + no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) + bypass_hosts = [] + if no_proxy: + bypass_hosts = no_proxy.split(',') + # special case, no_proxy=* means all hosts bypassed + if no_proxy == '*': + bypass_hosts = AllHosts + + pi.bypass_hosts = bypass_hosts + return pi + +def proxy_info_from_url(url, method='http'): + """ + Construct a ProxyInfo from a URL (such as http_proxy env var) + """ + url = urlparse.urlparse(url) + username = None + password = None + port = None + if '@' in url[1]: + ident, host_port = url[1].split('@', 1) + if ':' in ident: + username, password = ident.split(':', 1) + else: + password = ident + else: + host_port = url[1] + if ':' in host_port: + host, port = host_port.split(':', 1) + else: + host = host_port + + if port: + port = int(port) + else: + port = dict(https=443, http=80)[method] + + proxy_type = 3 # socks.PROXY_TYPE_HTTP + return ProxyInfo( + proxy_type = proxy_type, + proxy_host = host, + proxy_port = port, + proxy_user = username or None, + proxy_pass = password or None, + ) class HTTPConnectionWithTimeout(httplib.HTTPConnection): - """HTTPConnection subclass that supports timeouts""" + """ + HTTPConnection subclass that supports timeouts + + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): httplib.HTTPConnection.__init__(self, host, port, strict) @@ -725,27 +864,46 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): def connect(self): """Connect to the host and port specified in __init__.""" # Mostly verbatim from httplib.py. + if self.proxy_info and socks is None: + raise ProxiesUnavailableError( + 'Proxy support missing but proxy use was requested!') msg = "getaddrinfo returns an empty list" - for res in socket.getaddrinfo(self.host, self.port, 0, - socket.SOCK_STREAM): + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: - if self.proxy_info and self.proxy_info.isgood(): + if use_proxy: self.sock = socks.socksocket(af, socktype, proto) - self.sock.setproxy(*self.proxy_info.astuple()) + self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) else: self.sock = socket.socket(af, socktype, proto) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Different from httplib: support timeouts. if has_timeout(self.timeout): self.sock.settimeout(self.timeout) # End of difference from httplib. if self.debuglevel > 0: - print "connect: (%s, %s)" % (self.host, self.port) + print "connect: (%s, %s) ************" % (self.host, self.port) + if use_proxy: + print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) - self.sock.connect(sa) + self.sock.connect((self.host, self.port) + sa[2:]) except socket.error, msg: if self.debuglevel > 0: - print 'connect fail:', (self.host, self.port) + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) if self.sock: self.sock.close() self.sock = None @@ -755,56 +913,265 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): raise socket.error, msg class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): - "This class allows communication via SSL." + """ + This class allows communication via SSL. + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=None, proxy_info=None): - httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, - cert_file=cert_file, strict=strict) + strict=None, timeout=None, proxy_info=None, + ca_certs=None, disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, + key_file=key_file, + cert_file=cert_file, strict=strict) self.timeout = timeout self.proxy_info = proxy_info + if ca_certs is None: + ca_certs = CA_CERTS + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + + # The following two methods were adapted from https_wrapper.py, released + # with the Google Appengine SDK at + # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py + # under the following license: + # + # Copyright 2007 Google Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + def _GetValidHostsForCert(self, cert): + """Returns a list of valid host globs for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + Returns: + list: A list of valid host globs. + """ + if 'subjectAltName' in cert: + return [x[1] for x in cert['subjectAltName'] + if x[0].lower() == 'dns'] + else: + return [x[0][1] for x in cert['subject'] + if x[0][0].lower() == 'commonname'] + + def _ValidateCertificateHostname(self, cert, hostname): + """Validates that a given hostname is valid for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + hostname: The hostname to test. + Returns: + bool: Whether or not the hostname is valid for this certificate. + """ + hosts = self._GetValidHostsForCert(cert) + for host in hosts: + host_re = host.replace('.', '\.').replace('*', '[^.]*') + if re.search('^%s$' % (host_re,), hostname, re.I): + return True + return False def connect(self): "Connect to a host on a given (SSL) port." + msg = "getaddrinfo returns an empty list" if self.proxy_info and self.proxy_info.isgood(): - sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) - sock.setproxy(*self.proxy_info.astuple()) + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() else: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - if has_timeout(self.timeout): - sock.settimeout(self.timeout) - sock.connect((self.host, self.port)) - self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) + for family, socktype, proto, canonname, sockaddr in address_info: + try: + if use_proxy: + sock = socks.socksocket(family, socktype, proto) + + sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) + else: + sock = socket.socket(family, socktype, proto) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket( + sock, self.key_file, self.cert_file, + self.disable_ssl_certificate_validation, self.ca_certs) + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if not self.disable_ssl_certificate_validation: + cert = self.sock.getpeercert() + hostname = self.host.split(':', 0)[0] + if not self._ValidateCertificateHostname(cert, hostname): + raise CertificateHostnameMismatch( + 'Server presented certificate that does not match ' + 'host %s: %s' % (hostname, cert), hostname, cert) + except ssl_SSLError, e: + if sock: + sock.close() + if self.sock: + self.sock.close() + self.sock = None + # Unfortunately the ssl module doesn't seem to provide any way + # to get at more detailed error information, in particular + # whether the error is due to certificate validation or + # something else (such as SSL protocol mismatch). + if e.errno == ssl.SSL_ERROR_SSL: + raise SSLHandshakeError(e) + else: + raise + except (socket.timeout, socket.gaierror): + raise + except socket.error, msg: + if self.debuglevel > 0: + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +SCHEME_TO_CONNECTION = { + 'http': HTTPConnectionWithTimeout, + 'https': HTTPSConnectionWithTimeout +} + +# Use a different connection object for Google App Engine +try: + try: + from google.appengine.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google.appengine.api.urlfetch import fetch + from google.appengine.api.urlfetch import InvalidURLError + except (ImportError, AttributeError): + from google3.apphosting.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google3.apphosting.api.urlfetch import fetch + from google3.apphosting.api.urlfetch import InvalidURLError + + def _new_fixed_fetch(validate_certificate): + def fixed_fetch(url, payload=None, method="GET", headers={}, + allow_truncated=False, follow_redirects=True, + deadline=5): + return fetch(url, payload=payload, method=method, headers=headers, + allow_truncated=allow_truncated, + follow_redirects=follow_redirects, deadline=deadline, + validate_certificate=validate_certificate) + return fixed_fetch + + class AppEngineHttpConnection(httplib.HTTPConnection): + """Use httplib on App Engine, but compensate for its weirdness. + + The parameters key_file, cert_file, proxy_info, ca_certs, and + disable_ssl_certificate_validation are all dropped on the ground. + """ + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + httplib.HTTPConnection.__init__(self, host, port=port, + strict=strict, timeout=timeout) + + class AppEngineHttpsConnection(httplib.HTTPSConnection): + """Same as AppEngineHttpConnection, but for HTTPS URIs.""" + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, + key_file=key_file, + cert_file=cert_file, strict=strict, + timeout=timeout) + self._fetch = _new_fixed_fetch( + not disable_ssl_certificate_validation) + + # Update the connection classes to use the Googel App Engine specific ones. + SCHEME_TO_CONNECTION = { + 'http': AppEngineHttpConnection, + 'https': AppEngineHttpsConnection + } +except (ImportError, AttributeError): + pass class Http(object): """An HTTP client that handles: -- all methods -- caching -- ETags -- compression, -- HTTPS -- Basic -- Digest -- WSSE -and more. + - all methods + - caching + - ETags + - compression, + - HTTPS + - Basic + - Digest + - WSSE + + and more. """ - def __init__(self, cache=None, timeout=None, proxy_info=None): - """The value of proxy_info is a ProxyInfo instance. + def __init__(self, cache=None, timeout=None, + proxy_info=proxy_info_from_environment, + ca_certs=None, disable_ssl_certificate_validation=False): + """If 'cache' is a string then it is used as a directory name for + a disk cache. Otherwise it must be an object that supports the + same interface as FileCache. -If 'cache' is a string then it is used as a directory name -for a disk cache. Otherwise it must be an object that supports -the same interface as FileCache.""" + All timeouts are in seconds. If None is passed for timeout + then Python's default timeout for sockets will be used. See + for example the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + + `proxy_info` may be: + - a callable that takes the http scheme ('http' or 'https') and + returns a ProxyInfo instance per request. By default, uses + proxy_nfo_from_environment. + - a ProxyInfo instance (static proxy config). + - None (proxy disabled). + + ca_certs is the path of a file containing root CA certificates for SSL + server certificate validation. By default, a CA cert file bundled with + httplib2 is used. + + If disable_ssl_certificate_validation is true, SSL cert validation will + not be performed. + """ self.proxy_info = proxy_info + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + # Map domain name to an httplib connection self.connections = {} # The location of the cache, for now a directory # where cached responses are held. - if cache and isinstance(cache, str): + if cache and isinstance(cache, basestring): self.cache = FileCache(cache) else: self.cache = cache @@ -820,10 +1187,10 @@ the same interface as FileCache.""" # If set to False then no redirects are followed, even safe ones. self.follow_redirects = True - + # Which HTTP methods do we apply optimistic concurrency to, i.e. # which methods get an "if-match:" etag header added to them. - self.optimistic_concurrency_methods = ["PUT"] + self.optimistic_concurrency_methods = ["PUT", "PATCH"] # If 'follow_redirects' is True, and this is set to True then # all redirecs are followed, including unsafe ones. @@ -831,10 +1198,27 @@ the same interface as FileCache.""" self.ignore_etag = False - self.force_exception_to_status_code = False + self.force_exception_to_status_code = False self.timeout = timeout + # Keep Authorization: headers on a redirect. + self.forward_authorization_headers = False + + def __getstate__(self): + state_dict = copy.copy(self.__dict__) + # In case request is augmented by some foreign object such as + # credentials which handle auth + if 'request' in state_dict: + del state_dict['request'] + if 'connections' in state_dict: + del state_dict['connections'] + return state_dict + + def __setstate__(self, state): + self.__dict__.update(state) + self.connections = {} + def _auth_from_challenge(self, host, request_uri, headers, response, content): """A generator that creates Authorization objects that can be applied to requests. @@ -862,37 +1246,56 @@ the same interface as FileCache.""" self.authorizations = [] def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(2): + for i in range(RETRIES): try: + if hasattr(conn, 'sock') and conn.sock is None: + conn.connect() conn.request(method, request_uri, body, headers) + except socket.timeout: + raise except socket.gaierror: conn.close() raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except ssl_SSLError: + conn.close() + raise except socket.error, e: - if not hasattr(e, 'errno'): # I don't know what this is so lets raise it if it happens + err = 0 + if hasattr(e, 'args'): + err = getattr(e, 'args')[0] + else: + err = e.errno + if err == errno.ECONNREFUSED: # Connection refused raise - elif e.errno == errno.ECONNREFUSED: # Connection refused - raise - # Just because the server closed the connection doesn't apparently mean - # that the server didn't send a response. - pass except httplib.HTTPException: # Just because the server closed the connection doesn't apparently mean # that the server didn't send a response. - pass + if hasattr(conn, 'sock') and conn.sock is None: + if i < RETRIES-1: + conn.close() + conn.connect() + continue + else: + conn.close() + raise + if i < RETRIES-1: + conn.close() + conn.connect() + continue try: response = conn.getresponse() except (socket.error, httplib.HTTPException): - if i == 0: + if i < RETRIES-1: conn.close() conn.connect() continue else: + conn.close() raise else: content = "" if method == "HEAD": - response.close() + conn.close() else: content = response.read() response = Response(response) @@ -908,12 +1311,12 @@ the same interface as FileCache.""" auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] auth = auths and sorted(auths)[0][1] or None - if auth: + if auth: auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) - if auth: + if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers ) @@ -921,7 +1324,7 @@ the same interface as FileCache.""" if response.status == 401: for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): - authorization.request(method, request_uri, headers, body) + authorization.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) if response.status != 401: self.authorizations.append(authorization) @@ -944,26 +1347,31 @@ the same interface as FileCache.""" if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri _updateCache(headers, response, content, self.cache, cachekey) if headers.has_key('if-none-match'): del headers['if-none-match'] if headers.has_key('if-modified-since'): del headers['if-modified-since'] + if 'authorization' in headers and not self.forward_authorization_headers: + del headers['authorization'] if response.has_key('location'): location = response['location'] old_response = copy.deepcopy(response) if not old_response.has_key('content-location'): - old_response['content-location'] = absolute_uri - redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method + old_response['content-location'] = absolute_uri + redirect_method = method + if response.status in [302, 303]: + redirect_method = "GET" + body = None (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) response.previous = old_response else: - raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) - elif response.status in [200, 203] and method == "GET": + raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content) + elif response.status in [200, 203] and method in ["GET", "HEAD"]: # Don't cache 206's since we aren't going to handle byte range requests if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri _updateCache(headers, response, content, self.cache, cachekey) return (response, content) @@ -978,24 +1386,25 @@ the same interface as FileCache.""" def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): """ Performs a single HTTP request. -The 'uri' is the URI of the HTTP resource and can begin -with either 'http' or 'https'. The value of 'uri' must be an absolute URI. -The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. -There is no restriction on the methods allowed. + The 'uri' is the URI of the HTTP resource and can begin with either + 'http' or 'https'. The value of 'uri' must be an absolute URI. -The 'body' is the entity body to be sent with the request. It is a string -object. + The 'method' is the HTTP method to perform, such as GET, POST, DELETE, + etc. There is no restriction on the methods allowed. -Any extra headers that are to be sent with the request should be provided in the -'headers' dictionary. + The 'body' is the entity body to be sent with the request. It is a + string object. -The maximum number of redirect to follow before raising an -exception is 'redirections. The default is 5. + Any extra headers that are to be sent with the request should be + provided in the 'headers' dictionary. -The return value is a tuple of (response, content), the first -being and instance of the 'Response' class, the second being -a string that contains the response entity body. + The maximum number of redirect to follow before raising an + exception is 'redirections. The default is 5. + + The return value is a tuple of (response, content), the first + being and instance of the 'Response' class, the second being + a string that contains the response entity body. """ try: if headers is None: @@ -1004,7 +1413,7 @@ a string that contains the response entity body. headers = self._normalize_headers(headers) if not headers.has_key('user-agent'): - headers['user-agent'] = "Python-httplib2/%s" % __version__ + headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) @@ -1014,21 +1423,38 @@ a string that contains the response entity body. scheme = 'https' authority = domain_port[0] + proxy_info = self._get_proxy_info(scheme, authority) + conn_key = scheme+":"+authority if conn_key in self.connections: conn = self.connections[conn_key] else: if not connection_type: - connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout + connection_type = SCHEME_TO_CONNECTION[scheme] certs = list(self.certificates.iter(authority)) - if scheme == 'https' and certs: - conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], - cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) + if scheme == 'https': + if certs: + conn = self.connections[conn_key] = connection_type( + authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) + else: + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) else: - conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info) conn.set_debuglevel(debuglevel) - if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: + if 'range' not in headers and 'accept-encoding' not in headers: headers['accept-encoding'] = 'gzip, deflate' info = email.Message.Message() @@ -1048,7 +1474,7 @@ a string that contains the response entity body. feedparser.feed(info) info = feedparser.close() feedparser._parse = None - except IndexError, ValueError: + except (IndexError, ValueError): self.cache.delete(cachekey) cachekey = None cached_value = None @@ -1071,13 +1497,15 @@ a string that contains the response entity body. for header in vary_headers: key = '-varied-%s' % header value = info[key] - if headers.get(header, '') != value: - cached_value = None - break + if headers.get(header, None) != value: + cached_value = None + break if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: if info.has_key('-x-permanent-redirect-url'): # Should cached permanent redirects be counted in our redirection count? For now, yes. + if redirections <= 0: + raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "") (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) response.previous = Response(info) response.previous.fromcache = True @@ -1085,13 +1513,13 @@ a string that contains the response entity body. # Determine our course of action: # Is the cached entry fresh or stale? # Has the client requested a non-cached response? - # - # There seems to be three possible answers: + # + # There seems to be three possible answers: # 1. [FRESH] Return the cache entry w/o doing a GET # 2. [STALE] Do the GET (but add in cache validators if available) # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request - entry_disposition = _entry_disposition(info, headers) - + entry_disposition = _entry_disposition(info, headers) + if entry_disposition == "FRESH": if not cached_value: info['status'] = '504' @@ -1113,7 +1541,7 @@ a string that contains the response entity body. if response.status == 304 and method == "GET": # Rewrite the cache entry with the new end-to-end headers - # Take all headers that are in response + # Take all headers that are in response # and overwrite their values in info. # unless they are hop-by-hop, or are listed in the connection header. @@ -1125,14 +1553,14 @@ a string that contains the response entity body. _updateCache(headers, merged_response, content, self.cache, cachekey) response = merged_response response.status = 200 - response.fromcache = True + response.fromcache = True elif response.status == 200: content = new_content else: self.cache.delete(cachekey) - content = new_content - else: + content = new_content + else: cc = _parse_cache_control(headers) if cc.has_key('only-if-cached'): info['status'] = '504' @@ -1146,34 +1574,47 @@ a string that contains the response entity body. response = e.response content = e.content response.status = 500 - response.reason = str(e) - elif isinstance(e, socket.timeout) or (isinstance(e, socket.error) and 'timed out' in str(e)): + response.reason = str(e) + elif isinstance(e, socket.timeout): content = "Request Timeout" - response = Response( { - "content-type": "text/plain", - "status": "408", - "content-length": len(content) - }) + response = Response({ + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) response.reason = "Request Timeout" else: - content = str(e) - response = Response( { - "content-type": "text/plain", - "status": "400", - "content-length": len(content) - }) - response.reason = "Bad Request" + content = str(e) + response = Response({ + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" else: raise - + return (response, content) - + def _get_proxy_info(self, scheme, authority): + """Return a ProxyInfo instance (or None) based on the scheme + and authority. + """ + hostname, port = urllib.splitport(authority) + proxy_info = self.proxy_info + if callable(proxy_info): + proxy_info = proxy_info(scheme) + + if (hasattr(proxy_info, 'applies_to') + and not proxy_info.applies_to(hostname)): + proxy_info = None + return proxy_info + class Response(dict): """An object more like email.Message than httplib.HTTPResponse.""" - + """Is this response from our local cache""" fromcache = False @@ -1189,27 +1630,28 @@ class Response(dict): previous = None def __init__(self, info): - # info is either an email.Message or + # info is either an email.Message or # an httplib.HTTPResponse object. if isinstance(info, httplib.HTTPResponse): - for key, value in info.getheaders(): - self[key.lower()] = value + for key, value in info.getheaders(): + self[key.lower()] = value self.status = info.status self['status'] = str(self.status) self.reason = info.reason self.version = info.version elif isinstance(info, email.Message.Message): - for key, value in info.items(): - self[key] = value + for key, value in info.items(): + self[key.lower()] = value self.status = int(self['status']) else: - for key, value in info.iteritems(): - self[key] = value + for key, value in info.iteritems(): + self[key.lower()] = value self.status = int(self.get('status', self.status)) + self.reason = self.get('reason', self.reason) def __getattr__(self, name): if name == 'dict': - return self - else: - raise AttributeError, name + return self + else: + raise AttributeError, name diff --git a/libs/httplib2/cacerts.txt b/libs/httplib2/cacerts.txt new file mode 100644 index 00000000..d8a0027c --- /dev/null +++ b/libs/httplib2/cacerts.txt @@ -0,0 +1,739 @@ +# Certifcate Authority certificates for validating SSL connections. +# +# This file contains PEM format certificates generated from +# http://mxr.mozilla.org/seamonkey/source/security/nss/lib/ckfw/builtins/certdata.txt +# +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Netscape security libraries. +# +# The Initial Developer of the Original Code is +# Netscape Communications Corporation. +# Portions created by the Initial Developer are Copyright (C) 1994-2000 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +Verisign/RSA Secure Server CA +============================= + +-----BEGIN CERTIFICATE----- +MIICNDCCAaECEAKtZn5ORf5eV288mBle3cAwDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxIDAeBgNVBAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYD +VQQLEyVTZWN1cmUgU2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk0 +MTEwOTAwMDAwMFoXDTEwMDEwNzIzNTk1OVowXzELMAkGA1UEBhMCVVMxIDAeBgNV +BAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYDVQQLEyVTZWN1cmUgU2Vy +dmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGbMA0GCSqGSIb3DQEBAQUAA4GJ +ADCBhQJ+AJLOesGugz5aqomDV6wlAXYMra6OLDfO6zV4ZFQD5YRAUcm/jwjiioII +0haGN1XpsSECrXZogZoFokvJSyVmIlZsiAeP94FZbYQHZXATcXY+m3dM41CJVphI +uR2nKRoTLkoRWZweFdVJVCxzOmmCsZc5nG1wZ0jl3S3WyB57AgMBAAEwDQYJKoZI +hvcNAQECBQADfgBl3X7hsuyw4jrg7HFGmhkRuNPHoLQDQCYCPgmc4RKz0Vr2N6W3 +YQO2WxZpO8ZECAyIUwxrl0nHPjXcbLm7qt9cuzovk2C2qUtN8iD3zV9/ZHuO3ABc +1/p3yjkWWW8O6tO1g39NTUJWdrTJXwT4OPjr0l91X817/OWOgHz8UA== +-----END CERTIFICATE----- + +Thawte Personal Basic CA +======================== + +-----BEGIN CERTIFICATE----- +MIIDITCCAoqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFBlcnNvbmFsIEJhc2lj +IENBMSgwJgYJKoZIhvcNAQkBFhlwZXJzb25hbC1iYXNpY0B0aGF3dGUuY29tMB4X +DTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgcsxCzAJBgNVBAYTAlpBMRUw +EwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEaMBgGA1UE +ChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRpb24gU2Vy +dmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQZXJzb25hbCBCYXNpYyBD +QTEoMCYGCSqGSIb3DQEJARYZcGVyc29uYWwtYmFzaWNAdGhhd3RlLmNvbTCBnzAN +BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvLyTU23AUE+CFeZIlDWmWr5vQvoPR+53 +dXLdjUmbllegeNTKP1GzaQuRdhciB5dqxFGTS+CN7zeVoQxN2jSQHReJl+A1OFdK +wPQIcOk8RHtQfmGakOMj04gRRif1CwcOu93RfyAKiLlWCy4cgNrx454p7xS9CkT7 +G1sY0b8jkyECAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQQF +AAOBgQAt4plrsD16iddZopQBHyvdEktTwq1/qqcAXJFAVyVKOKqEcLnZgA+le1z7 +c8a914phXAPjLSeoF+CEhULcXpvGt7Jtu3Sv5D/Lp7ew4F2+eIMllNLbgQ95B21P +9DkVWlIBe94y1k049hJcBlDfBVu9FEuh3ym6O0GN92NWod8isQ== +-----END CERTIFICATE----- + +Thawte Personal Premium CA +========================== + +-----BEGIN CERTIFICATE----- +MIIDKTCCApKgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBzzELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEjMCEGA1UEAxMaVGhhd3RlIFBlcnNvbmFsIFByZW1p +dW0gQ0ExKjAoBgkqhkiG9w0BCQEWG3BlcnNvbmFsLXByZW1pdW1AdGhhd3RlLmNv +bTAeFw05NjAxMDEwMDAwMDBaFw0yMDEyMzEyMzU5NTlaMIHPMQswCQYDVQQGEwJa +QTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xGjAY +BgNVBAoTEVRoYXd0ZSBDb25zdWx0aW5nMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9u +IFNlcnZpY2VzIERpdmlzaW9uMSMwIQYDVQQDExpUaGF3dGUgUGVyc29uYWwgUHJl +bWl1bSBDQTEqMCgGCSqGSIb3DQEJARYbcGVyc29uYWwtcHJlbWl1bUB0aGF3dGUu +Y29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJZtn4B0TPuYwu8KHvE0Vs +Bd/eJxZRNkERbGw77f4QfRKe5ZtCmv5gMcNmt3M6SK5O0DI3lIi1DbbZ8/JE2dWI +Et12TfIa/G8jHnrx2JhFTgcQ7xZC0EN1bUre4qrJMf8fAHB8Zs8QJQi6+u4A6UYD +ZicRFTuqW/KY3TZCstqIdQIDAQABoxMwETAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBBAUAA4GBAGk2ifc0KjNyL2071CKyuG+axTZmDhs8obF1Wub9NdP4qPIH +b4Vnjt4rueIXsDqg8A6iAJrf8xQVbrvIhVqYgPn/vnQdPfP+MCXRNzRn+qVxeTBh +KXLA4CxM+1bkOqhv5TJZUtt1KFBZDPgLGeSs2a+WjS9Q2wfD6h+rM+D1KzGJ +-----END CERTIFICATE----- + +Thawte Personal Freemail CA +=========================== + +-----BEGIN CERTIFICATE----- +MIIDLTCCApagAwIBAgIBADANBgkqhkiG9w0BAQQFADCB0TELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEkMCIGA1UEAxMbVGhhd3RlIFBlcnNvbmFsIEZyZWVt +YWlsIENBMSswKQYJKoZIhvcNAQkBFhxwZXJzb25hbC1mcmVlbWFpbEB0aGF3dGUu +Y29tMB4XDTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgdExCzAJBgNVBAYT +AlpBMRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEa +MBgGA1UEChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xJDAiBgNVBAMTG1RoYXd0ZSBQZXJzb25hbCBG +cmVlbWFpbCBDQTErMCkGCSqGSIb3DQEJARYccGVyc29uYWwtZnJlZW1haWxAdGhh +d3RlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1GnX1LCUZFtx6UfY +DFG26nKRsIRefS0Nj3sS34UldSh0OkIsYyeflXtL734Zhx2G6qPduc6WZBrCFG5E +rHzmj+hND3EfQDimAKOHePb5lIZererAXnbr2RSjXW56fAylS1V/Bhkpf56aJtVq +uzgkCGqYx7Hao5iR/Xnb5VrEHLkCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zAN +BgkqhkiG9w0BAQQFAAOBgQDH7JJ+Tvj1lqVnYiqk8E0RYNBvjWBYYawmu1I1XAjP +MPuoSpaKH2JCI4wXD/S6ZJwXrEcp352YXtJsYHFcoqzceePnbgBHH7UNKOgCneSa +/RP0ptl8sfjcXyMmCZGAc9AUG95DqYMl8uacLxXK/qarigd1iwzdUYRr5PjRznei +gQ== +-----END CERTIFICATE----- + +Thawte Server CA +================ + +-----BEGIN CERTIFICATE----- +MIIDEzCCAnygAwIBAgIBATANBgkqhkiG9w0BAQQFADCBxDELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEm +MCQGCSqGSIb3DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wHhcNOTYwODAx +MDAwMDAwWhcNMjAxMjMxMjM1OTU5WjCBxDELMAkGA1UEBhMCWkExFTATBgNVBAgT +DFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYDVQQKExRUaGF3 +dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNl +cyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEmMCQGCSqGSIb3 +DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD +gY0AMIGJAoGBANOkUG7I/1Zr5s9dtuoMaHVHoqrC2oQl/Kj0R1HahbUgdJSGHg91 +yekIYfUGbTBuFRkC6VLAYttNmZ7iagxEOM3+vuNkCXDF/rFrKbYvScg71CcEJRCX +L+eQbcAoQpnXTEPew/UhbVSfXcNY4cDk2VuwuNy0e982OsK1ZiIS1ocNAgMBAAGj +EzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEEBQADgYEAB/pMaVz7lcxG +7oWDTSEwjsrZqG9JGubaUeNgcGyEYRGhGshIPllDfU+VPaGLtwtimHp1it2ITk6e +QNuozDJ0uW8NxuOzRAvZim+aKZuZGCg70eNAKJpaPNW15yAbi8qkq43pUdniTCxZ +qdq5snUb9kLy78fyGPmJvKP/iiMucEc= +-----END CERTIFICATE----- + +Thawte Premium Server CA +======================== + +-----BEGIN CERTIFICATE----- +MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy +dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t +MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB +MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG +A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl +cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv +bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE +VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ +ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR +uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG +9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI +hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM +pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg== +-----END CERTIFICATE----- + +Equifax Secure CA +================= + +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV +UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy +dGlmaWNhdGUgQXV0aG9yaXR5MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1 +MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VxdWlmYXgxLTArBgNVBAsTJEVx +dWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPRfM6f +BeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+A +cJkVV5MW8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kC +AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQ +MA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlm +aWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTgw +ODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvSspXXR9gj +IBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQF +MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA +A4GBAFjOKer89961zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y +7qj/WsjTVbJmcVfewCHrPSqnI0kBBIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh +1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee9570+sB3c4 +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPTCCAaYCEQDNun9W8N/kvFT+IqyzcqpVMA0GCSqGSIb3DQEBAgUAMF8xCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xh +c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05 +NjAxMjkwMDAwMDBaFw0yODA4MDEyMzU5NTlaMF8xCzAJBgNVBAYTAlVTMRcwFQYD +VQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xhc3MgMSBQdWJsaWMgUHJp +bWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOB +jQAwgYkCgYEA5Rm/baNWYS2ZSHH2Z965jeu3noaACpEO+jglr0aIguVzqKCbJF0N +H8xlbgyw0FaEGIeaBpsQoXPftFg5a27B9hXVqKg/qhIGjTGsf7A01480Z4gJzRQR +4k5FVmkfeAKA2txHkSm7NsljXMXg1y2He6G3MrB7MLoqLzGq7qNn2tsCAwEAATAN +BgkqhkiG9w0BAQIFAAOBgQBMP7iLxmjf7kMzDl3ppssHhE16M/+SG/Q2rdiVIjZo +EWx8QszznC7EBz8UsA9P/5CSdvnivErpj82ggAr3xSnxgiJduLHdgSOjeyUVRjB5 +FvjqBUuUfx3CHMjjt/QQQDwTw18fU+hI5Ia0e6E1sHslurjTjqs/OJ0ANACY89Fx +lA== +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEC0b/EoXjaOR6+f/9YtFvgswDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAyIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAyIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQC2WoujDWojg4BrzzmH9CETMwZMJaLtVRKXxaeAufqDwSCg+i8VDXyh +YGt+eSz6Bg86rvYbb7HS/y8oUl+DfUvEerf4Zh+AVPy3wo5ZShRXRtGak75BkQO7 +FYCTXOvnzAhsPz6zSvz/S2wj1VCCJkQZjiPDceoZJEcEnnW/yKYAHwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBAIobK/o5wXTXXtgZZKJYSi034DNHD6zt96rbHuSLBlxg +J8pFUs4W7z8GZOeUaHxgMxURaa+dYo2jA1Rrpr7l7gUYYAS/QoD90KioHgE796Nc +r6Pc5iaAIzy4RHT3Cq5Ji2F4zCS/iIqnDupzGUH9TQPwiNHleI2lKk/2lw0Xd8rY +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE +BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is +I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do +lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc +AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEEzH6qqYPnHTkxD4PTqJkZIwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMSBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQCq0Lq+Fi24g9TK0g+8djHKlNgdk4xWArzZbxpvUjZudVYK +VdPfQ4chEWWKfo+9Id5rMj8bhDSVBZ1BNeuS65bdqlk/AVNtmU/t5eIqWpDBucSm +Fc/IReumXY6cPvBkJHalzasab7bYe1FhbqZ/h8jit+U03EGI6glAvnOSPWvndQID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAKlPww3HZ74sy9mozS11534Vnjty637rXC0J +h9ZrbWB85a7FkCMMXErQr7Fd88e2CtvgFZMN3QO8x3aKtd1Pw5sTdbgBwObJW2ul +uIncrKTdcu1OofdPvAbT6shkdHvClUGcZXNY8ZCaPGqxmMnEh7zPRW1F4m4iP/68 +DzFc6PLZ +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAzCCAmwCEQC5L2DMiJ+hekYJuFtwbIqvMA0GCSqGSIb3DQEBBQUAMIHBMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0Ns +YXNzIDIgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH +MjE6MDgGA1UECxMxKGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9y +aXplZCB1c2Ugb25seTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazAe +Fw05ODA1MTgwMDAwMDBaFw0yODA4MDEyMzU5NTlaMIHBMQswCQYDVQQGEwJVUzEX +MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0NsYXNzIDIgUHVibGlj +IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBHMjE6MDgGA1UECxMx +KGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s +eTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEAp4gBIXQs5xoD8JjhlzwPIQjxnNuX6Zr8wgQGE75fUsjM +HiwSViy4AWkszJkfrbCWrnkE8hM5wXuYuggs6MKEEyyqaekJ9MepAqRCwiNPStjw +DqL7MWzJ5m+ZJwf15vRMeJ5t60aG+rmGyVTyssSv1EYcWskVMP8NbPUtDm3Of3cC +AwEAATANBgkqhkiG9w0BAQUFAAOBgQByLvl/0fFx+8Se9sVeUYpAmLho+Jscg9ji +nb3/7aHmZuovCfTK1+qlK5X2JGCGTUQug6XELaDTrnhpb3LabK4I8GOSN+a7xDAX +rXfMSTWqz9iP0b63GJZHc2pUIjRkLbYWm1lbtFFZOrMLFPQS32eg9K0yZF6xRnIn +jBJ7xUS0rg== +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMyBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMyBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQDMXtERXVxp0KvTuWpMmR9ZmDCOFoUgRm1HP9SFIIThbbP4 +pO0M8RcPO/mn+SXXwc+EY/J8Y8+iR/LGWzOOZEAEaMGAuWQcRXfH2G71lSk8UOg0 +13gfqLptQ5GVj0VXXn7F+8qkBOvqlzdUMG+7AUcyM83cV5tkaWH4mx0ciU9cZwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAFFNzb5cy5gZnBWyATl4Lk0PZ3BwmcYQWpSk +U01UbSuvDV1Ai2TT1+7eVmGSX6bEHRBhNtMsJzzoKQm5EWR0zLVznxxIqbxhAe7i +F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY +oJ2daZH9 +-----END CERTIFICATE----- + +Verisign Class 4 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEDKIjprS9esTR/h/xCA3JfgwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgNCBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgNCBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQC68OTP+cSuhVS5B1f5j8V/aBH4xBewRNzjMHPVKmIquNDM +HO0oW369atyzkSTKQWI8/AIBvxwWMZQFl3Zuoq29YRdsTjCG8FE3KlDHqGKB3FtK +qsGgtG7rL+VXxbErQHDbWk2hjh+9Ax/YA9SPTJlxvOKCzFjomDqG04Y48wApHwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAIWMEsGnuVAVess+rLhDityq3RS6iYF+ATwj +cSGIL4LcY/oCRaxFWdcqWERbt5+BO5JoPeI3JPV7bI92NZYJqFmduc4jq3TWg/0y +cyfYaT5DdPauxYma51N86Xv2S/PBZYPejYqcPIiNOVn8qj8ijaHBZlCBckztImRP +T8qAkbYp +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCLW3VWhFSFCwDPrzhIzrGkMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN2E1Lm0+afY8wR4 +nN493GwTFtl63SRRZsDHJlkNrAYIwpTRMx/wgzUfbhvI3qpuFU5UJ+/EbRrsC+MO +8ESlV8dAWB6jRx9x7GD2bZTIGDnt/kIYVt/kTEkQeE4BdjVjEjbdZrwBBDajVWjV +ojYJrKshJlQGrT/KFOCsyq0GHZXi+J3x4GD/wn91K0zM2v6HmSHquv4+VNfSWXjb +PG7PoBMAGrgnoeS+Z5bKoMWznN3JdZ7rMJpfo83ZrngZPyPpXNspva1VyBtUjGP2 +6KbqxzcSXKMpHgLZ2x87tNcPVkeBFQRKr4Mn0cVYiMHd9qqnoxjaaKptEVHhv2Vr +n5Z20T0CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAq2aN17O6x5q25lXQBfGfMY1a +qtmqRiYPce2lrVNWYgFHKkTp/j90CxObufRNG7LRX7K20ohcs5/Ny9Sn2WCVhDr4 +wTcdYcrnsMXlkdpUpqwxga6X3s0IrLjAl4B/bnKk52kTlWUfxJM8/XmPBNQ+T+r3 +ns7NZ3xPZQL/kYVUc8f/NveGLezQXk//EZ9yBta4GvFMDSZl4kSAHsef493oCtrs +pSCAaWihT37ha88HQfqDjrw43bAuEbFrskLMmrz5SCJ5ShkPshw+IHTZasO+8ih4 +E1Z5T21Q6huwtVexN2ZYI/PcD98Kh8TvhgXVOBRgmaNL3gaWcSzy27YfpO8/7g== +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGTCCAwECEGFwy0mMX5hFKeewptlQW3owDQYJKoZIhvcNAQEFBQAwgcoxCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVy +aVNpZ24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24s +IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNp +Z24gQ2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eSAtIEczMB4XDTk5MTAwMTAwMDAwMFoXDTM2MDcxNjIzNTk1OVowgcoxCzAJBgNV +BAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVyaVNp +Z24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24sIElu +Yy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNpZ24g +Q2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAt +IEczMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArwoNwtUs22e5LeWU +J92lvuCwTY+zYVY81nzD9M0+hsuiiOLh2KRpxbXiv8GmR1BeRjmL1Za6tW8UvxDO +JxOeBUebMXoT2B/Z0wI3i60sR/COgQanDTAM6/c8DyAd3HJG7qUCyFvDyVZpTMUY +wZF7C9UTAJu878NIPkZgIIUq1ZC2zYugzDLdt/1AVbJQHFauzI13TccgTacxdu9o +koqQHgiBVrKtaaNS0MscxCM9H5n+TOgWY47GCI72MfbS+uV23bUckqNJzc0BzWjN +qWm6o+sdDZykIKbBoMXRRkwXbdKsZj+WjOCE1Db/IlnF+RFgqF8EffIa9iVCYQ/E +Srg+iQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQA0JhU8wI1NQ0kdvekhktdmnLfe +xbjQ5F1fdiLAJvmEOjr5jLX77GDx6M4EsMjdpwOPMPOY36TmpDHf0xwLRtxyID+u +7gU8pDM/CzmscHhzS5kr3zDCVLCoO1Wh/hYozUK9dG6A2ydEp85EXdQbkJgNHkKU +sQAsBNB0owIFImNjzYO1+8FtYmtpdf1dcEG59b98377BMnMiIYtYgXsVkXq642RI +sH/7NiXaldDxJBQX3RiAa0YjOVT1jmIJBB2UkKab5iXiQkWquJCtvgiPqQtCGJTP +cjnhsUPgKM+351psE2tJs//jGHyJizNdrDPXp/naOlXJWBD5qu9ats9LS98q +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b +N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t +KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu +kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm +CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ +Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu +imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te +2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe +DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC +/Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p +F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt +TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ== +-----END CERTIFICATE----- + +Verisign Class 4 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQDsoKeLbnVqAc/EfMwvlF7XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3LpRFpxlmr8Y+1 +GQ9Wzsy1HyDkniYlS+BzZYlZ3tCD5PUPtbut8XzoIfzk6AzufEUiGXaStBO3IFsJ ++mGuqPKljYXCKtbeZjbSmwL0qJJgfJxptI8kHtCGUvYynEFYHiK9zUVilQhu0Gbd +U6LM8BDcVHOLBKFGMzNcF0C5nk3T875Vg+ixiY5afJqWIpA7iCXy0lOIAgwLePLm +NxdLMEYH5IBtptiWLugs+BGzOA1mppvqySNb247i8xOOGlktqgLw7KSHZtzBP/XY +ufTsgsbSPZUd5cBPhMnZo0QoBmrXRazwa2rvTl/4EYIeOGM0ZlDUPpNz+jDDZq3/ +ky2X7wMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAj/ola09b5KROJ1WrIhVZPMq1 +CtRK26vdoV9TxaBXOcLORyu+OshWv8LZJxA6sQU8wHcxuzrTBXttmhwwjIDLk5Mq +g6sFUYICABFna/OIYUdfA5PVWw3g8dShMjWFsjrbsIKr0csKvE+MW8VLADsfKoKm +fjaF3H48ZwC15DtS4KjrXRX5xm3wrR0OhbepmnMUWluPQSjA1egtTaRezarZ7c7c +2NU8Qh0XwRJdRTjDOPP8hS6DRkiy1yBfkjaP53kPmF6Z6PDQpLv1U70qzlmwr25/ +bLvSHgCwIe34QWKCudiyxLtGUPMxxY8BqHTr9Xgn2uf3ZkPznoM+IKrDNWCRzg== +-----END CERTIFICATE----- + +Equifax Secure Global eBusiness CA +================================== + +-----BEGIN CERTIFICATE----- +MIICkDCCAfmgAwIBAgIBATANBgkqhkiG9w0BAQQFADBaMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEtMCsGA1UEAxMkRXF1aWZheCBT +ZWN1cmUgR2xvYmFsIGVCdXNpbmVzcyBDQS0xMB4XDTk5MDYyMTA0MDAwMFoXDTIw +MDYyMTA0MDAwMFowWjELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0VxdWlmYXggU2Vj +dXJlIEluYy4xLTArBgNVBAMTJEVxdWlmYXggU2VjdXJlIEdsb2JhbCBlQnVzaW5l +c3MgQ0EtMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAuucXkAJlsTRVPEnC +UdXfp9E3j9HngXNBUmCbnaEXJnitx7HoJpQytd4zjTov2/KaelpzmKNc6fuKcxtc +58O/gGzNqfTWK8D3+ZmqY6KxRwIP1ORROhI8bIpaVIRw28HFkM9yRcuoWcDNM50/ +o5brhTMhHD4ePmBudpxnhcXIw2ECAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgAH +MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUvqigdHJQa0S3ySPY+6j/s1dr +aGwwHQYDVR0OBBYEFL6ooHRyUGtEt8kj2Puo/7NXa2hsMA0GCSqGSIb3DQEBBAUA +A4GBADDiAVGqx+pf2rnQZQ8w1j7aDRRJbpGTJxQx78T3LUX47Me/okENI7SS+RkA +Z70Br83gcfxaz2TE4JaY0KNA4gGK7ycH8WUBikQtBmV1UsCGECAhX2xrD2yuCRyv +8qIYNMR1pHMc8Y3c7635s3a0kr/clRAevsvIO1qEYBlWlKlV +-----END CERTIFICATE----- + +Equifax Secure eBusiness CA 1 +============================= + +-----BEGIN CERTIFICATE----- +MIICgjCCAeugAwIBAgIBBDANBgkqhkiG9w0BAQQFADBTMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEmMCQGA1UEAxMdRXF1aWZheCBT +ZWN1cmUgZUJ1c2luZXNzIENBLTEwHhcNOTkwNjIxMDQwMDAwWhcNMjAwNjIxMDQw +MDAwWjBTMQswCQYDVQQGEwJVUzEcMBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5j +LjEmMCQGA1UEAxMdRXF1aWZheCBTZWN1cmUgZUJ1c2luZXNzIENBLTEwgZ8wDQYJ +KoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vGbwXt3fek6lfWg0XTzQaDJj0ItlZ1MRo +RvC0NcWFAyDGr0WlIVFFQesWWDYyb+JQYmT5/VGcqiTZ9J2DKocKIdMSODRsjQBu +WqDZQu4aIZX5UkxVWsUPOE9G+m34LjXWHXzr4vCwdYDIqROsvojvOm6rXyo4YgKw +Env+j6YDAgMBAAGjZjBkMBEGCWCGSAGG+EIBAQQEAwIABzAPBgNVHRMBAf8EBTAD +AQH/MB8GA1UdIwQYMBaAFEp4MlIR21kWNl7fwRQ2QGpHfEyhMB0GA1UdDgQWBBRK +eDJSEdtZFjZe38EUNkBqR3xMoTANBgkqhkiG9w0BAQQFAAOBgQB1W6ibAxHm6VZM +zfmpTMANmvPMZWnmJXbMWbfWVMMdzZmsGd20hdXgPfxiIKeES1hl8eL5lSE/9dR+ +WB5Hh1Q+WKG1tfgq73HnvMP2sUlG4tega+VWeponmHxGYhTnyfxuAxJ5gDgdSIKN +/Bf+KpYrtWKmpj29f5JZzVoqgrI3eQ== +-----END CERTIFICATE----- + +Equifax Secure eBusiness CA 2 +============================= + +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIEN3DPtTANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2Vj +dXJlIGVCdXNpbmVzcyBDQS0yMB4XDTk5MDYyMzEyMTQ0NVoXDTE5MDYyMzEyMTQ0 +NVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkVxdWlmYXggU2VjdXJlMSYwJAYD +VQQLEx1FcXVpZmF4IFNlY3VyZSBlQnVzaW5lc3MgQ0EtMjCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEA5Dk5kx5SBhsoNviyoynF7Y6yEb3+6+e0dMKP/wXn2Z0G +vxLIPw7y1tEkshHe0XMJitSxLJgJDR5QRrKDpkWNYmi7hRsgcDKqQM2mll/EcTc/ +BPO3QSQ5BxoeLmFYoBIL5aXfxavqN3HMHMg3OrmXUqesxWoklE6ce8/AatbfIb0C +AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEX +MBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2VjdXJl +IGVCdXNpbmVzcyBDQS0yMQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTkw +NjIzMTIxNDQ1WjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUUJ4L6q9euSBIplBq +y/3YIHqngnYwHQYDVR0OBBYEFFCeC+qvXrkgSKZQasv92CB6p4J2MAwGA1UdEwQF +MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA +A4GBAAyGgq3oThr1jokn4jVYPSm0B482UJW/bsGe68SQsoWou7dC4A8HOd/7npCy +0cE+U58DRLB+S/Rv5Hwf5+Kx5Lia78O9zt4LMjTZ3ijtM2vE1Nc9ElirfQkty3D1 +E4qUoSek1nDFbZS1yX2doNLGCEnZZpum0/QL3MUmV+GRMOrN +-----END CERTIFICATE----- + +Thawte Time Stamping CA +======================= + +-----BEGIN CERTIFICATE----- +MIICoTCCAgqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBizELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzAN +BgNVBAoTBlRoYXd0ZTEdMBsGA1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAd +BgNVBAMTFlRoYXd0ZSBUaW1lc3RhbXBpbmcgQ0EwHhcNOTcwMTAxMDAwMDAwWhcN +MjAxMjMxMjM1OTU5WjCBizELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4g +Q2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzANBgNVBAoTBlRoYXd0ZTEdMBsG +A1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAdBgNVBAMTFlRoYXd0ZSBUaW1l +c3RhbXBpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYrWHhhRYZT +6jR7UZztsOYuGA7+4F+oJ9O0yeB8WU4WDnNUYMF/9p8u6TqFJBU820cEY8OexJQa +Wt9MevPZQx08EHp5JduQ/vBR5zDWQQD9nyjfeb6Uu522FOMjhdepQeBMpHmwKxqL +8vg7ij5FrHGSALSQQZj7X+36ty6K+Ig3AgMBAAGjEzARMA8GA1UdEwEB/wQFMAMB +Af8wDQYJKoZIhvcNAQEEBQADgYEAZ9viwuaHPUCDhjc1fR/OmsMMZiCouqoEiYbC +9RAIDb/LogWK0E02PvTX72nGXuSwlG9KuefeW4i2e9vjJ+V2w/A1wcu1J5szedyQ +pgCed/r8zSeUQhac0xxo7L9c3eWpexAKMnRUEzGLhQOEkbdYATAUOK8oyvyxUBkZ +CayJSdM= +-----END CERTIFICATE----- + +thawte Primary Root CA +====================== + +-----BEGIN CERTIFICATE----- +MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB +qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV +BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw +NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j +LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG +A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl +IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs +W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta +3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk +6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6 +Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J +NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA +MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP +r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU +DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz +YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX +xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2 +/qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/ +LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7 +jVaMaA== +-----END CERTIFICATE----- + +VeriSign Class 3 Public Primary Certification Authority - G5 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB +yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp +U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW +ZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL +MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW +ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln +biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp +U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y +aXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1 +nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex +t0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz +SdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG +BO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+ +rCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/ +NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH +BgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy +aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv +MzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE +p6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y +5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK +WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ +4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N +hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq +-----END CERTIFICATE----- + +Entrust.net Secure Server Certification Authority +================================================= + +-----BEGIN CERTIFICATE----- +MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC +VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u +ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc +KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u +ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1 +MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE +ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j +b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF +bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg +U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA +A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/ +I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3 +wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC +AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb +oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5 +BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p +dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk +MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp +b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu +dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0 +MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi +E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa +MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI +hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN +95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd +2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI= +-----END CERTIFICATE----- + +Go Daddy Certification Authority Root Certificate Bundle +======================================================== + +-----BEGIN CERTIFICATE----- +MIIE3jCCA8agAwIBAgICAwEwDQYJKoZIhvcNAQEFBQAwYzELMAkGA1UEBhMCVVMx +ITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28g +RGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjExMTYw +MTU0MzdaFw0yNjExMTYwMTU0MzdaMIHKMQswCQYDVQQGEwJVUzEQMA4GA1UECBMH +QXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5j +b20sIEluYy4xMzAxBgNVBAsTKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5j +b20vcmVwb3NpdG9yeTEwMC4GA1UEAxMnR28gRGFkZHkgU2VjdXJlIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5MREwDwYDVQQFEwgwNzk2OTI4NzCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMQt1RWMnCZM7DI161+4WQFapmGBWTtwY6vj3D3H +KrjJM9N55DrtPDAjhI6zMBS2sofDPZVUBJ7fmd0LJR4h3mUpfjWoqVTr9vcyOdQm +VZWt7/v+WIbXnvQAjYwqDL1CBM6nPwT27oDyqu9SoWlm2r4arV3aLGbqGmu75RpR +SgAvSMeYddi5Kcju+GZtCpyz8/x4fKL4o/K1w/O5epHBp+YlLpyo7RJlbmr2EkRT +cDCVw5wrWCs9CHRK8r5RsL+H0EwnWGu1NcWdrxcx+AuP7q2BNgWJCJjPOq8lh8BJ +6qf9Z/dFjpfMFDniNoW1fho3/Rb2cRGadDAW/hOUoz+EDU8CAwEAAaOCATIwggEu +MB0GA1UdDgQWBBT9rGEyk2xF1uLuhV+auud2mWjM5zAfBgNVHSMEGDAWgBTSxLDS +kdRMEXGzYcs9of7dqGrU4zASBgNVHRMBAf8ECDAGAQH/AgEAMDMGCCsGAQUFBwEB +BCcwJTAjBggrBgEFBQcwAYYXaHR0cDovL29jc3AuZ29kYWRkeS5jb20wRgYDVR0f +BD8wPTA7oDmgN4Y1aHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNvbS9yZXBv +c2l0b3J5L2dkcm9vdC5jcmwwSwYDVR0gBEQwQjBABgRVHSAAMDgwNgYIKwYBBQUH +AgEWKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5jb20vcmVwb3NpdG9yeTAO +BgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBANKGwOy9+aG2Z+5mC6IG +OgRQjhVyrEp0lVPLN8tESe8HkGsz2ZbwlFalEzAFPIUyIXvJxwqoJKSQ3kbTJSMU +A2fCENZvD117esyfxVgqwcSeIaha86ykRvOe5GPLL5CkKSkB2XIsKd83ASe8T+5o +0yGPwLPk9Qnt0hCqU7S+8MxZC9Y7lhyVJEnfzuz9p0iRFEUOOjZv2kWzRaJBydTX +RE4+uXR21aITVSzGh6O1mawGhId/dQb8vxRMDsxuxN89txJx9OjxUUAiKEngHUuH +qDTMBqLdElrRhjZkAzVvb3du6/KFUJheqwNTrZEjYx8WnM25sgVjOuH0aBsXBTWV +U+4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIE+zCCBGSgAwIBAgICAQ0wDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1Zh +bGlDZXJ0IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIElu +Yy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24g +QXV0aG9yaXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAe +BgkqhkiG9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTA0MDYyOTE3MDYyMFoX +DTI0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBE +YWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3MgMiBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgC +ggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv +2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+q +N1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiO +r18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lN +f4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+YihfukEH +U1jPEX44dMX4/7VpkI+EdOqXG68CAQOjggHhMIIB3TAdBgNVHQ4EFgQU0sSw0pHU +TBFxs2HLPaH+3ahq1OMwgdIGA1UdIwSByjCBx6GBwaSBvjCBuzEkMCIGA1UEBxMb +VmFsaUNlcnQgVmFsaWRhdGlvbiBOZXR3b3JrMRcwFQYDVQQKEw5WYWxpQ2VydCwg +SW5jLjE1MDMGA1UECxMsVmFsaUNlcnQgQ2xhc3MgMiBQb2xpY3kgVmFsaWRhdGlv +biBBdXRob3JpdHkxITAfBgNVBAMTGGh0dHA6Ly93d3cudmFsaWNlcnQuY29tLzEg +MB4GCSqGSIb3DQEJARYRaW5mb0B2YWxpY2VydC5jb22CAQEwDwYDVR0TAQH/BAUw +AwEB/zAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3NwLmdv +ZGFkZHkuY29tMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jZXJ0aWZpY2F0ZXMu +Z29kYWRkeS5jb20vcmVwb3NpdG9yeS9yb290LmNybDBLBgNVHSAERDBCMEAGBFUd +IAAwODA2BggrBgEFBQcCARYqaHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNv +bS9yZXBvc2l0b3J5MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOBgQC1 +QPmnHfbq/qQaQlpE9xXUhUaJwL6e4+PrxeNYiY+Sn1eocSxI0YGyeR+sBjUZsE4O +WBsUs5iB0QQeyAfJg594RAoYC5jcdnplDQ1tgMQLARzLrUc+cb53S8wGd9D0Vmsf +SxOaFIqII6hR8INMqzW/Rn453HWkrugp++85j09VZw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 +IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz +BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y +aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG +9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy +NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y +azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs +YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw +Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl +cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY +dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9 +WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS +v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v +UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu +IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC +W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd +-----END CERTIFICATE----- + +GeoTrust Global CA +================== + +-----BEGIN CERTIFICATE----- +MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT +MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 +aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAw +WjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UE +AxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9m +OSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIu +T8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6c +JmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmR +Cw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5asz +PeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjm +aPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrM +TjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+g +LaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBO +BgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2Vv +dHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GB +AHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrL +NhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1W +b8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S +-----END CERTIFICATE----- + diff --git a/libs/httplib2/iri2uri.py b/libs/httplib2/iri2uri.py index 70667edf..d88c91fd 100644 --- a/libs/httplib2/iri2uri.py +++ b/libs/httplib2/iri2uri.py @@ -16,7 +16,7 @@ import urlparse # Convert an IRI to a URI following the rules in RFC 3987 -# +# # The characters we need to enocde and escape are defined in the spec: # # iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD @@ -28,28 +28,28 @@ import urlparse # / %xD0000-DFFFD / %xE1000-EFFFD escape_range = [ - (0xA0, 0xD7FF ), - (0xE000, 0xF8FF ), - (0xF900, 0xFDCF ), - (0xFDF0, 0xFFEF), - (0x10000, 0x1FFFD ), - (0x20000, 0x2FFFD ), - (0x30000, 0x3FFFD), - (0x40000, 0x4FFFD ), - (0x50000, 0x5FFFD ), - (0x60000, 0x6FFFD), - (0x70000, 0x7FFFD ), - (0x80000, 0x8FFFD ), - (0x90000, 0x9FFFD), - (0xA0000, 0xAFFFD ), - (0xB0000, 0xBFFFD ), - (0xC0000, 0xCFFFD), - (0xD0000, 0xDFFFD ), - (0xE1000, 0xEFFFD), - (0xF0000, 0xFFFFD ), - (0x100000, 0x10FFFD) + (0xA0, 0xD7FF), + (0xE000, 0xF8FF), + (0xF900, 0xFDCF), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD), + (0x20000, 0x2FFFD), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD), + (0x50000, 0x5FFFD), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD), + (0x80000, 0x8FFFD), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD), + (0xB0000, 0xBFFFD), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD), + (0x100000, 0x10FFFD), ] - + def encode(c): retval = c i = ord(c) @@ -63,19 +63,19 @@ def encode(c): def iri2uri(uri): - """Convert an IRI to a URI. Note that IRIs must be + """Convert an IRI to a URI. Note that IRIs must be passed in a unicode strings. That is, do not utf-8 encode - the IRI before passing it into the function.""" + the IRI before passing it into the function.""" if isinstance(uri ,unicode): (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) authority = authority.encode('idna') # For each character in 'ucschar' or 'iprivate' # 1. encode as utf-8 - # 2. then %-encode each octet of that utf-8 + # 2. then %-encode each octet of that utf-8 uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) uri = "".join([encode(c) for c in uri]) return uri - + if __name__ == "__main__": import unittest @@ -83,7 +83,7 @@ if __name__ == "__main__": def test_uris(self): """Test that URIs are invariant under the transformation.""" - invariant = [ + invariant = [ u"ftp://ftp.is.co.za/rfc/rfc1808.txt", u"http://www.ietf.org/rfc/rfc2396.txt", u"ldap://[2001:db8::7]/c=GB?objectClass?one", @@ -94,7 +94,7 @@ if __name__ == "__main__": u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] for uri in invariant: self.assertEqual(uri, iri2uri(uri)) - + def test_iri(self): """ Test that the right type of escaping is done for each part of the URI.""" self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) @@ -107,4 +107,4 @@ if __name__ == "__main__": unittest.main() - + diff --git a/libs/httplib2/socks.py b/libs/httplib2/socks.py new file mode 100644 index 00000000..0991f4cf --- /dev/null +++ b/libs/httplib2/socks.py @@ -0,0 +1,438 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +""" + +import base64 +import socket +import struct +import sys + +if getattr(socket, 'socket', None) is None: + raise ImportError('socket.socket missing, proxy support unusable') + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 +PROXY_TYPE_HTTP_NO_TUNNEL = 4 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): pass +class GeneralProxyError(ProxyError): pass +class Socks5AuthError(ProxyError): pass +class Socks5Error(ProxyError): pass +class Socks4Error(ProxyError): pass +class HTTPError(ProxyError): pass + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + +def wrapmodule(module): + """wrapmodule(module) + Attempts to replace a module's socket library with a SOCKS socket. Must set + a default proxy using setdefaultproxy(...) first. + This will only work on modules that import socket directly into the namespace; + most of the Python Standard Library falls into this category. + """ + if _defaultproxy != None: + module.socket.socket = socksocket + else: + raise GeneralProxyError((4, "no proxy specified")) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + self.__httptunnel = True + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = self.recv(count) + while len(data) < count: + d = self.recv(count-len(data)) + if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) + data = data + d + return data + + def sendall(self, content, *args): + """ override socket.socket.sendall method to rewrite the header + for non-tunneling proxies if needed + """ + if not self.__httptunnel: + content = self.__rewriteproxy(content) + return super(socksocket, self).sendall(content, *args) + + def __rewriteproxy(self, header): + """ rewrite HTTP request headers to support non-tunneling proxies + (i.e. those which do not support the CONNECT method). + This only works for HTTP (not HTTPS) since HTTPS requires tunneling. + """ + host, endpt = None, None + hdrs = header.split("\r\n") + for hdr in hdrs: + if hdr.lower().startswith("host:"): + host = hdr + elif hdr.lower().startswith("get") or hdr.lower().startswith("post"): + endpt = hdr + if host and endpt: + hdrs.remove(host) + hdrs.remove(endpt) + host = host.split(" ")[1] + endpt = endpt.split(" ") + if (self.__proxy[4] != None and self.__proxy[5] != None): + hdrs.insert(0, self.__getauthheader()) + hdrs.insert(0, "Host: %s" % host) + hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2])) + return "\r\n".join(hdrs) + + def __getauthheader(self): + auth = self.__proxy[4] + ":" + self.__proxy[5] + return "Proxy-Authorization: Basic " + base64.b64encode(auth) + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02)) + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00)) + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1:2] == chr(0x00).encode(): + # No authentication is required + pass + elif chosenauth[1:2] == chr(0x02).encode(): + # Okay, we need to perform a basic username/password + # authentication. + self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0:1] != chr(0x01).encode(): + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1:2] != chr(0x00).encode(): + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == chr(0xFF).encode(): + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = struct.pack('BBB', 0x05, 0x01, 0x00) + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + chr(0x01).encode() + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + chr(0x01).encode() + ipaddr + req = req + struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1:2] != chr(0x00).encode(): + # Connection failed + self.close() + if ord(resp[1:2])<=8: + raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3:4] == chr(0x01).encode(): + boundaddr = self.__recvall(4) + elif resp[3:4] == chr(0x03).encode(): + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4:5])) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01) + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + chr(0x00).encode() + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + chr(0x00).encode() + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0:1] != chr(0x00).encode(): + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1:2] != chr(0x5A).encode(): + # Server returned an error + self.close() + if ord(resp[1:2]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"] + headers += ["Host: ", destaddr, "\r\n"] + if (self.__proxy[4] != None and self.__proxy[5] != None): + headers += [self.__getauthheader(), "\r\n"] + headers.append("\r\n") + self.sendall("".join(headers).encode()) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n".encode()) == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ".encode(), 2) + if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self, despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + if destpair[1] == 443: + self.__negotiatehttp(destpair[0],destpair[1]) + else: + self.__httptunnel = False + elif self.__proxy[0] == None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/libs/pyasn1/__init__.py b/libs/pyasn1/__init__.py index 7de39fe5..88aff79c 100644 --- a/libs/pyasn1/__init__.py +++ b/libs/pyasn1/__init__.py @@ -1 +1,8 @@ -majorVersionId = '1' +import sys + +# http://www.python.org/dev/peps/pep-0396/ +__version__ = '0.1.7' + +if sys.version_info[:2] < (2, 4): + raise RuntimeError('PyASN1 requires Python 2.4 or later') + diff --git a/libs/pyasn1/codec/__init__.py b/libs/pyasn1/codec/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/__init__.py +++ b/libs/pyasn1/codec/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/ber/__init__.py b/libs/pyasn1/codec/ber/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/ber/__init__.py +++ b/libs/pyasn1/codec/ber/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/ber/decoder.py b/libs/pyasn1/codec/ber/decoder.py index ae9311cb..be0cf490 100644 --- a/libs/pyasn1/codec/ber/decoder.py +++ b/libs/pyasn1/codec/ber/decoder.py @@ -1,21 +1,24 @@ # BER decoder from pyasn1.type import tag, base, univ, char, useful, tagmap from pyasn1.codec.ber import eoo -from pyasn1.compat.octets import oct2int, octs2ints -from pyasn1 import error +from pyasn1.compat.octets import oct2int, octs2ints, isOctetsType +from pyasn1 import debug, error class AbstractDecoder: protoComponent = None def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - raise error.PyAsn1Error('Decoder not implemented for %s' % tagSet) + length, state, decodeFun, substrateFun): + raise error.PyAsn1Error('Decoder not implemented for %s' % (tagSet,)) def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % tagSet) + length, state, decodeFun, substrateFun): + raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,)) class AbstractSimpleDecoder(AbstractDecoder): + tagFormats = (tag.tagFormatSimple,) def _createComponent(self, asn1Spec, tagSet, value=None): + if tagSet[0][1] not in self.tagFormats: + raise error.PyAsn1Error('Invalid tag format %r for %r' % (tagSet[0], self.protoComponent,)) if asn1Spec is None: return self.protoComponent.clone(value, tagSet) elif value is None: @@ -24,7 +27,10 @@ class AbstractSimpleDecoder(AbstractDecoder): return asn1Spec.clone(value) class AbstractConstructedDecoder(AbstractDecoder): + tagFormats = (tag.tagFormatConstructed,) def _createComponent(self, asn1Spec, tagSet, value=None): + if tagSet[0][1] not in self.tagFormats: + raise error.PyAsn1Error('Invalid tag format %r for %r' % (tagSet[0], self.protoComponent,)) if asn1Spec is None: return self.protoComponent.clone(tagSet) else: @@ -32,19 +38,34 @@ class AbstractConstructedDecoder(AbstractDecoder): class EndOfOctetsDecoder(AbstractSimpleDecoder): def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - return eoo.endOfOctets, substrate[:length] + length, state, decodeFun, substrateFun): + return eoo.endOfOctets, substrate[length:] class ExplicitTagDecoder(AbstractSimpleDecoder): + protoComponent = univ.Any('') + tagFormats = (tag.tagFormatConstructed,) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - return decodeFun(substrate[:length], asn1Spec, tagSet, length) + length, state, decodeFun, substrateFun): + if substrateFun: + return substrateFun( + self._createComponent(asn1Spec, tagSet, ''), + substrate, length + ) + head, tail = substrate[:length], substrate[length:] + value, _ = decodeFun(head, asn1Spec, tagSet, length) + return value, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): + if substrateFun: + return substrateFun( + self._createComponent(asn1Spec, tagSet, ''), + substrate, length + ) value, substrate = decodeFun(substrate, asn1Spec, tagSet, length) terminator, substrate = decodeFun(substrate) - if terminator == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(terminator) and \ + terminator == eoo.endOfOctets: return value, substrate else: raise error.PyAsn1Error('Missing end-of-octets terminator') @@ -71,79 +92,71 @@ class IntegerDecoder(AbstractSimpleDecoder): '\xfb': -5 } - def _valueFilter(self, value): - try: - return int(value) - except OverflowError: - return value - def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: - raise error.PyAsn1Error('Empty substrate') - if substrate in self.precomputedValues: - value = self.precomputedValues[substrate] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: + return self._createComponent(asn1Spec, tagSet, 0), tail + if head in self.precomputedValues: + value = self.precomputedValues[head] else: - firstOctet = oct2int(substrate[0]) + firstOctet = oct2int(head[0]) if firstOctet & 0x80: value = -1 else: value = 0 - for octet in substrate: + for octet in head: value = value << 8 | oct2int(octet) - value = self._valueFilter(value) - return self._createComponent(asn1Spec, tagSet, value), substrate + return self._createComponent(asn1Spec, tagSet, value), tail class BooleanDecoder(IntegerDecoder): protoComponent = univ.Boolean(0) - def _valueFilter(self, value): - if value: - return 1 - else: - return 0 + def _createComponent(self, asn1Spec, tagSet, value=None): + return IntegerDecoder._createComponent(self, asn1Spec, tagSet, value and 1 or 0) class BitStringDecoder(AbstractSimpleDecoder): protoComponent = univ.BitString(()) + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] if tagSet[0][1] == tag.tagFormatSimple: # XXX what tag to check? - if not substrate: - raise error.PyAsn1Error('Missing initial octet') - trailingBits = oct2int(substrate[0]) + if not head: + raise error.PyAsn1Error('Empty substrate') + trailingBits = oct2int(head[0]) if trailingBits > 7: raise error.PyAsn1Error( 'Trailing bits overflow %s' % trailingBits ) - substrate = substrate[1:] - lsb = p = 0; l = len(substrate)-1; b = () + head = head[1:] + lsb = p = 0; l = len(head)-1; b = () while p <= l: if p == l: lsb = trailingBits j = 7 - o = oct2int(substrate[p]) + o = oct2int(head[p]) while j >= lsb: b = b + ((o>>j)&0x01,) j = j - 1 p = p + 1 - return self._createComponent(asn1Spec, tagSet, b), '' + return self._createComponent(asn1Spec, tagSet, b), tail r = self._createComponent(asn1Spec, tagSet, ()) - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun(substrate) + if substrateFun: + return substrateFun(r, substrate, length) + while head: + component, head = decodeFun(head) r = r + component - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -154,27 +167,29 @@ class BitStringDecoder(AbstractSimpleDecoder): class OctetStringDecoder(AbstractSimpleDecoder): protoComponent = univ.OctetString('') + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] if tagSet[0][1] == tag.tagFormatSimple: # XXX what tag to check? - return self._createComponent(asn1Spec, tagSet, substrate), '' + return self._createComponent(asn1Spec, tagSet, head), tail r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun(substrate) + if substrateFun: + return substrateFun(r, substrate, length) + while head: + component, head = decodeFun(head) r = r + component - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -186,93 +201,89 @@ class OctetStringDecoder(AbstractSimpleDecoder): class NullDecoder(AbstractSimpleDecoder): protoComponent = univ.Null('') def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) - if substrate: - raise error.PyAsn1Error('Unexpected substrate for Null') - return r, substrate + if head: + raise error.PyAsn1Error('Unexpected %d-octet substrate for Null' % length) + return r, tail class ObjectIdentifierDecoder(AbstractSimpleDecoder): protoComponent = univ.ObjectIdentifier(()) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: raise error.PyAsn1Error('Empty substrate') - oid = (); index = 0 - # Get the first subid - subId = oct2int(substrate[index]) - oid = oid + divmod(subId, 40) - index = index + 1 - substrateLen = len(substrate) - + # Get the first subid + subId = oct2int(head[0]) + oid = divmod(subId, 40) + + index = 1 + substrateLen = len(head) while index < substrateLen: - subId = oct2int(substrate[index]) - if subId < 128: - oid = oid + (subId,) - index = index + 1 - else: + subId = oct2int(head[index]) + index = index + 1 + if subId == 128: + # ASN.1 spec forbids leading zeros (0x80) in sub-ID OID + # encoding, tolerating it opens a vulnerability. + # See http://www.cosic.esat.kuleuven.be/publications/article-1432.pdf page 7 + raise error.PyAsn1Error('Invalid leading 0x80 in sub-OID') + elif subId > 128: # Construct subid from a number of octets nextSubId = subId subId = 0 - while nextSubId >= 128 and index < substrateLen: + while nextSubId >= 128: subId = (subId << 7) + (nextSubId & 0x7F) + if index >= substrateLen: + raise error.SubstrateUnderrunError( + 'Short substrate for sub-OID past %s' % (oid,) + ) + nextSubId = oct2int(head[index]) index = index + 1 - nextSubId = oct2int(substrate[index]) - if index == substrateLen: - raise error.SubstrateUnderrunError( - 'Short substrate for OID %s' % oid - ) subId = (subId << 7) + nextSubId - oid = oid + (subId,) - index = index + 1 - return self._createComponent(asn1Spec, tagSet, oid), substrate[index:] + oid = oid + (subId,) + return self._createComponent(asn1Spec, tagSet, oid), tail class RealDecoder(AbstractSimpleDecoder): protoComponent = univ.Real() def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] - if not length: - raise error.SubstrateUnderrunError('Short substrate for Real') - fo = oct2int(substrate[0]); substrate = substrate[1:] - if fo & 0x40: # infinite value - value = fo & 0x01 and '-inf' or 'inf' - elif fo & 0x80: # binary enoding - if fo & 0x11 == 0: - n = 1 - elif fo & 0x01: - n = 2 - elif fo & 0x02: - n = 3 - else: - n = oct2int(substrate[0]) - eo, substrate = substrate[:n], substrate[n:] - if not eo or not substrate: + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: + return self._createComponent(asn1Spec, tagSet, 0.0), tail + fo = oct2int(head[0]); head = head[1:] + if fo & 0x80: # binary enoding + n = (fo & 0x03) + 1 + if n == 4: + n = oct2int(head[0]) + eo, head = head[:n], head[n:] + if not eo or not head: raise error.PyAsn1Error('Real exponent screwed') - e = 0 + e = oct2int(eo[0]) & 0x80 and -1 or 0 while eo: # exponent e <<= 8 e |= oct2int(eo[0]) eo = eo[1:] p = 0 - while substrate: # value + while head: # value p <<= 8 - p |= oct2int(substrate[0]) - substrate = substrate[1:] + p |= oct2int(head[0]) + head = head[1:] if fo & 0x40: # sign bit p = -p value = (p, 2, e) + elif fo & 0x40: # infinite value + value = fo & 0x01 and '-inf' or 'inf' elif fo & 0xc0 == 0: # character encoding try: if fo & 0x3 == 0x1: # NR1 - value = (int(substrate), 10, 0) + value = (int(head), 10, 0) elif fo & 0x3 == 0x2: # NR2 - value = float(substrate) + value = float(head) elif fo & 0x3 == 0x3: # NR3 - value = float(substrate) + value = float(head) else: raise error.SubstrateUnderrunError( 'Unknown NR (tag %s)' % fo @@ -281,13 +292,11 @@ class RealDecoder(AbstractSimpleDecoder): raise error.SubstrateUnderrunError( 'Bad character Real syntax' ) - elif fo & 0xc0 == 0x40: # special real value - pass else: raise error.SubstrateUnderrunError( 'Unknown encoding (tag %s)' % fo ) - return self._createComponent(asn1Spec, tagSet, value), substrate + return self._createComponent(asn1Spec, tagSet, value), tail class SequenceDecoder(AbstractConstructedDecoder): protoComponent = univ.Sequence() @@ -301,17 +310,15 @@ class SequenceDecoder(AbstractConstructedDecoder): return r.getComponentPositionNearType(t, idx) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) idx = 0 - if not decodeFun: - return r, substrate - while substrate: + if substrateFun: + return substrateFun(r, substrate, length) + while head: asn1Spec = self._getComponentTagMap(r, idx) - component, substrate = decodeFun( - substrate, asn1Spec - ) + component, head = decodeFun(head, asn1Spec) idx = self._getComponentPositionByType( r, component.getEffectiveTagSet(), idx ) @@ -319,18 +326,19 @@ class SequenceDecoder(AbstractConstructedDecoder): idx = idx + 1 r.setDefaultComponents() r.verifySizeSpec() - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) idx = 0 while substrate: asn1Spec = self._getComponentTagMap(r, idx) - if not decodeFun: - return r, substrate component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break idx = self._getComponentPositionByType( r, component.getEffectiveTagSet(), idx @@ -348,32 +356,31 @@ class SequenceDecoder(AbstractConstructedDecoder): class SequenceOfDecoder(AbstractConstructedDecoder): protoComponent = univ.SequenceOf() def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) asn1Spec = r.getComponentType() idx = 0 - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun( - substrate, asn1Spec - ) + while head: + component, head = decodeFun(head, asn1Spec) r.setComponentByPosition(idx, component, asn1Spec is None) idx = idx + 1 r.verifySizeSpec() - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) asn1Spec = r.getComponentType() idx = 0 - if not decodeFun: - return r, substrate while substrate: component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r.setComponentByPosition(idx, component, asn1Spec is None) idx = idx + 1 @@ -401,43 +408,68 @@ class SetOfDecoder(SequenceOfDecoder): class ChoiceDecoder(AbstractConstructedDecoder): protoComponent = univ.Choice() + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) if r.getTagSet() == tagSet: # explicitly tagged Choice - component, substrate = decodeFun( - substrate, r.getComponentTagMap() + component, head = decodeFun( + head, r.getComponentTagMap() ) else: - component, substrate = decodeFun( - substrate, r.getComponentTagMap(), tagSet, length, state + component, head = decodeFun( + head, r.getComponentTagMap(), tagSet, length, state ) if isinstance(component, univ.Choice): effectiveTagSet = component.getEffectiveTagSet() else: effectiveTagSet = component.getTagSet() r.setComponentByType(effectiveTagSet, component, 0, asn1Spec is None) + return r, tail + + def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, + length, state, decodeFun, substrateFun): + r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) + if r.getTagSet() == tagSet: # explicitly tagged Choice + component, substrate = decodeFun(substrate, r.getComponentTagMap()) + eooMarker, substrate = decodeFun(substrate) # eat up EOO marker + if not eoo.endOfOctets.isSameTypeWith(eooMarker) or \ + eooMarker != eoo.endOfOctets: + raise error.PyAsn1Error('No EOO seen before substrate ends') + else: + component, substrate= decodeFun( + substrate, r.getComponentTagMap(), tagSet, length, state + ) + if isinstance(component, univ.Choice): + effectiveTagSet = component.getEffectiveTagSet() + else: + effectiveTagSet = component.getTagSet() + r.setComponentByType(effectiveTagSet, component, 0, asn1Spec is None) return r, substrate - indefLenValueDecoder = valueDecoder - class AnyDecoder(AbstractSimpleDecoder): protoComponent = univ.Any() + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): if asn1Spec is None or \ asn1Spec is not None and tagSet != asn1Spec.getTagSet(): # untagged Any container, recover inner header substrate length = length + len(fullSubstrate) - len(substrate) substrate = fullSubstrate - substrate = substrate[:length] - return self._createComponent(asn1Spec, tagSet, value=substrate), '' + if substrateFun: + return substrateFun(self._createComponent(asn1Spec, tagSet), + substrate, length) + head, tail = substrate[:length], substrate[length:] + return self._createComponent(asn1Spec, tagSet, value=head), tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): if asn1Spec is not None and tagSet == asn1Spec.getTagSet(): # tagged Any type -- consume header substrate header = '' @@ -450,11 +482,12 @@ class AnyDecoder(AbstractSimpleDecoder): # Any components do not inherit initial tag asn1Spec = self.protoComponent - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -550,7 +583,10 @@ class Decoder: self.__tagSetCache = {} def __call__(self, substrate, asn1Spec=None, tagSet=None, - length=None, state=stDecodeTag, recursiveFlag=1): + length=None, state=stDecodeTag, recursiveFlag=1, + substrateFun=None): + if debug.logger & debug.flagDecoder: + debug.logger('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate))) fullSubstrate = substrate while state != stStop: if state == stDecodeTag: @@ -559,6 +595,9 @@ class Decoder: raise error.SubstrateUnderrunError( 'Short octet stream on tag decoding' ) + if not isOctetsType(substrate) and \ + not isinstance(substrate, univ.OctetString): + raise error.PyAsn1Error('Bad octet stream type') firstOctet = substrate[0] substrate = substrate[1:] @@ -598,6 +637,7 @@ class Decoder: else: tagSet = lastTag + tagSet state = stDecodeLength + debug.logger and debug.logger & debug.flagDecoder and debug.logger('tag decoded into %r, decoding length' % tagSet) if state == stDecodeLength: # Decode length if not substrate: @@ -625,12 +665,13 @@ class Decoder: for char in lengthString: length = (length << 8) | oct2int(char) size = size + 1 - state = stGetValueDecoder substrate = substrate[size:] if length != -1 and len(substrate) < length: raise error.SubstrateUnderrunError( '%d-octet short' % (length - len(substrate)) ) + state = stGetValueDecoder + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length]))) if state == stGetValueDecoder: if asn1Spec is None: state = stGetValueDecoderByTag @@ -669,14 +710,27 @@ class Decoder: state = stDecodeValue else: state = stTryAsExplicitTag + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('codec %s chosen by a built-in type, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as explicit tag')) + debug.scope.push(concreteDecoder is None and '?' or concreteDecoder.protoComponent.__class__.__name__) if state == stGetValueDecoderByAsn1Spec: if isinstance(asn1Spec, (dict, tagmap.TagMap)): if tagSet in asn1Spec: __chosenSpec = asn1Spec[tagSet] else: __chosenSpec = None + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('candidate ASN.1 spec is a map of:') + for t, v in asn1Spec.getPosMap().items(): + debug.logger(' %r -> %s' % (t, v.__class__.__name__)) + if asn1Spec.getNegMap(): + debug.logger('but neither of: ') + for i in asn1Spec.getNegMap().items(): + debug.logger(' %r -> %s' % (t, v.__class__.__name__)) + debug.logger('new candidate ASN.1 spec is %s, chosen by %r' % (__chosenSpec is None and '<none>' or __chosenSpec.__class__.__name__, tagSet)) else: __chosenSpec = asn1Spec + debug.logger and debug.logger & debug.flagDecoder and debug.logger('candidate ASN.1 spec is %s' % asn1Spec.__class__.__name__) if __chosenSpec is not None and ( tagSet == __chosenSpec.getTagSet() or \ tagSet in __chosenSpec.getTagMap() @@ -687,9 +741,11 @@ class Decoder: __chosenSpec.typeId in self.__typeMap: # ambiguous type concreteDecoder = self.__typeMap[__chosenSpec.typeId] + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value decoder chosen for an ambiguous type by type ID %s' % (__chosenSpec.typeId,)) elif baseTagSet in self.__tagMap: # base type or tagged subtype concreteDecoder = self.__tagMap[baseTagSet] + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value decoder chosen by base %r' % (baseTagSet,)) else: concreteDecoder = None if concreteDecoder: @@ -700,8 +756,13 @@ class Decoder: elif tagSet == self.__endOfOctetsTagSet: concreteDecoder = self.__tagMap[tagSet] state = stDecodeValue + debug.logger and debug.logger & debug.flagDecoder and debug.logger('end-of-octets found') else: + concreteDecoder = None state = stTryAsExplicitTag + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('codec %s chosen by ASN.1 spec, decoding %s' % (state == stDecodeValue and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as explicit tag')) + debug.scope.push(__chosenSpec is None and '?' or __chosenSpec.__class__.__name__) if state == stTryAsExplicitTag: if tagSet and \ tagSet[0][1] == tag.tagFormatConstructed and \ @@ -710,34 +771,35 @@ class Decoder: concreteDecoder = explicitTagDecoder state = stDecodeValue else: + concreteDecoder = None state = self.defaultErrorState + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s chosen, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as failure')) if state == stDumpRawValue: concreteDecoder = self.defaultRawDecoder + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s chosen, decoding value' % concreteDecoder.__class__.__name__) state = stDecodeValue if state == stDecodeValue: - if recursiveFlag: - decodeFun = self - else: - decodeFun = None + if recursiveFlag == 0 and not substrateFun: # legacy + substrateFun = lambda a,b,c: (a,b[:c]) if length == -1: # indef length value, substrate = concreteDecoder.indefLenValueDecoder( fullSubstrate, substrate, asn1Spec, tagSet, length, - stGetValueDecoder, decodeFun + stGetValueDecoder, self, substrateFun ) else: - value, _substrate = concreteDecoder.valueDecoder( + value, substrate = concreteDecoder.valueDecoder( fullSubstrate, substrate, asn1Spec, tagSet, length, - stGetValueDecoder, decodeFun + stGetValueDecoder, self, substrateFun ) - if recursiveFlag: - substrate = substrate[length:] - else: - substrate = _substrate state = stStop + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, value.prettyPrint(), substrate and debug.hexdump(substrate) or '<none>')) if state == stErrorCondition: raise error.PyAsn1Error( '%r not in asn1Spec: %r' % (tagSet, asn1Spec) ) + if debug.logger and debug.logger & debug.flagDecoder: + debug.scope.pop() + debug.logger('decoder left scope %s, call completed' % debug.scope) return value, substrate decode = Decoder(tagMap, typeMap) diff --git a/libs/pyasn1/codec/ber/encoder.py b/libs/pyasn1/codec/ber/encoder.py index 2149b0ba..173949d0 100644 --- a/libs/pyasn1/codec/ber/encoder.py +++ b/libs/pyasn1/codec/ber/encoder.py @@ -1,8 +1,8 @@ # BER encoder from pyasn1.type import base, tag, univ, char, useful from pyasn1.codec.ber import eoo -from pyasn1.compat.octets import int2oct, ints2octs, null, str2octs -from pyasn1 import error +from pyasn1.compat.octets import int2oct, oct2int, ints2octs, null, str2octs +from pyasn1 import debug, error class Error(Exception): pass @@ -78,9 +78,24 @@ class ExplicitlyTaggedItemEncoder(AbstractItemEncoder): explicitlyTaggedItemEncoder = ExplicitlyTaggedItemEncoder() +class BooleanEncoder(AbstractItemEncoder): + supportIndefLenMode = 0 + _true = ints2octs((1,)) + _false = ints2octs((0,)) + def encodeValue(self, encodeFun, value, defMode, maxChunkSize): + return value and self._true or self._false, 0 + class IntegerEncoder(AbstractItemEncoder): supportIndefLenMode = 0 + supportCompactZero = False def encodeValue(self, encodeFun, value, defMode, maxChunkSize): + if value == 0: # shortcut for zero value + if self.supportCompactZero: + # this seems to be a correct way for encoding zeros + return null, 0 + else: + # this seems to be a widespread way for encoding zeros + return ints2octs((0,)), 0 octets = [] value = int(value) # to save on ops on asn1 type while 1: @@ -149,18 +164,15 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): index = 5 else: if len(oid) < 2: - raise error.PyAsn1Error('Short OID %s' % value) + raise error.PyAsn1Error('Short OID %s' % (value,)) # Build the first twos - index = 0 - subid = oid[index] * 40 - subid = subid + oid[index+1] - if subid < 0 or subid > 0xff: + if oid[0] > 6 or oid[1] > 39 or oid[0] == 6 and oid[1] > 15: raise error.PyAsn1Error( - 'Initial sub-ID overflow %s in OID %s' % (oid[index:], value) + 'Initial sub-ID overflow %s in OID %s' % (oid[:2], value) ) - octets = (subid,) - index = index + 2 + octets = (oid[0] * 40 + oid[1],) + index = 2 # Cycle through subids for subid in oid[index:]: @@ -184,6 +196,7 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): return ints2octs(octets), 0 class RealEncoder(AbstractItemEncoder): + supportIndefLenMode = 0 def encodeValue(self, encodeFun, value, defMode, maxChunkSize): if value.isPlusInfinity(): return int2oct(0x40), 0 @@ -206,9 +219,11 @@ class RealEncoder(AbstractItemEncoder): m >>= 1 e += 1 eo = null - while e: + while e not in (0, -1): eo = int2oct(e&0xff) + eo e >>= 8 + if e == 0 and eo and oct2int(eo[0]) & 0x80: + eo = int2oct(0) + eo n = len(eo) if n > 0xff: raise error.PyAsn1Error('Real exponent overflow') @@ -268,7 +283,7 @@ class AnyEncoder(OctetStringEncoder): tagMap = { eoo.endOfOctets.tagSet: EndOfOctetsEncoder(), - univ.Boolean.tagSet: IntegerEncoder(), + univ.Boolean.tagSet: BooleanEncoder(), univ.Integer.tagSet: IntegerEncoder(), univ.BitString.tagSet: BitStringEncoder(), univ.OctetString.tagSet: OctetStringEncoder(), @@ -313,6 +328,7 @@ class Encoder: self.__typeMap = typeMap def __call__(self, value, defMode=1, maxChunkSize=0): + debug.logger & debug.flagEncoder and debug.logger('encoder called in %sdef mode, chunk size %s for type %s, value:\n%s' % (not defMode and 'in' or '', maxChunkSize, value.__class__.__name__, value.prettyPrint())) tagSet = value.getTagSet() if len(tagSet) > 1: concreteEncoder = explicitlyTaggedItemEncoder @@ -322,13 +338,16 @@ class Encoder: elif tagSet in self.__tagMap: concreteEncoder = self.__tagMap[tagSet] else: - baseTagSet = value.baseTagSet - if baseTagSet in self.__tagMap: - concreteEncoder = self.__tagMap[baseTagSet] + tagSet = value.baseTagSet + if tagSet in self.__tagMap: + concreteEncoder = self.__tagMap[tagSet] else: - raise Error('No encoder for %s' % value) - return concreteEncoder.encode( + raise Error('No encoder for %s' % (value,)) + debug.logger & debug.flagEncoder and debug.logger('using value codec %s chosen by %r' % (concreteEncoder.__class__.__name__, tagSet)) + substrate = concreteEncoder.encode( self, value, defMode, maxChunkSize ) + debug.logger & debug.flagEncoder and debug.logger('built %s octets of substrate: %s\nencoder completed' % (len(substrate), debug.hexdump(substrate))) + return substrate encode = Encoder(tagMap, typeMap) diff --git a/libs/pyasn1/codec/cer/__init__.py b/libs/pyasn1/codec/cer/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/cer/__init__.py +++ b/libs/pyasn1/codec/cer/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/cer/decoder.py b/libs/pyasn1/codec/cer/decoder.py index 71395d22..9fd37c13 100644 --- a/libs/pyasn1/codec/cer/decoder.py +++ b/libs/pyasn1/codec/cer/decoder.py @@ -7,22 +7,25 @@ from pyasn1 import error class BooleanDecoder(decoder.AbstractSimpleDecoder): protoComponent = univ.Boolean(0) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: raise error.PyAsn1Error('Empty substrate') - byte = oct2int(substrate[0]) + byte = oct2int(head[0]) + # CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while + # BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1 + # in http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf if byte == 0xff: value = 1 elif byte == 0x00: value = 0 else: raise error.PyAsn1Error('Boolean CER violation: %s' % byte) - return self._createComponent(asn1Spec, tagSet, value), substrate[1:] + return self._createComponent(asn1Spec, tagSet, value), tail tagMap = decoder.tagMap.copy() tagMap.update({ - univ.Boolean.tagSet: BooleanDecoder(), + univ.Boolean.tagSet: BooleanDecoder() }) typeMap = decoder.typeMap diff --git a/libs/pyasn1/codec/der/__init__.py b/libs/pyasn1/codec/der/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/der/__init__.py +++ b/libs/pyasn1/codec/der/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/der/decoder.py b/libs/pyasn1/codec/der/decoder.py index 0f5a24ca..604abec2 100644 --- a/libs/pyasn1/codec/der/decoder.py +++ b/libs/pyasn1/codec/der/decoder.py @@ -2,4 +2,8 @@ from pyasn1.type import univ from pyasn1.codec.cer import decoder -decode = decoder.Decoder(decoder.tagMap, decoder.typeMap) +tagMap = decoder.tagMap +typeMap = decoder.typeMap +Decoder = decoder.Decoder + +decode = Decoder(tagMap, typeMap) diff --git a/libs/pyasn1/compat/__init__.py b/libs/pyasn1/compat/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/compat/__init__.py +++ b/libs/pyasn1/compat/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/compat/octets.py b/libs/pyasn1/compat/octets.py index d0303eaa..f7f2a29b 100644 --- a/libs/pyasn1/compat/octets.py +++ b/libs/pyasn1/compat/octets.py @@ -8,6 +8,7 @@ if version_info[0] <= 2: octs2ints = lambda s: [ oct2int(x) for x in s ] str2octs = lambda x: x octs2str = lambda x: x + isOctetsType = lambda s: isinstance(s, str) else: ints2octs = bytes int2oct = lambda x: ints2octs((x,)) @@ -16,3 +17,4 @@ else: octs2ints = lambda s: [ x for x in s ] str2octs = lambda x: x.encode() octs2str = lambda x: x.decode() + isOctetsType = lambda s: isinstance(s, bytes) diff --git a/libs/pyasn1/debug.py b/libs/pyasn1/debug.py new file mode 100644 index 00000000..c27cb1d4 --- /dev/null +++ b/libs/pyasn1/debug.py @@ -0,0 +1,65 @@ +import sys +from pyasn1.compat.octets import octs2ints +from pyasn1 import error +from pyasn1 import __version__ + +flagNone = 0x0000 +flagEncoder = 0x0001 +flagDecoder = 0x0002 +flagAll = 0xffff + +flagMap = { + 'encoder': flagEncoder, + 'decoder': flagDecoder, + 'all': flagAll + } + +class Debug: + defaultPrinter = sys.stderr.write + def __init__(self, *flags): + self._flags = flagNone + self._printer = self.defaultPrinter + self('running pyasn1 version %s' % __version__) + for f in flags: + if f not in flagMap: + raise error.PyAsn1Error('bad debug flag %s' % (f,)) + self._flags = self._flags | flagMap[f] + self('debug category \'%s\' enabled' % f) + + def __str__(self): + return 'logger %s, flags %x' % (self._printer, self._flags) + + def __call__(self, msg): + self._printer('DBG: %s\n' % msg) + + def __and__(self, flag): + return self._flags & flag + + def __rand__(self, flag): + return flag & self._flags + +logger = 0 + +def setLogger(l): + global logger + logger = l + +def hexdump(octets): + return ' '.join( + [ '%s%.2X' % (n%16 == 0 and ('\n%.5d: ' % n) or '', x) + for n,x in zip(range(len(octets)), octs2ints(octets)) ] + ) + +class Scope: + def __init__(self): + self._list = [] + + def __str__(self): return '.'.join(self._list) + + def push(self, token): + self._list.append(token) + + def pop(self): + return self._list.pop() + +scope = Scope() diff --git a/libs/pyasn1/type/__init__.py b/libs/pyasn1/type/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/type/__init__.py +++ b/libs/pyasn1/type/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/type/base.py b/libs/pyasn1/type/base.py index db31671e..40873719 100644 --- a/libs/pyasn1/type/base.py +++ b/libs/pyasn1/type/base.py @@ -120,7 +120,12 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): def prettyIn(self, value): return value def prettyOut(self, value): return str(value) - def prettyPrint(self, scope=0): return self.prettyOut(self._value) + def prettyPrint(self, scope=0): + if self._value is noValue: + return '<no value>' + else: + return self.prettyOut(self._value) + # XXX Compatibility stub def prettyPrinter(self, scope=0): return self.prettyPrint(scope) diff --git a/libs/pyasn1/type/namedtype.py b/libs/pyasn1/type/namedtype.py index aa9c5678..48967a5f 100644 --- a/libs/pyasn1/type/namedtype.py +++ b/libs/pyasn1/type/namedtype.py @@ -60,12 +60,12 @@ class NamedTypes: tagMap = self.__namedTypes[idx].getType().getTagMap() for t in tagMap.getPosMap(): if t in self.__tagToPosIdx: - raise error.PyAsn1Error('Duplicate type %s' % t) + raise error.PyAsn1Error('Duplicate type %s' % (t,)) self.__tagToPosIdx[t] = idx try: return self.__tagToPosIdx[tagSet] except KeyError: - raise error.PyAsn1Error('Type %s not found' % tagSet) + raise error.PyAsn1Error('Type %s not found' % (tagSet,)) def getNameByPosition(self, idx): try: @@ -79,12 +79,12 @@ class NamedTypes: idx = idx - 1 n = self.__namedTypes[idx].getName() if n in self.__nameToPosIdx: - raise error.PyAsn1Error('Duplicate name %s' % n) + raise error.PyAsn1Error('Duplicate name %s' % (n,)) self.__nameToPosIdx[n] = idx try: return self.__nameToPosIdx[name] except KeyError: - raise error.PyAsn1Error('Name %s not found' % name) + raise error.PyAsn1Error('Name %s not found' % (name,)) def __buildAmbigiousTagMap(self): ambigiousTypes = () diff --git a/libs/pyasn1/type/namedval.py b/libs/pyasn1/type/namedval.py index 815e2d42..d0fea7cc 100644 --- a/libs/pyasn1/type/namedval.py +++ b/libs/pyasn1/type/namedval.py @@ -15,10 +15,10 @@ class NamedValues: name = namedValue val = automaticVal if name in self.nameToValIdx: - raise error.PyAsn1Error('Duplicate name %s' % name) + raise error.PyAsn1Error('Duplicate name %s' % (name,)) self.nameToValIdx[name] = val if val in self.valToNameIdx: - raise error.PyAsn1Error('Duplicate value %s' % name) + raise error.PyAsn1Error('Duplicate value %s=%s' % (name, val)) self.valToNameIdx[val] = name self.namedValues = self.namedValues + ((name, val),) automaticVal = automaticVal + 1 diff --git a/libs/pyasn1/type/tag.py b/libs/pyasn1/type/tag.py index 0cf67ebd..1144907f 100644 --- a/libs/pyasn1/type/tag.py +++ b/libs/pyasn1/type/tag.py @@ -18,7 +18,7 @@ class Tag: def __init__(self, tagClass, tagFormat, tagId): if tagId < 0: raise error.PyAsn1Error( - 'Negative tag ID (%s) not allowed' % tagId + 'Negative tag ID (%s) not allowed' % (tagId,) ) self.__tag = (tagClass, tagFormat, tagId) self.uniq = (tagClass, tagId) diff --git a/libs/pyasn1/type/tagmap.py b/libs/pyasn1/type/tagmap.py index 53e1791a..7cec3a10 100644 --- a/libs/pyasn1/type/tagmap.py +++ b/libs/pyasn1/type/tagmap.py @@ -28,7 +28,7 @@ class TagMap: def clone(self, parentType, tagMap, uniq=False): if self.__defType is not None and tagMap.getDef() is not None: - raise error.PyAsn1Error('Duplicate default value at %s' % self) + raise error.PyAsn1Error('Duplicate default value at %s' % (self,)) if tagMap.getDef() is not None: defType = tagMap.getDef() else: @@ -37,7 +37,7 @@ class TagMap: posMap = self.__posMap.copy() for k in tagMap.getPosMap(): if uniq and k in posMap: - raise error.PyAsn1Error('Duplicate positive key %s' % k) + raise error.PyAsn1Error('Duplicate positive key %s' % (k,)) posMap[k] = parentType negMap = self.__negMap.copy() diff --git a/libs/pyasn1/type/univ.py b/libs/pyasn1/type/univ.py index cb4f49b7..9cd16f8a 100644 --- a/libs/pyasn1/type/univ.py +++ b/libs/pyasn1/type/univ.py @@ -69,13 +69,18 @@ class Integer(base.AbstractSimpleAsn1Item): def prettyIn(self, value): if not isinstance(value, str): - return int(value) + try: + return int(value) + except: + raise error.PyAsn1Error( + 'Can\'t coerce %s into integer: %s' % (value, sys.exc_info()[1]) + ) r = self.__namedValues.getValue(value) if r is not None: return r try: return int(value) - except ValueError: + except: raise error.PyAsn1Error( 'Can\'t coerce %s into integer: %s' % (value, sys.exc_info()[1]) ) @@ -224,14 +229,14 @@ class BitString(base.AbstractSimpleAsn1Item): return tuple(r) else: raise error.PyAsn1Error( - 'Bad BIT STRING value notation %s' % value + 'Bad BIT STRING value notation %s' % (value,) ) else: for i in value.split(','): j = self.__namedValues.getValue(i) if j is None: raise error.PyAsn1Error( - 'Unknown bit identifier \'%s\'' % i + 'Unknown bit identifier \'%s\'' % (i,) ) if j >= len(r): r.extend([0]*(j-len(r)+1)) @@ -528,7 +533,7 @@ class Real(base.AbstractSimpleAsn1Item): ) if value[1] not in (2, 10): raise error.PyAsn1Error( - 'Prohibited base for Real value: %s' % value[1] + 'Prohibited base for Real value: %s' % (value[1],) ) if value[1] == 10: value = self.__normalizeBase10(value) @@ -648,7 +653,7 @@ class SetOf(base.AbstractConstructedAsn1Item): def _verifyComponent(self, idx, value): if self._componentType is not None and \ not self._componentType.isSuperTypeOf(value): - raise error.PyAsn1Error('Component type error %s' % value) + raise error.PyAsn1Error('Component type error %s' % (value,)) def getComponentByPosition(self, idx): return self._componentValues[idx] def setComponentByPosition(self, idx, value=None, verifyConstraints=True): @@ -924,9 +929,9 @@ class Choice(Set): return self._componentValues[self._currentIdx] >= other return NotImplemented if sys.version_info[0] <= 2: - def __nonzero__(self, other): return bool(self._componentValues) + def __nonzero__(self): return bool(self._componentValues) else: - def __bool__(self, other): return bool(self._componentValues) + def __bool__(self): return bool(self._componentValues) def __len__(self): return self._currentIdx is not None and 1 or 0 diff --git a/libs/pyutil/_version.py b/libs/pyutil/_version.py index 617d2205..376b2b9b 100644 --- a/libs/pyutil/_version.py +++ b/libs/pyutil/_version.py @@ -6,7 +6,7 @@ # pyutil.version_class for a description of what the different fields mean. __pkgname__ = "pyutil" -verstr = "1.9.3" +verstr = "1.9.7" try: from pyutil.version_class import Version as pyutil_Version __version__ = pyutil_Version(verstr) diff --git a/libs/pyutil/benchutil.py b/libs/pyutil/benchutil.py index 3e773a63..6c286346 100644 --- a/libs/pyutil/benchutil.py +++ b/libs/pyutil/benchutil.py @@ -1,4 +1,4 @@ -# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn +# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn # This file is part of pyutil; see README.rst for licensing terms. """ @@ -21,10 +21,10 @@ the second, e.g.: >>> rep_bench(fib, 25, UNITS_PER_SECOND=1000) best: 1.968e+00, 3th-best: 1.987e+00, mean: 2.118e+00, 3th-worst: 2.175e+00, worst: 2.503e+00 (of 10) -The output is reporting the number of milliseconds that executing the function -took, divided by N, from ten different invocations of fib(). It reports the -best, worst, M-th best, M-th worst, and mean, where "M" is the natural log of -the number of invocations (in this case 10). +The output is reporting the number of milliseconds that executing the +function took, divided by N, from ten different invocations of +fib(). It reports the best, worst, M-th best, M-th worst, and mean, +where "M" is 1/4 of the number of invocations (in this case 10). 2. Now run it with different values of N and look for patterns: @@ -74,10 +74,12 @@ and the main function is to make them be methods of the same object, e.g.: 4. Things to fix: - a. I used to have it hooked up to use the "hotshot" profiler on the code being - measured. I recently tried to change it to use the newer cProfile profiler - instead, but I don't understand the interface to cProfiler so it just gives an - exception if you pass profile=True. Please fix this and send me a patch. + a. I used to have it hooked up to use the "hotshot" profiler on the + code being measured. I recently tried to change it to use the newer + cProfile profiler instead, but I don't understand the interface to + cProfiler so it just gives an exception if you pass + profile=True. Please fix this and send me a patch. xxx change it to + statprof b. Wouldn't it be great if this script emitted results in a json format that was understood by a tool to make pretty interactive explorable graphs? The @@ -122,7 +124,7 @@ def mult(a, b): except TypeError: return to_decimal(a) * to_decimal(b) -def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): +def rep_bench(func, n, runtime=1.0, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ Will run the func up to MAXREPS times, but won't start a new run if MAXTIME (wall-clock time) has already elapsed (unless MAXTIME is None). @@ -130,33 +132,43 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p @param quiet Don't print anything--just return the results dict. """ assert isinstance(n, int), (n, type(n)) + global worstemptymeasure + emsta = clock() + do_nothing(2**32) + emstop = clock() + empty = emstop - emsta + if empty > worstemptymeasure: + worstemptymeasure = empty + if (worstemptymeasure*2) >= runtime: + raise BadMeasure("Apparently simply invoking an empty Python function can take as long as %0.10f seconds, and we were running iterations for only about %0.10f seconds. So the measurement of the runtime of the code under benchmark is not reliable. Please pass a higher number for the 'runtime' argument to bench_it().") + startwallclocktime = time.time() - tls = [] # elapsed time in seconds + tls = [] # (elapsed time per iter in seconds, iters) bmes = [] while ((len(tls) < MAXREPS) or (MAXREPS is None)) and ((MAXTIME is None) or ((time.time() - startwallclocktime) < MAXTIME)): if initfunc: initfunc(n) try: - tl = bench_it(func, n, profile=profile, profresults=profresults) + tl, iters = bench_it(func, n, runtime=runtime, profile=profile, profresults=profresults) except BadMeasure, bme: bmes.append(bme) else: - tls.append(tl) + tls.append((tl, iters)) if len(tls) == 0: raise Exception("Couldn't get any measurements within time limits or number-of-attempts limits. Maybe something is wrong with your clock? %s" % (bmes,)) - sumtls = reduce(operator.__add__, tls) + sumtls = sum([tl for (tl, iters) in tls]) mean = sumtls / len(tls) tls.sort() - worst = tls[-1] - best = tls[0] - _assert(best > worstemptymeasure*MARGINOFERROR, "%s(n=%s) took %0.10f seconds, but we cannot measure times much less than about %0.10f seconds. Try a more time-consuming variant (such as higher n)." % (func, n, best, worstemptymeasure*MARGINOFERROR,)) + worst = tls[-1][0] + best = tls[0][0] + m = len(tls)/4 if m > 0: - mthbest = tls[m-1] - mthworst = tls[-m] + mthbest = tls[m-1][0] + mthworst = tls[-m][0] else: - mthbest = tls[0] - mthworst = tls[-1] + mthbest = tls[0][0] + mthworst = tls[-1][0] # The +/-0 index is the best/worst, the +/-1 index is the 2nd-best/worst, # etc, so we use mp1 to name it. @@ -196,26 +208,22 @@ class BadMeasure(Exception): def do_nothing(n): pass -def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"): +def bench_it(func, n, runtime=1.0, profile=False, profresults="pyutil-benchutil.prof"): if profile: - st = clock() - cProfile.run('func(n)', profresults) - sto = clock() + raise NotImplementedException() else: + iters = 0 st = clock() - func(n) + deadline = st + runtime sto = clock() + while sto < deadline: + func(n) + iters += 1 + sto = clock() timeelapsed = sto - st - if timeelapsed <= 0: - raise BadMeasure(timeelapsed) - global worstemptymeasure - emsta = clock() - do_nothing(2**32) - emstop = clock() - empty = emstop - emsta - if empty > worstemptymeasure: - worstemptymeasure = empty - return timeelapsed + if (timeelapsed <= 0) or (iters == 0): + raise BadMeasure((timeelapsed, iters)) + return (timeelapsed / iters, iters) def bench(func, initfunc=None, TOPXP=21, MAXREPS=5, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", outputjson=False, jsonresultsfname="pyutil-benchutil-results.json", UNITS_PER_SECOND=1): BSIZES = [] diff --git a/libs/pyutil/benchutil.py~ b/libs/pyutil/benchutil.py~ index 3ec323eb..a33111e8 100644 --- a/libs/pyutil/benchutil.py~ +++ b/libs/pyutil/benchutil.py~ @@ -1,4 +1,4 @@ -# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn +# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn # This file is part of pyutil; see README.rst for licensing terms. """ @@ -104,6 +104,24 @@ def makeg(func): func() return blah +def to_decimal(x): + """ + See if D(x) returns something. If instead it raises TypeError, x must have been a float, so convert it to Decimal by way of string. (In Python >= 2.7, D(x) does this automatically. + """ + try: + return D(x) + except TypeError: + return D("%0.54f" % (x,)) + +def mult(a, b): + """ + If we get TypeError from * (possibly because one is float and the other is Decimal), then promote them both to Decimal. + """ + try: + return a * b + except TypeError: + return to_decimal(a) * to_decimal(b) + def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ Will run the func up to MAXREPS times, but won't start a new run if MAXTIME @@ -144,12 +162,12 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p # etc, so we use mp1 to name it. mp1 = m+1 res = { - 'worst': (worst*UNITS_PER_SECOND)/n, - 'best': (best*UNITS_PER_SECOND)/n, + 'worst': mult(worst, UNITS_PER_SECOND)/n, + 'best': mult(best, UNITS_PER_SECOND)/n, 'mp1': mp1, - 'mth-best': (mthbest*UNITS_PER_SECOND)/n, - 'mth-worst': (mthworst*UNITS_PER_SECOND)/n, - 'mean': (mean*UNITS_PER_SECOND)/n, + 'mth-best': mult(mthbest, UNITS_PER_SECOND)/n, + 'mth-worst': mult(mthworst, UNITS_PER_SECOND)/n, + 'mean': mult(mean, UNITS_PER_SECOND)/n, 'num': len(tls), } @@ -178,7 +196,10 @@ class BadMeasure(Exception): def do_nothing(n): pass -def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"): +def bench_it(func, n, runtime=0.1, profile=False, profresults="pyutil-benchutil.prof"): + """ + runtime is how many seconds to + """ if profile: st = clock() cProfile.run('func(n)', profresults) diff --git a/libs/pyutil/data/wordlist.txt b/libs/pyutil/data/wordlist.txt new file mode 100644 index 00000000..e1048b99 --- /dev/null +++ b/libs/pyutil/data/wordlist.txt @@ -0,0 +1,7248 @@ +fawn +yellow +four +prefix +payoff +scold +outwit +lore +lord +swivel +deli +pigment +foul +fur +disturb +prize +broiler +wooden +satchel +crotch +fritter +charter +tired +miller +bacon +second +tether +ruthless +thunder +fossil +succumb +cull +specialist +hero +avert +herb +splinter +here +herd +china +dogwood +cult +shriek +chink +pancreas +robin +neurologist +climber +diplomat +golden +gridiron +lengthen +summons +remnant +stern +unit +spoke +exhort +statesmanship +music +bedrock +passport +strike +teaspoon +relay +relax +hurt +meteorologist +glass +hurl +hole +hold +unpack +sweeten +blade +locker +locket +plunger +wand +wane +unjust +household +digit +malign +caution +want +rayon +hog +hoe +travel +copious +cutback +revisit +how +hot +hop +cheetah +diagram +possum +modest +antonym +pigtail +revolt +alias +decoy +wing +squint +wine +feedback +misdemeanor +kickoff +foodstuff +butcher +dreamer +fir +bowlder +fix +fib +fig +fin +undercut +enrich +slate +interrupt +sixteen +silver +scholar +thyme +seamstress +debut +arrow +debug +volcano +burial +whim +concord +knockout +garment +allah +spider +crocus +turnip +yiddish +fortnight +allay +whir +whip +diction +smirk +mason +semiconductor +re +adapt +outburst +knit +scruff +silicon +miaow +thumbtack +shopper +wasp +wash +instruct +rhododendron +tango +master +architect +bitter +listen +wisdom +swish +sulphur +crawl +trek +peril +outlay +coward +tree +shower +pneumonia +sheen +acclaim +entail +girder +runner +spectrum +headland +increment +quay +dozen +kidnap +gripe +hum +greenback +tipi +matriarch +stirrup +object +toil +microsecond +mouth +addict +letter +fluster +drought +thriller +expound +singer +upend +grove +professor +camp +detriment +nineteenth +scream +marvel +bomb +reactor +heckler +ulcer +caper +layout +menu +bust +cougar +bush +bliss +rich +mend +rice +plate +pocket +cushion +fetish +relish +jaguar +boarder +pretzel +patch +hasten +respond +fair +heirloom +radium +radius +result +fail +crouch +clef +best +irk +yogurt +ire +wage +extend +vestment +souvenir +extent +wheelbarrow +carbon +debt +roller +accident +trickster +veer +disdain +cup +logic +genus +rehash +gopher +canyon +bewilder +chrome +onomatopoeia +advert +grapefruit +stadium +jackass +counterattack +life +retrospect +spit +worker +wish +lift +toboggan +chile +child +chili +spin +wildcat +dissect +employ +calcium +delicatessen +locksmith +letdown +player +elicit +eighteen +violin +doorman +specter +hone +toaster +honk +rebellion +split +bid +european +typhoid +boiler +ownership +supper +tuna +tune +furlough +noblewoman +unhook +abound +bellow +beset +plight +brandish +previous +ham +hag +hay +prison +falter +east +hat +quirk +birth +shadow +gangplank +remind +pavement +battlefield +attorney +right +old +creek +crowd +creed +crown +glove +billboard +creep +chorus +okra +bottom +circumvent +inhuman +fox +foe +fog +binder +yoke +slither +recollect +despair +rebut +eightieth +sob +sod +overshadow +honeymoon +overgrow +sop +sow +wrap +fabric +panorama +support +tame +avail +width +hothead +call +overhand +overhang +telegraph +offer +thesaurus +beech +squalid +safeguard +otter +duel +misinform +paprika +vanguard +pest +duet +proud +tournament +proven +exist +quintuplet +dealer +leer +floor +glacier +actor +flood +role +entomologist +sunset +smell +leek +intend +glutton +ointment +asterisk +taurus +intent +cleaver +entrust +windscreen +puss +lowdown +time +push +gown +chain +viaduct +skate +chair +midst +millisecond +ballet +uneven +vex +crater +oversight +jerk +ameba +embark +flora +mourn +knapsack +southpaw +exact +epic +judaism +tear +teas +teat +crustacean +subway +team +skewer +prevent +meadow +gremlin +attic +sigh +milligram +heavyweight +crescent +playpen +crackpot +melt +current +boost +abscond +gnaw +splice +address +brilliant +endow +queue +influx +love +radish +prefer +piranha +fake +instal +forefront +sky +homesick +turret +wicker +wicket +scope +prosecutor +wicked +afford +refrain +visual +appendix +behalf +mascot +lumberjack +pretend +descriptor +dispossess +stole +winter +savor +sputter +meddler +slush +spot +textual +date +suck +dove +pulley +stress +conscious +bluster +wheelchair +quadrant +mango +so +skirmish +truce +drunken +archeologist +footstep +yearn +jig +disconnect +thumb +accordion +nearsighted +councillor +hubbub +suspicion +thump +apron +civilian +insomnia +nation +amulet +twilight +ketchup +handiwork +revert +fisherman +quarter +quartet +receipt +fireproof +breakthrough +sponsor +troll +naked +canvas +onrush +trauma +formula +dumbfound +million +envelop +vicious +disrespect +mime +plea +byte +workmanship +punk +wrong +ostrich +punt +footwear +neglect +gunshot +potter +one +reopen +chide +conifer +vote +paleontologist +languish +boulevard +wrath +convent +bite +extortion +shiver +draft +cite +starfish +shawl +artifact +snatch +antic +boyfriend +iceberg +rival +stammer +counselor +janitor +prospect +sac +greyhound +argument +alley +sad +say +borough +saw +handicraft +tulip +general +knead +zoo +note +take +destroy +printer +buffer +squalor +compress +buffet +crochet +knee +byway +lawn +enamel +blockhead +sale +cocoanut +wind +axe +salt +cobra +homespun +lotus +friction +bright +slot +slow +slop +unkind +gourd +transact +cloak +debunk +slog +hockey +slob +robe +clank +dissimilar +psychiatrist +clang +outlet +prime +artist +saliva +borrow +soloist +carrion +handcuff +primp +landlord +tortilla +where +xmas +vision +gout +gangster +cheesecloth +diver +bugler +mutton +plummet +bootleg +teacup +bureau +mope +vender +jumper +spars +screen +dome +supermarket +adept +jovial +spare +spark +quack +oust +fit +madcap +mane +flipper +backpack +twin +boar +supervisor +extinct +twig +boat +companionship +stretch +west +breath +reflex +gist +thousand +photon +cloudburst +turtleneck +former +jute +scarlet +straighten +spotlight +girth +brow +canon +dubious +monk +blab +fame +spunk +breakdown +hideaway +deft +barber +disown +booster +driftwood +veal +pewter +dimension +scholarship +summer +manifold +poach +disconcert +slime +rest +invalid +alarmist +mandolin +instrument +overthrow +stopwatch +haystack +joyful +sportsmanship +rejoin +dart +dark +brazier +snarl +traffic +cranium +vacuum +world +snare +dare +clan +clam +stranger +shutter +glamor +clay +claw +inter +kennel +clap +auditorium +obstruct +grub +potion +lobster +racial +endeavor +tote +tube +moslem +tuba +nook +exit +refer +zest +ration +leadership +standpoint +stone +ace +slender +meal +tumor +neighbor +act +mean +invert +braggart +homeless +wade +hypnotist +racquet +hew +burglar +her +gleam +glean +mindless +harpsichord +italic +hem +hen +defrost +epilog +pull +regimen +darken +wafer +rage +hooray +tripe +ruse +flirt +reprimand +whiz +torso +pulp +rust +ohm +gong +ad +fright +certain +epaulet +catchup +hoodlum +ay +ax +tranquil +jargon +slobber +cream +yoga +collector +abolish +tight +backgammon +congress +annex +slant +midget +brotherhood +slang +rostrum +neuter +thorn +groom +mask +kilogram +mash +mimic +mast +mass +ringworm +waiver +retch +gingham +influenza +consider +neigh +upkeep +taxicab +tinsel +to +tail +smile +norm +debit +baton +candid +salesperson +cobalt +strand +laud +pedant +sand +adjust +small +mammal +peon +ninetieth +plaid +past +burnish +gossip +canvass +healer +hick +offbeat +clock +section +succinct +method +contrast +full +hash +lobbyist +saleswoman +dramatist +backlash +brutal +prior +hamster +skyrocket +social +action +welder +raze +depart +vie +sherbet +regiment +captor +coercion +entrap +select +casket +enliven +petroleum +maltreat +pearl +sitter +morn +ballad +more +teen +teem +door +tester +signpost +nomad +doom +cunning +fatal +malt +chisel +patriarch +knocker +midstream +mall +learn +grope +male +stewardess +prompt +taunt +gallop +scab +accept +autumn +gallon +scar +rustler +condemn +huge +speedboat +fruition +cling +clink +plant +anoint +blotter +variant +unsound +plane +waver +flutter +pucker +wrench +trellis +patio +pant +instep +trade +paper +pang +brim +mislay +hearsay +buttercup +epoch +coarsen +bypass +motley +sucker +gadget +consign +imperil +skipper +harrow +nugget +fount +found +lantern +status +eyelash +clockwork +scribe +penicillin +lipstick +research +highway +bungler +belief +porcelain +bedlam +cockpit +loafer +suntan +acorn +riser +reproach +prefab +drivel +sicken +bumper +testament +clump +major +purport +limerick +number +feeder +slipper +footprint +florist +glitter +guess +guest +jet +swipe +vocalist +saint +gnash +relationship +tightwad +typhoon +mural +consult +grace +frock +getaway +vocal +video +defect +waft +pedestrian +graffito +caress +blond +gasket +sell +ballerina +ragamuffin +tarnish +spaghetti +self +trowel +poplar +brace +bobbin +kneecap +hypochondriac +blackboard +nasal +twine +raucous +virus +plan +wive +foyer +oyster +unequal +arson +covet +cover +barren +barrel +bulletin +chowder +golf +cruiser +affix +session +freight +impact +condor +writer +peculiar +condom +factor +downpour +dandelion +streamer +resent +actress +compass +banner +tumult +sojourn +caramel +enema +weaver +river +outlaw +prospectus +manger +set +creator +overwhelm +jade +sex +see +sea +contour +analog +project +urchin +fission +crossword +pickup +crosswalk +kneel +candor +mildew +hardship +disallow +incident +dividend +pagan +scatterbrain +lass +last +thou +opal +feminist +amoeba +lash +whole +load +loaf +electrician +pendulum +bell +loam +loan +hollow +scallop +church +psychoanalyst +underlay +napalm +airfield +devil +filth +imbed +proprietor +veneer +firm +sweetheart +champion +fire +infect +upstart +fund +deport +hostess +straight +budget +error +outskirt +real +pound +moth +vow +chasm +vanish +chase +starlight +seek +shorten +wasteland +specimen +commune +snail +teeter +cigar +epithet +alert +opinion +stack +recent +expend +clime +person +sixtieth +crayfish +telegram +aroma +belabor +amp +demerit +sandal +goblet +chest +eager +horseradish +homeland +wrongdoer +input +limp +cordon +format +bureaucrat +quest +cataclysm +blackjack +falcon +abduct +flannel +spine +consensus +crescendo +spring +beckon +palm +pall +sight +curious +sprint +battalion +pale +gruel +benefit +religion +be +odor +agreement +carol +by +scepter +coexist +hatchet +sacrament +ambush +biennial +repair +contributor +next +span +sock +submit +custom +spay +suit +spar +spat +blueprint +perplex +poster +lint +slump +pastor +overbear +link +atom +line +up +slander +foist +hornet +insignia +genial +aerial +nationalist +haunt +char +chap +chat +parsec +breather +phantom +paradox +tuft +uranium +scrape +parakeet +swirl +freighter +tart +tedium +scapegoat +trouser +scrub +gardenia +hackney +lane +land +fighter +algorithm +scotch +age +feud +summit +walker +fresh +crowbar +rescuer +hello +essay +code +partial +serviceman +scratch +broaden +totem +soften +leggin +renown +prim +flashback +young +send +moor +tremor +garret +armament +garden +quadruplet +llama +precinct +wipe +magic +harbor +eve +anxious +race +rack +mishap +crook +croon +odd +ode +victor +index +yock +sauerkraut +apparatus +indian +proffer +bird +inspect +leg +punch +acquit +let +fifteen +vinegar +great +casino +screech +scatter +survey +insulin +grandchild +buss +popcorn +mussel +maker +grower +sire +disobey +causal +zip +archbishop +theme +aeon +eleven +doubt +yardstick +midday +pencil +babe +shipwreck +patrol +rubbish +central +hoard +pout +pour +thin +drill +coffin +cherubim +bent +pawn +process +lock +slim +high +slit +bend +slip +pelvis +martyr +trumpet +weaken +rhubarb +delay +blackhead +luster +stow +halter +singular +await +wristwatch +notebook +tier +marrow +hawk +autograph +tomato +counter +robot +element +writ +allot +allow +alloy +thigh +mute +insight +spatula +comma +mutt +warren +perfect +decay +shudder +garnet +derelict +prosper +python +belch +bat +launder +dock +snake +kiss +bar +cage +wrangler +truth +scorch +subset +bump +static +thirteen +mete +jagged +disco +tenth +wander +matrix +bag +fatherland +venom +czar +oblong +lob +shut +perish +tempo +graze +tempt +shun +embarrass +minstrel +chilli +mainland +spill +length +stickler +scare +scarf +manuscript +scene +cobweb +owner +scent +prank +lop +opossum +sergeant +spaceship +painful +stomach +chagrin +vouch +rotunda +haven +steel +wet +bother +aggressor +psalm +disband +unman +steep +torrent +misunderstand +beggar +viewer +partnership +correspond +tonight +receptionist +fourteenth +mischief +depict +soak +bacterium +bassoon +hammer +adjunct +lilt +soap +soar +calculus +manor +raindrop +cipher +vise +segment +fervent +instil +locust +enlist +soprano +fiasco +brew +fact +bring +brine +bedroom +rough +asylum +trivial +brink +redirect +disillusion +planter +jay +jaw +jar +jam +tape +bourbon +flinch +hope +jackpot +move +familiar +scorn +sinus +wring +antagonist +smash +shaver +summon +stuff +rein +withstand +pronoun +packer +frame +packet +bellhop +airmail +dungeon +wire +mien +partisan +unravel +piston +pistol +email +browbeat +fetich +physicist +courtyard +lawsuit +tantrum +drum +quitter +ramp +drug +doorway +puff +roughen +medallion +revamp +migrant +distil +javelin +indict +chromium +lectern +mailman +gondola +quaver +blatant +feather +ballast +sheepish +crisscross +federalist +mannequin +altruism +banish +laser +runway +bathtub +maul +groin +ripe +lush +site +lust +mockingbird +tenor +passbook +ransom +tattoo +inquest +terrorist +buffoon +outbreak +android +balm +ball +balk +dusk +fiesta +bale +bald +dust +broccoli +mosaic +audit +off +shotgun +polyp +command +diphtheria +audio +maggot +compel +glut +glue +rambler +web +generous +clergyman +wee +wed +arrest +crack +scoundrel +government +chancellor +crux +haul +cedar +desk +password +recurs +placenta +crisp +onion +resin +alkali +stagger +imprison +nymph +sprain +overcast +foray +habitat +thief +daylight +flush +wisecrack +ballot +transport +henchman +disbelief +hoarder +avoid +disk +doer +passion +saucepan +stairway +putt +drift +ornithologist +stage +iris +sister +adverb +peal +ingest +union +artefact +parsley +assess +lung +mere +muck +commission +caviar +watchman +stamina +much +function +funnel +cosmopolitan +frisk +shyness +grate +rectum +count +congresswoman +smooth +monument +problem +baptism +cordial +kowtow +sirloin +retina +inn +replica +ink +anesthesia +furl +sexual +saviour +behold +reckless +chum +monday +repeal +veil +vein +ghost +eon +rule +dynamo +torrid +pension +tryout +abhor +buoy +inning +dote +rapid +mansion +defraud +voter +spew +bludgeon +bike +daze +regal +chill +regalia +whack +ale +compassion +blanket +distort +mania +chauvinist +chapel +whisk +daydream +pinch +scalar +handout +roadblock +unblock +math +triumph +chew +paperback +phoney +speck +heliport +sabbath +horn +chef +aristocrat +panda +stardom +lizard +walkout +toll +crunch +dustpan +pursuit +paraffin +sorceress +hairdo +daughter +envoy +adopt +tankard +smoke +loincloth +bunker +anarchist +envious +sanatorium +infield +spigot +thrust +hindsight +total +bra +plot +plow +plop +sweater +gloss +ploy +insult +plod +knoll +beeswax +solicit +award +yard +tariff +overrun +word +err +crest +work +grovel +tinder +era +elbow +spendthrift +quiver +serpent +flunk +impair +liter +chameleon +sever +moron +disappoint +beach +pizza +fever +lad +ladder +lag +lab +lay +law +arch +cosmonaut +retort +greet +greek +green +south +worst +order +greed +salon +gumption +devote +muffin +misconduct +mayor +sheaf +avocado +valor +carton +shear +then +fragment +safe +break +band +bang +coffer +overprint +tzar +bank +bread +crock +gallows +lisp +iguana +schemer +transient +prawn +sled +flock +slew +hostel +burlap +network +diesel +fellowship +amethyst +marigold +barrier +veto +standard +stencil +lollipop +morass +drench +ticket +maniac +raisin +flawless +renew +sprig +regress +vanquish +kin +render +system +hamstring +chopper +disembark +comic +overs +neck +upshot +tourniquet +kiwi +emblem +luncheon +cereal +rebuff +minibus +guild +target +tavern +hike +medley +iota +guilt +iron +minus +pessimist +lull +ponder +strength +realm +widen +silversmith +latter +hamper +transmit +curfew +maiden +boxcar +sue +negro +phase +proverb +grave +deacon +swamp +bracket +aunt +rickshaw +oppress +mitten +crust +boyhood +nephew +toast +geyser +layman +geologist +predecessor +do +ardor +ecologist +roundabout +slack +rebirth +runt +rune +rung +crucifixion +steak +steal +steam +ghoul +reread +misdirect +christian +goulash +pastel +gentleman +cellist +contraband +drawl +accord +unfold +kitchen +cop +cot +cow +brat +excrement +ill +cob +brag +cod +cog +bran +coo +con +emporium +eyesight +tone +spear +royal +trunk +nonconformist +infirm +speak +charisma +scarecrow +warmth +leech +baud +hacksaw +millionth +hoist +spellbind +gracious +physician +inhibit +gnu +launcher +air +aim +ail +abrupt +thrash +aid +stink +have +sticker +sting +throat +brake +cone +hebrew +uplift +stint +descent +perform +descend +decibel +wheel +raid +fuss +nil +swell +hang +evil +hand +fuse +nip +nit +scenario +drip +ragged +client +mamma +kinship +indigo +photo +victim +extol +thyroid +exalt +shout +cognac +board +zillion +righteous +plasma +intercom +fusion +boxer +cape +retreat +cooler +night +flatter +born +rile +flatten +bore +orchid +cede +humor +peek +peel +pose +confer +peer +peep +chafe +foreskin +chaff +diner +coral +visa +banker +horizon +cherish +gingerbread +octopus +croak +faint +dilemma +tetanus +float +profession +bound +curios +sedan +loin +beet +piggyback +wag +bookend +wad +frill +sovereign +fight +gybe +way +wax +burro +war +fizz +peninsula +holdup +boon +true +reset +absent +nursemaid +smidgeon +maximum +crystal +veterinarian +emir +emit +aorta +flat +abstract +molt +flaw +postscript +subsist +prayer +cacao +face +mold +mole +stake +shrine +test +upholster +unwilling +frolic +shrink +heyday +chairperson +hemophilia +faze +affidavit +loyal +longshoreman +igloo +concept +matron +consul +fulcrum +datum +horseback +supplement +toothpick +varnish +grape +zone +mallet +flask +graph +hump +flash +manicurist +glad +rhythm +tusk +terror +idealist +southwest +brown +congest +kitten +blast +brows +ophthalmologist +gun +gum +gut +guy +diarrhoea +upper +brave +regret +bravo +thinker +cost +helpless +tempest +cargo +appear +economist +menthol +medal +havoc +uniform +tarantula +appeal +caveat +genes +gawk +jester +disclaim +goldfish +teacher +buck +merriment +fogey +precursor +plotter +eke +disavow +trial +convertor +pillow +bolt +extra +paragon +keeper +marker +firearm +market +streetcar +prove +subvert +live +matador +club +cluck +clue +logarithm +prepay +graphic +slogan +car +cap +caw +cat +meow +can +cab +heart +hears +chip +sake +bridesmaid +abort +chin +chic +serum +bankrupt +freezer +write +lobe +storeroom +criterion +entrench +afternoon +product +dive +southern +bawl +motorway +pave +drastic +flourish +crepe +grandson +explicit +offend +barnyard +forfeit +haircut +ledger +brain +nitrogen +cold +braid +ethic +willow +theorem +window +artisan +factual +tiara +gizzard +nought +halt +fling +nod +rake +overcrowd +dishearten +hale +half +recap +courtship +taillight +provision +discuss +halo +wont +concerto +servant +drop +domain +supplant +year +operand +wavelength +happen +album +accomplish +space +thirst +rational +thong +carp +cart +virtuoso +quart +rebel +marina +prospector +card +care +fungus +tomahawk +british +honest +nonprofit +profess +blind +madam +blink +rink +rind +ring +drove +tomorrow +size +sheep +sheer +sheet +silent +bookmark +breed +callous +traction +checker +tragic +heartburn +friend +pomp +courier +that +peck +scalpel +rugged +recruit +magnesium +optimist +extinguish +angel +slay +slat +premier +slap +racetrack +slam +anger +breakfast +recover +slab +upbeat +veteran +shore +snout +siesta +begin +prick +halibut +price +foothold +dream +tooth +aerosol +washcloth +fifth +ground +gnat +snack +ratio +stair +proportion +jolt +stain +juror +shrill +pumpkin +cannon +loath +stroll +leather +thermal +husband +druggist +concert +burst +spore +whitewash +unfit +staunch +sport +incisor +concern +crawfish +glaze +complexion +import +clench +pluck +blame +broil +impromptu +whisker +guffaw +pertain +priestess +temper +aura +comet +evict +adroit +dispatch +exploit +semicolon +lioness +harmless +rebuild +toss +sisterhood +textbook +bloodhound +crumb +these +trick +scum +cherub +fool +marksman +zenith +mucus +soil +agnostic +laggard +bias +eras +bestial +beaver +waterway +petunia +helium +develop +media +pester +poetic +document +sweeper +finish +thunderbolt +foal +foam +cymbal +fruit +volley +trawler +smelt +quartz +theater +framework +patchwork +demean +taxi +livestock +fester +battleship +typhus +neon +touch +speed +death +refurbish +treatment +baloney +momentum +lade +ream +hover +frown +spectacular +larva +read +ruler +swig +leapfrog +earnest +detract +stunt +execution +reap +hovel +rear +postcard +incest +roll +engross +oblivion +output +downward +falsehood +laugh +verbal +landslid +squirm +garland +putter +cleanser +deficit +squirt +wheat +sadden +throb +sixth +tuition +strainer +bazaar +throw +comparison +placard +hiatus +chop +fell +wolf +parson +chow +ruff +assassin +processor +heater +outlook +earring +watchdog +your +restless +stare +grater +log +area +start +stealth +low +lot +wigwam +groan +pitcher +rump +posterior +recoil +omelet +lymph +thaw +corduroy +wholes +hire +fraud +default +bucket +draftsman +cornmeal +gibberish +charlatan +scanner +pickax +sheath +mover +antenna +housework +valid +colt +you +houseboat +poor +polar +poop +peat +pear +peas +overreact +podium +peak +pool +fiscal +assert +moonlight +repay +forswear +mourner +skeleton +breadth +groundwork +angler +month +unrest +bequeath +carpet +corps +gymnast +foster +spearhead +fervor +fountain +washroom +horror +verb +minded +heaven +ceaseless +saboteur +tendril +blackout +smock +homonym +resound +exodus +casual +bomber +protractor +lurch +milk +turmoil +vet +excess +strong +arena +divisor +noticeboard +outgrowth +vegetarian +whine +soldier +amount +base +trainer +put +haemoglobin +seventeenth +taker +helper +pup +titbit +assessor +chestnut +yuck +suction +grill +nine +parasol +transcend +pusher +boycott +archipelago +tract +phrase +magenta +frostbit +sheikh +warhead +spreadsheet +lyre +snippet +reject +gash +circuit +rude +sneak +denial +gasp +reelect +undergo +figment +overnight +ego +dread +egg +lynch +earthworm +help +reservoir +slouch +auburn +reclaim +preempt +soot +helm +hell +clarinet +limelight +prowler +lateral +heron +astonish +forbear +dim +food +musket +terrain +vomit +foot +stopper +holiday +payer +twelfth +bless +radial +trailer +pamper +thirteenth +talisman +event +magnet +vertigo +wedlock +teak +publish +eardrum +sustain +shrivel +outrun +ass +pun +drink +bass +dirt +dung +dune +cornstarch +reason +heroin +ask +ash +turnout +bask +bash +pus +launch +curtsey +round +bridegroom +caption +liaison +heartbeat +blush +assign +arsenal +demagog +elder +effortless +mist +miss +blossom +minion +station +expand +fingernail +scheme +banana +merciless +slosh +behind +bowel +trapper +boredom +sign +leotard +tunic +lament +ouster +hyaena +bride +bobsled +currant +toward +weapon +phobia +chipmunk +yowl +null +sensual +lid +lie +koala +cave +lip +useless +honeycomb +popular +quota +plunder +mace +watt +smoulder +clear +cleat +succor +clean +skein +blend +humid +phenomenon +cowgirl +flicker +sheik +crayon +copyright +paranoid +scooter +less +ikon +ampersand +waterproof +custodian +outlast +strut +strum +basement +chimney +monogram +fluff +chasten +geld +courteous +cramp +backtrack +grey +close +despatch +bandwagon +haddock +aqueduct +wow +grocer +won +woodchuck +wok +woe +stalk +bettor +wreath +philanthropist +spray +distinguish +zipper +garrison +delimit +eggplant +buzz +vault +protector +mausoleum +onward +oversleep +liken +proton +header +badminton +vessel +catwalk +stamp +damp +nape +damn +threaten +dame +alto +liven +exempt +deter +liver +hobnob +furrow +pact +loom +utmost +look +socialist +governor +rope +pace +while +smart +fleet +loot +loop +pack +petal +hoax +grant +belong +makeshift +discredit +grand +conflict +sham +hallway +optic +dime +bonus +banter +overweight +user +boa +grind +auditor +five +ambassador +chore +abstain +bearer +morsel +tick +botch +pier +carat +march +albino +game +jibe +banjo +optician +signal +manifest +eel +sleigh +sketch +creation +undress +yolk +urgent +impoverish +mustang +clothespin +fundamentalist +gild +simmer +slash +slapstick +run +rum +rub +booklet +benefactor +rug +stem +step +stew +taboo +subtract +rut +discus +shine +faith +pigpen +letup +portico +reappear +aye +congressman +block +foreswear +misbehavior +dude +within +gilt +pentagon +connector +syllabus +palomino +harem +frost +reed +womankind +reef +reek +reel +dull +skulk +swagger +chiropractor +ringlet +foresight +similar +psychopath +kidney +straitjacket +nab +sullen +nag +objection +obelisk +nap +department +nay +draw +resign +drag +tundra +drab +formal +horseplay +outing +orbit +depth +bribe +pinion +underbrush +cheerful +go +emboss +dunk +ammonia +compact +aquarium +baron +aria +stave +ameer +shack +geranium +warpath +epitaph +velour +schooner +virtuous +fond +wave +trough +cellular +tenet +nausea +stiff +asparagus +gender +button +hive +verdict +cloister +pilfer +picket +blitz +jump +booth +ardent +languor +cartel +click +poke +wallet +colonel +valet +cell +rotten +experiment +stanch +brooch +fifteenth +quell +weirdo +convert +chant +gent +repel +behead +ricksha +wig +daybreak +danger +win +clout +wit +ligament +infest +gimmick +wiz +cloud +metaphor +snoop +copperhead +crag +crab +cram +expressway +compatriot +mismatch +starter +salad +consort +ride +donut +archer +meet +control +wharf +halloween +glade +skirt +bandana +filament +circular +fare +farm +thunderstorm +canker +foment +corral +scoop +encyclopedia +scoot +agenda +american +cadet +sperm +gunman +hock +brood +broom +brook +walnut +youngster +skater +frond +auto +dike +relentless +snorkel +placid +stout +hands +front +refuel +muff +perfectionist +mode +upward +commonwealth +unwind +chunk +mollusc +seesaw +apartheid +mollusk +special +gallant +armor +confess +fathom +remiss +activist +playground +wick +obsess +umbrella +hopscotch +watermark +jilt +undo +advisor +sneer +princess +shrew +timer +keep +counterpart +elector +keel +mad +blogger +seventeen +bitch +drinker +equinox +dump +wrapper +chintz +attach +attack +jellyfish +final +beard +introvert +punish +feint +noun +plough +piecework +waist +photograph +spurn +cartoonist +beg +bed +bee +discolor +swindler +firework +spurt +bet +are +exhibit +fume +tabu +torment +sundown +portrait +need +border +rotor +bastard +sprinkler +gunner +jaunt +tactic +truck +detector +visor +brand +african +camper +rigor +awe +plumber +eject +spleen +urn +upset +snapshot +businessman +constrain +skunk +affair +indoor +crate +molest +cavort +sheriff +fiberglass +winner +wreak +rash +earner +rasp +gradual +fuel +sulfur +joint +fallout +buyer +endless +gray +tobacco +gust +ordain +topaz +nutmeg +she +contain +recoup +grab +conduct +widow +hardwood +shake +orphan +portend +computer +driveway +equip +portent +unearth +southward +tend +state +lug +tens +antler +tent +bleed +castoff +blinker +keg +bemoan +key +overhaul +thank +sniff +career +admit +spatter +plankton +jersey +christen +tuesday +poem +sari +tread +shaikh +yap +cent +quiz +yam +treat +yak +whisper +poet +fibber +spaniel +nuptial +brunch +debtor +novel +ripen +pandemonium +harden +neuron +hearten +steer +generic +balloon +speaker +northwest +blight +fireman +flesh +absorb +powwow +inbreed +spree +magician +rift +weld +surfboard +lunch +glower +well +drone +welt +underdog +discord +mistaken +dose +distant +laurel +skill +cinder +jackal +dais +ovum +snicker +stratum +possess +warrant +homework +canter +rafter +fate +burden +loss +clown +tablespoons +lose +divest +satirist +rote +page +backlog +shed +glare +twitter +hush +redress +home +peter +competitor +pinpoint +overlay +broad +overlap +hinder +individualist +journal +offset +instinct +smidgen +refuge +freedom +cleans +nightclub +rodeo +dominion +wallop +buzzard +cocoa +pointless +gerbil +snowplow +mastermind +museum +poinsettia +drumstick +mohair +jinx +backhand +cricket +north +gait +admonish +neutral +ho +technician +overflow +ear +eat +he +leper +limit +cello +display +wringer +twist +entreat +contest +meteor +finch +chemist +fodder +star +stay +stag +foil +stab +phosphorus +appoint +sphinx +shunt +broncho +atheist +atheism +portion +pardon +mackerel +demand +unfurl +protest +asian +captain +gamut +swab +swan +sinew +swat +swap +anticlimax +sway +loon +appal +void +vase +smack +govern +affect +hitch +vast +pilgrim +naturalist +vector +washout +whirlwind +quilt +crave +yack +cactus +quill +pander +wreck +orchestra +bikini +spokesman +haze +new +net +maverick +seventh +mew +onus +cardboard +interpret +taper +credit +harass +jamb +permit +prolog +menial +hunch +campaign +bayonet +moral +handlebar +overhear +ore +overheat +calk +overhead +calm +intrust +recommend +type +tell +calf +demon +wart +warp +warn +dogma +warm +pecan +adult +qualm +flotilla +ward +blindfold +confound +rook +room +candlestick +worth +bungalow +headway +root +defer +vodka +give +climax +assent +honey +surveyor +quail +freshen +polio +rib +stockyard +answer +abdomen +plank +coup +fracas +passageway +waterfront +lesbian +guerilla +attempt +third +maintain +capitalist +fetus +deck +keyboard +windshield +furlong +harmonica +crew +better +persist +pass +workout +microfilm +caterpillar +grammar +meat +mistrust +roast +side +bone +luck +caustic +aids +dawn +extract +jell +contend +velvet +gradient +open +crucial +content +reader +linear +whiff +bestow +mistress +needlework +steward +fleck +loud +skinflint +playwright +grade +hoop +hoot +buttock +hook +ditch +hoof +hood +hydrant +acquaint +spinach +historian +enthral +woodwind +brainwash +dwell +inferno +twister +gym +somewhat +gambler +symptom +preacher +affront +keyword +matter +loiter +mink +seep +quench +modern +mind +mine +ginger +seed +seem +churn +mint +unfasten +alibi +desist +chess +sleeper +quarterback +phonograph +chatterbox +regular +condominium +blacklist +don +alarm +impostor +dog +doe +solemn +digress +constrict +consumer +dot +hunger +visitor +probe +syntax +chord +sundial +northeast +explain +jailer +sugar +folder +inventor +edict +patter +smut +stop +coast +pincushion +watermelon +smug +earl +earn +peacock +bay +reload +bad +troop +cower +ban +stinger +linguist +enchant +attest +subject +snuff +scrap +sail +causeway +scram +baboon +warrior +triplet +vitamin +lotion +orthodontist +beautician +cousin +motto +sprinter +pate +typist +height +gusher +aftermath +arsenic +ether +accustom +tint +recur +three +erect +ting +chrysanthemum +trigger +interest +basic +basil +basin +idol +chug +mushroom +suppress +dismiss +deepen +encyclopaedia +unpopular +tank +affirm +tang +near +moratorium +neat +motorist +anchor +spawn +seven +cane +diaphragm +it +shame +jest +in +id +disappear +if +abscess +growl +sap +parish +make +kit +delight +squat +garlic +warden +unicorn +jealous +overt +bequest +kid +butter +romp +smoker +inherit +bedspread +diphthong +left +protocol +just +yen +unfair +psych +human +yes +cretin +yew +legion +character +wretch +save +opt +discreet +background +shoulder +nude +manual +pillar +dean +squander +deal +deaf +maxim +dead +revel +intern +dear +strife +sprawl +pail +collect +normal +councilor +flounder +bullfight +tartar +bold +statistician +burn +blackmail +cottontail +sift +protagonist +burp +burr +tartan +super +innuendo +crucifix +craftsman +commit +marshal +unsay +paunch +chimp +down +lieu +chime +initial +lampoon +editor +fraction +unseat +fork +undergrowth +form +fore +ford +diaper +overburden +analyst +fort +pavilion +whiskey +boomerang +cosmos +propaganda +shin +disciplinarian +classic +covert +sidestep +drive +gland +scrawl +fatherhood +ship +graft +vista +marxist +smidgin +excel +handed +venison +congeal +marxism +sling +faction +handicap +slink +felt +diet +parenthood +journey +reign +stoke +weekend +derail +billion +potato +jacket +gorilla +almanac +teeth +meander +befriend +proletariat +woodwork +skip +skit +invent +adjourn +mild +mile +skim +skin +mill +primer +proletarian +skid +surplus +seasick +misread +depend +swoon +father +countdown +deject +swoop +regatta +unburden +string +yeast +pathologist +merit +join +jettison +stiffen +hoorah +din +stapler +nectar +die +dig +democrat +noiseless +item +dip +blur +shave +thresher +villa +worm +slake +sunup +talker +fillet +suspect +drunkard +shoo +dwarf +dweller +wail +guardian +clerk +makeup +stallion +waif +detest +tangent +deceit +rue +wait +box +boy +cuckoo +shift +bow +dither +boo +raccoon +cyst +bob +nylon +bog +elect +plumb +kayak +surmount +transplant +saki +wealth +perk +visit +vineyard +somersault +sharpen +yoghurt +aspirin +labyrinth +curriculum +downtown +tandem +rigid +savior +effort +gnome +demolish +pageant +moccasin +melodrama +flu +soul +impel +soup +sour +claim +plaza +reflector +predict +agent +drawer +council +craze +pink +purr +arbor +tilt +clever +parch +pine +till +sunday +sword +tile +pathway +pint +map +mar +mat +may +gelatin +membership +mankind +tablecloth +grow +man +relinquish +aimless +hemlock +omen +tale +switch +jail +deposit +talc +unleash +basket +longhand +talk +shield +rabbi +moralist +lyric +pitch +solder +pointer +group +monitor +bedbug +maid +drummer +maim +mail +main +tonic +killer +shatter +minuet +safari +teller +rattler +outweigh +feverish +peasant +careless +rock +hijack +eyelid +latin +bookshop +poker +gavel +unlock +manifesto +girl +stitch +monolog +priest +dutch +blubber +sensor +correct +monster +zombi +vinyl +jaywalk +cough +underwear +waiter +buzzer +thing +registrar +blacksmith +think +frequent +first +lone +crib +long +extrovert +thoughtless +lap +autocrat +escort +daunt +mermaid +anus +yoghourt +memo +broadcast +butt +proofread +tractor +coconut +lick +piccolo +marijuana +dash +comedian +sulk +nazi +sherbert +stopgap +daredevil +acumen +squad +interior +channel +pain +trace +roster +track +acrid +zigzag +whizz +assault +billow +pair +synonym +napkin +typeset +scowl +voodoo +toucan +amir +shop +lexicon +shot +show +cornea +veranda +therapist +shoe +threshold +corner +label +cornet +enthusiast +fend +objector +flapjack +dice +plume +enough +syphon +black +consent +enthusiasm +fiendish +plump +get +straggler +stomp +midriff +slyness +gee +gibber +neckerchief +gem +disinherit +beseech +skull +businesswoman +yield +stupid +nostril +tallow +kernel +sear +eighteenth +seat +seam +seal +stigma +calendar +wonder +puma +parent +limber +ornament +forecast +gage +pump +august +foreword +slingshot +tacit +wednesday +gauntlet +childbirth +tug +tuck +trader +tour +tout +delirium +stretcher +cancer +spank +cancel +tub +mare +underworld +imp +undershirt +mark +mart +workshop +rancher +fiftieth +chalet +graveyard +squash +wake +sound +litterbug +epidermis +slumber +cock +strait +strain +sudden +protein +par +pat +harsh +paw +pay +woodland +same +heartbreak +pad +cotton +pal +pan +exhaust +oil +chloroform +munch +companion +foghorn +polygon +drain +vertebra +soundproof +outdoor +suitor +money +imprint +leeway +aspect +flavor +asthma +godchild +comradeship +forgo +pile +pill +grip +grit +mop +mow +moo +mom +mob +railroad +grim +grin +oxygen +server +chamber +nose +hallelujah +fulfil +sneaker +afflict +witchcraft +ascend +dole +ascent +spasm +gross +confirm +pioneer +inject +gladden +highbrow +linoleum +intravenous +knife +raincoat +broker +squall +bravado +racoon +opium +contagion +roar +island +insect +mixer +thrive +partizan +road +checkup +dagger +coupon +splint +empress +whiten +strip +uptown +skillet +paraphernalia +jigsaw +totalitarian +madden +tycoon +tripod +striker +shroud +hiccup +gore +spice +ember +magnolia +grouch +conqueror +embed +deadlock +affection +deer +deep +fellow +planetarium +deem +file +girlfriend +deed +hound +film +fill +tedious +selfish +personnel +hybrid +repent +drouth +field +prism +astronaut +fruitless +lapel +shelter +gander +unload +burrow +god +gangway +oral +motel +represent +forger +pheasant +forget +founder +suburban +dollar +rebind +zinc +implement +crimson +hideous +premium +parcel +straightforward +scout +scour +fall +bottleneck +pueblo +hinterland +dampen +dictatorship +flyover +neighborhood +clinch +gnarl +burger +zero +cottonwood +lawyer +further +misrepresent +ribbon +dial +skeptic +stool +trinket +stoop +plush +movement +girlhood +malaria +intrench +twang +mule +ranger +beacon +bigamist +capacitor +search +stupor +margin +airport +chipper +chieftain +narrow +fatten +quotient +wizard +caravan +transit +sadist +sadism +establish +dachshund +hobgoblin +eye +score +distinct +two +splash +libel +furor +wiper +diamond +brisk +opportunist +particular +disfavor +nineteen +town +hour +cluster +fast +dew +remain +paragraph +den +abandon +stubborn +shark +buttress +onslaught +share +sphere +minimum +rainstorm +attain +junket +sharp +botanist +siren +awkward +comfort +rapport +stir +bleat +whopper +blacken +blood +bloom +chute +coax +orchard +coat +doctor +spiteful +electron +blunder +mislead +coal +sect +infant +setback +radar +dough +lava +suffer +hundredth +sodium +bosom +late +speech +clamor +lath +lookout +goof +good +goon +detour +frigid +compound +detach +complain +bombard +headroom +countersign +token +monsoon +clamp +harm +hark +mental +hare +hard +beret +banquet +connect +fist +callus +hart +orient +harp +flower +creditor +trooper +pigeon +seaport +granola +print +foreground +assist +cockroach +pleasant +gig +faucet +prophet +omit +wither +pure +corkscrew +copper +perturb +barbarian +shoal +cups +jabber +razor +construct +paint +leash +statement +mama +hummingbird +catapult +pare +park +selector +glycerin +dentist +part +pars +youth +totter +plead +hangout +cistern +blanch +mountain +cardigan +couch +onset +build +zucchini +flute +salmon +chart +most +charm +moss +eskimo +organist +humanitarian +mammoth +pennant +squelch +weigh +standoff +sector +sparrow +fine +find +giant +merger +nervous +ruin +fiend +boulder +prowess +paperweight +cholera +express +ferret +cheapen +batter +breast +theft +silk +pellet +restart +silo +huff +common +archaeologist +printout +vine +lion +overeat +tender +expert +burner +myriad +stowaway +subscript +hypochondria +premiss +egotist +complement +figurehead +mailbox +pagoda +aircraft +sultan +archway +annual +foreign +point +smother +newborn +pamphlet +dancer +esophagus +platinum +pocketbook +secret +amnesia +reformat +finalist +ram +gay +gas +gap +holler +gal +understand +gag +chatter +gab +bile +politician +metro +solid +bill +holocaust +crutch +fun +lingo +manner +mystic +astound +rancor +eczema +ingrain +anaesthesia +sociologist +dishonor +ewe +seminar +corridor +neutron +itch +leopard +yesterday +moment +stripe +unveil +timid +task +werewolf +withdraw +landmark +grid +recant +spend +howl +darn +shape +snot +timber +rundown +impetus +cut +cur +pollster +snag +forbid +cue +punter +cub +snap +bridal +easter +brainstorm +bin +squawk +rebound +bib +judgement +redeem +bit +knock +disrepair +blemish +flue +fagot +flux +bamboo +foolish +walrus +sequin +transgress +often +back +impeach +extremist +mirror +lightning +scald +scale +culprit +pet +pelt +pew +pep +pen +scalp +lard +lark +peg +pea +patient +fed +megaton +constraint +oatmeal +drama +catnip +pediatrician +offshoot +obstetrician +gambit +maelstrom +tiff +clack +lesson +jockey +few +doll +errand +camera +handbook +forward +nougat +sideshow +showman +switchboard +calico +lifeguard +planet +jumbo +azalea +constant +flow +possessor +lye +curd +cure +curb +curl +prevail +stagecoach +leaflet +crypt +underweight +cellar +lend +tablespoon +papa +lens +cater +desert +statesman +mantel +notion +uterus +anguish +caribou +stroller +seaman +golfer +strew +parrakeet +peanut +welter +mower +rudder +compost +blaze +atlas +gravel +queen +dessert +rhyme +claustrophobia +surgeon +molar +verandah +knight +shock +crow +queer +crop +append +power +junior +anthem +access +clipboard +bachelor +intercept +sink +sing +roof +bode +implicit +remark +talent +conceit +resurrect +weekday +climb +honor +blizzard +liqueur +talon +oval +scandal +gateway +sermon +lime +patron +asteroid +butler +charcoal +trait +kiosk +thatch +trail +train +armadillo +harvest +fan +account +tunnel +carrot +obvious +smear +parkway +unread +fetch +employe +truism +sanitarium +teamster +boney +spruce +serial +contempt +hangar +lamb +lame +lamp +forest +goner +stock +roam +leukemia +bluff +terrier +fray +drape +bind +guru +liner +linen +chief +poacher +furious +furnish +disarm +meter +bunch +marshmallow +decorum +labor +kindergarten +heroism +willing +marsh +dad +junction +dab +dam +spell +swordfish +mention +courtroom +sonata +day +strive +flail +snowdrift +thrill +slacken +cider +memoir +sawdust +disregard +flair +thwart +jailor +jugular +pivot +cupboard +lentil +salesman +hippopotamus +matt +defend +rev +repress +stub +mate +barley +stud +smog +stun +red +franc +frank +hanker +fourteen +salami +likelihood +afterward +squadron +indent +mortar +skateboard +yarn +mortal +workbook +retain +retail +waitress +suffix +overshoot +ethnic +sack +brute +whoop +puppet +guidebook +vandal +pauper +ancient +monkey +bologna +laps +vulgar +vagina +hexagon +scant +liquor +cabin +sixteenth +gear +eavesdrop +bulldog +smolder +forethought +springboard +nun +bodyguard +prune +shrapnel +shampoo +linchpin +lover +anthropologist +tide +cavern +pedlar +countryman +waken +optimum +mix +parka +spinster +meek +dryness +hazel +eight +clamber +handbag +hoodwink +transcript +payment +gullet +gulley +gather +request +absurd +rendezvous +occasion +thicken +recess +kite +text +hamlet +traitor +industrialist +sidetrack +portfolio +floodlight +thicket +staff +communism +scorpion +madman +prolong +resubmit +satan +oppressor +communist +inferior +equilibrium +gumdrop +starch +beat +rush +bear +beam +bean +october +beak +bead +organ +ashtray +nutriment +eyebrow +motherhood +mascara +conform +showdown +infidel +racket +interview +reform +pattern +nebula +brunt +hammock +progress +tailspin +sorrow +stratagem +deliver +blackbird +boloney +exclaim +instant +joke +equal +kosher +swim +swallow +highland +guerrilla +glorious +wear +comment +vent +denim +overcoat +commend +vend +harpoon +manhood +citizenship +copier +newscast +gaze +teamwork +gulch +curtain +curtail +hyacinth +juggler +censor +goddess +bulk +bull +bulb +skew +carburetor +cypher +plain +homey +bray +kinfolk +bicker +dissent +squid +blimp +creak +prose +partner +inspector +lynchpin +portray +whirl +grinder +matchbook +defiant +anorak +tumbler +infer +whirr +tighten +pockmark +sauna +ion +grandstand +sunburn +judgment +retard +center +builder +pickpocket +thought +starboard +usual +coaster +humdrum +fingerprint +storey +clinic +interim +surpass +tough +earshot +flashlight +tong +flee +lupin +lake +bench +add +citizen +ado +crossroad +ravel +match +raven +cantaloup +punctual +newsstand +dryer +insert +flamingo +like +success +sofa +journalist +heed +arraign +chick +soft +heel +outfield +propel +fuze +hail +hair +convey +proper +paddock +novelist +shrug +shrub +slide +tureen +regain +pepper +hose +slight +host +panel +beaker +actual +socket +flake +preen +toadstool +pickaback +discard +tomb +tome +snitch +chronic +guard +esteem +custard +underpass +glimmer +gene +maze +globe +buy +bus +coke +sequel +but +bun +bum +bug +bud +embargo +woodsman +wise +ecosystem +debrief +flip +wisp +wist +trapezoid +condiment +plutonium +pin +garter +domino +circus +pie +pig +pit +campus +gush +oaf +cashew +oak +detail +virtual +detain +sewer +oar +redden +dresser +wallow +nutrient +godsend +yelp +baker +jab +hiker +pupil +yell +cookbook +vermin +sleek +sleet +sleep +liar +hate +trolley +sallow +tweet +glider +under +tweed +pride +merchant +lure +risk +rise +lurk +jack +confetti +anemia +school +parrot +enjoy +overdo +cracker +almond +direct +nail +street +monorail +ransack +blue +hide +worsen +poison +beater +supplier +dashboard +wink +even +pontoon +studio +path +crossbow +enrol +connoisseur +forum +ravish +auction +settler +mentor +midway +blowtorch +stray +straw +strap +cassino +would +phlegm +bayou +asset +spike +preview +musk +mush +saber +muse +grief +phone +muss +pouch +must +shoot +hutch +ma +ms +mr +machinist +fortress +quarrel +loosen +joyous +hemoglobin +dolphin +mayhem +attract +end +trill +keen +bunk +vagrant +rhinoceros +shred +toxin +gate +ancestor +dialect +moisten +kilowatt +mess +lump +mesh +sparkler +parallel +stronghold +splendid +spout +patent +enter +vapor +hedgehog +fetter +deform +clapper +sprout +over +bleach +mallard +oven +caster +digest +forehead +theologian +womanhood +comprehend +tramp +drawback +fade +croquet +tourist +plaster +roost +knighthood +monarch +rental +gloom +chuck +choir +prohibit +hanger +unscrew +gymnasium +poncho +truant +saturday +depress +goo +lair +dinosaur +nitwit +psychic +tonsil +gob +emphysema +nite +washer +resistor +carcass +rail +free +rain +acrobat +fret +harpist +ritual +filter +hopeless +soda +rang +accent +puck +rank +restrict +rant +sober +toy +their +sarcasm +top +tow +tot +fiction +ton +duress +toe +urban +murder +overdraw +tool +hearth +embellish +solicitor +toot +incur +western +nonpartisan +lather +prong +flame +mirth +countess +rag +donkey +fashion +handkerchief +ruckus +taint +raw +rat +rap +protract +spade +ray +snow +thorough +contact +hatch +snob +cleft +extravert +quicken +rider +evangelist +shallow +milkman +coil +coin +glow +interject +flop +metal +freeway +policewoman +flog +yank +chariot +bait +endear +saga +alight +random +sage +dupe +radio +rector +earth +bail +shellfish +spite +stanza +disgust +axiom +waltz +gees +watch +fluid +ultimatum +report +reconstruct +noon +spokesperson +egoism +public +erupt +pacifist +pummel +habit +wrest +nut +resist +corrupt +hourglass +mull +mud +mug +finger +mum +approach +wean +weak +contort +boss +toothbrush +southeast +larynx +devour +devout +censorship +newt +protect +irregular +fault +papyrus +facet +elf +smuggler +trust +bingo +bathroom +beef +legend +beer +spread +communion +loft +bladder +uncommon +craft +spearmint +catch +snipe +teapot +misfit +lessen +thousandth +referendum +pyramid +handrail +broth +lollypop +exterior +suggest +wound +overstep +utilitarian +complex +papaya +screw +pick +deflect +suburb +portal +postmark +tassel +ocean +mother +bough +bugger +rodent +shorthand +enlighten +elk +elm +moonbeam +flutist +kelp +misprint +teetotal +upturn +ramrod +dismount +quicksand +spanner +authorship +roach +befit +rumor +apart +ditto +gift +zeal +contradict +hunt +dishonest +zoom +mongrel +hunk +mosquito +hunchback +sanction +excerpt +curio +accost +usher +indirect +intellect +doorstep +nobleman +cooper +combat +letterhead +ice +rhino +newsprint +skylight +convict +christmas +splendor +cord +core +khaki +brawl +corn +brawn +cork +discount +shuck +plug +census +cowboy +plum +choke +surround +caulk +dinner +plus +alga +duke +abet +civic +civil +bath +engulf +cafeteria +art +scamper +transform +sunlight +forbad +virgin +gin +head +medium +amateur +heal +stereo +heat +hear +heap +raft +counsel +muster +bargain +bide +latch +adorn +trim +trio +forearm +cobbler +trip +no +tit +when +junta +tin +whet +tie +implant +depot +pseudonym +evergreen +cleric +toad +geneticist +bullet +navel +yacht +withhold +fasten +backward +coach +impression +rob +rod +focus +livelihood +snip +yokel +rot +discern +environment +aplomb +melon +prop +coop +impend +plantain +cook +cool +looney +level +brother +encroach +quick +lever +pork +drier +trend +bullfrog +pore +inland +voucher +takeoff +bake +port +colic +hymn +choral +postman +spire +theist +thresh +tormentor +humorist +water +fluke +entertain +witch +twentieth +tire +boast +catnap +blotch +cinnamon +prude +weird +tweak +brighten +touchdown +post +panacea +concoct +scan +handler +prey +today +chapter +conductor +altar +cashier +drown +dismal +inhabit +judo +conceal +flagship +hullabaloo +fauna +laughter +streak +overpass +sandbag +trump +stream +despot +stroke +cube +hydrogen +bigot +dress +vital +fourth +dope +ballroom +fascist +clone +scoff +fascism +birthday +apprenticeship +eighth +repeat +classroom +twitch +liquid +inform +reaper +lagoon +superscript +refund +rye +midnight +blare +worship +thermostat +apex +platform +farmer +meridian +cutter +underneath +conquer +fern +rescind +wagon +term +name +realist +opera +bunion +bullion +realism +ailment +torch +zebra +distributor +hysteria +hacker +concur +profit +middleman +gram +clover +hull +hulk +flyer +tuner +flare +highjack +motion +turn +place +swine +swing +turf +preach +childhood +origin +pelican +feign +suspend +insist +scollop +bobcat +array +peddler +given +afterthought +district +opus +trillion +plastic +assort +white +hue +hug +hub +cope +season +hut +enigma +naught +grunt +holder +wide +bewitch +spokeswoman +oath +powder +rend +froth +pro +ani +ant +rent +dragon +stolid +marathon +ideal +blunt +surf +sure +aspen +equestrian +tornado +freshman +librarian +bluebird +icon +latex +tendon +annul +seafood +later +koran +readjust +senior +slope +perch +convoy +cheat +cheap +trespass +hack +broach +hustler +trot +woolen +gulf +genius +gull +shimmer +crime +gulp +woof +wood +deign +wool +entrant +viewpoint +lighten +jazz +festoon +tailor +lighter +dye +homestead +reveal +aluminum +workman +joker +dumfound +bison +picnic +pane +vizor +prowl +optometrist +detect +crooked +review +spoons +hiss +smokestack +caucus +fearless +guitar +coma +comb +come +zodiac +isthmus +reaction +superstar +region +quiet +contract +railway +penal +adjoin +color +armchair +pot +period +pop +pole +colon +polo +pod +poll +runaway +turkey +hobo +schoolboy +tiger +padlock +hernia +careful +spirit +robber +pilot +case +shaft +amend +mount +cash +cask +cast +mound +ventriloquist +vest +exult +clutter +helmet +projector +author +alphabet +fender +bowl +check +macaroni +catfish +bellboy +hermit +week +sang +applaud +nest +driver +weed +director +petticoat +lute +puke +vowel +muffler +weep +cartoon +ranch +relief +model +reward +sinner +clod +clog +tip +kilt +ninth +clot +lavish +violent +kill +kiln +kilo +polish +satin +blow +blot +hint +rose +regent +except +blog +bloc +blob +hind +scrapbook +disrupt +impound +kingdom +blowout +sandman +mugger +towel +bracelet +snort +friar +tower +node +deduct +wombat +interlock +canteen +slice +mood +slick +legal +moon +moos +teardrop +moot +heir +porter +metropolis +quit +unmask +slaughter +quip +ok +oh +of +jeer +shrimp +pistachio +karat +stand +ox +doze +accredit +amber +tribe +vicar +polka +garb +spinal +forewarn +feudal +whaler +there +racism +strict +racist +valley +fish +gibe +relic +jug +regard +cabinet +castaway +strenuous +jut +terminus +feeler +grasp +grass +toilet +ruffian +cinema +frighten +lichen +encompass +bishop +incorrect +abyss +fiddler +heather +idiot +diarrhea +rubber +idiom +heathen +trash +stalwart +championship +symbol +cove +nucleus +serious +brass +wife +invest +derrick +treason +apparel +platter +all +lace +duplex +lack +spacecraft +disc +dish +follow +settlement +titter +wanton +thursday +program +neglig +woman +song +fat +roe +psychologist +retract +ultraviolet +awful +dapper +fad +induct +stimulus +list +trench +align +flick +ten +tea +tee +rate +design +chalk +what +sub +sun +sum +whimper +rascal +brief +overload +crush +version +pulpit +intersect +row +womb +lacquer +pumpernickel +backer +goodby +thrift +misinterpret +heifer +jogger +cataract +haemophilia +murmur +snug +snub +herring +proceed +tarpaulin +wield +hurray +rustic +quash +inlay +garnish +hurrah +minor +ladybug +wretched +flap +mire +protestor +stutter +flit +flag +stick +mellow +chaplain +berth +wrestler +plagiarist +searchlight +sunscreen +pond +court +goal +goad +goat +sandwich +okay +algebra +headrest +embalm +reflect +catalog +numb +short +ricochet +tsar +shade +waylay +mission +scientist +flaunt +reconnect +pretext +stride +islam +thirtieth +style +glide +pray +inward +wilder +abbey +mattress +resort +airstrip +bout +soccer +might +alter +return +hunter +underground +abacus +mathematician +liniment +policeman +refresh +tactless +friendship +weight +needless +duchess +falsetto +expect +inflict +wager +alcohol +disquiet +hilt +dugout +loll +health +hill +shipment +fiber +solvent +friday +differ +effect +disinfect +octagon +physic +teach +sidewalk +jew +blister +thread +threat +bushel +feed +dine +feel +sailor +revolution +least +blank +cigaret +idea +moan +script +gourmet +interact +grime +stork +swarm +storm +moat +syrup +store +mainstay +imperfect +option +hotel +fidget +king +kind +vial +kink +stall +cuff +foreleg +stale +restful +amass +cleaner +exert +strengthen +shrewd +bookworm +gale +gala +gall +remodel +smallpox +toughen +bacteria +chairman +donor +pianist +buff +gill +foreman +rapist +reckon +english +reach +react +nothing +quorum +hyena +amphibian +saloon +notch +scaffold +asphalt +memorandum +felon +font +anvil +firewood +betray +hip +shepherd +hit +deaden +reprint +him +adulthood +snowstorm +forego +stump +martyrdom +arc +bare +bard +bark +ark +arm +barn +blurt +parchment +various +plywood +nincompoop +solo +muslim +sole +outfit +succeed +inertia +orangutang +blazer +bandit +context +bond +cynic +sloth +flier +distress +chaperon +sweet +wastebasket +sweep +weasel +rave +shaykh +bolster +dub +overlook +dud +due +buttermilk +pa +watchword +brick +pi +flight +quintet +dropout +marihuana +cinch +temperament +instructor +heighten +toga +shove +batch +pitchfork +kick +behavior +incognito +lodger +bluegrass +sirup +rip +shamrock +rim +frantic +rig +rid +reprogram +chauffeur +shirt +kimono +viola +shirk +sliver +straightjacket +restraint +painless +throwback +cement +birch +robust +knack +lower +earmark +cheek +cheep +cheer +pollen +facial +vigilant +cabaret +continent +tablet +contractor +plateau +tuxedo +complaint +vendor +foreshadow +awaken +confront +uproar +distrust +breeder +hallmark +play +global +litter +wonderland +butterscotch +saucer +prow +seller +prom +prod +sag +perpendicular +tinker +raider +vivid +cautious +undertow +yawn +ordeal +militia +dialog +tomboy +conquest +momma +piteous +holster +vagabond +stench +canal +pundit +question +swill +parsnip +etch +filet +potassium +glamour +cloth +crank +usurp +delta +upright +crane +outpost +penguin +patriot +consist +apricot +caller +peppermint +husk +cartwheel +highlight +dill +freak +dismay +sublet +sagebrush +rainbow +lemon +riot +peach +grouper +saffron +nick +parlor +ferment +bandstand +mock +nice +mustard +chirp +meaning +vigil +vice +ocular +remit +pyre +buffalo +scroll +pervert +lean +alien +dispel +gang +theorist +gold +uphold +floss +breach +sniper +materialist +toenail +spool +spoon +spook +spoof +harlot +outdo +pleas +pleat +trawl +procession +fold +reunion +acid +folk +sandstorm +outsmart +acronym +relent +kangaroo +gloat +miser +cyclist +barb +survivor +guarantor +orangutan +armpit +shovel +duct +ensign +apt +volt +motor +duck +thick +redo +ape +use +fee +fen +frog +germ +modicum +fez +sort +parliament +porch +musician +impress +sore +rabbit +recount +penis +sculptor +annoy +topic +augment +critic +lumber +executor +proof +bittersweet +tap +tar +tax +villain +tag +condescend +tab +spa +silt +tan +rape +counterfeit +sir +sip +scuff +sit +tamper +six +outclass +occur +sic +carrier +goldsmith +toddler +panic +sin +defeat +tension +lesion +attend +tact +hazard +discomfort +tack +wrist +taco +footpath +aftereffect +light +arduous +schoolchild +sailboat +stamped +minnow +damsel +accompanist +hemp +tyrant +badger +glen +superior +inlet +sill +glee +nostalgia +flank +restrain +glisten +turban +redhead +bye +flex +crash +citrus +flour +flout +emerald +flea +republican +investor +successor +easel +footstool +profound +edit +feast +fuzz +trap +blacktop +cocoon +tray +lilac +mincemeat +interplay +our +proclaim +out +semen +tabloid +cocktail +sentiment +frontier +vehement +disarray +clatter +impart +plural +proviso +planner +utensil +tenement +pendant +gospel +tenant +greenhorn +tanker +zoologist +rivet +uproot +embryo +sew +bouquet +echo +bonnet +eleventh +synagog +salient +droop +unknown +galley +snore +anaemia +drool +boil +tidbit +shell +shelf +transistor +woo +diminish +persecutor +goblin +institution +kickback +frugal +brazen +yodel +laughingstock +clip +fowl +splatter +flunkey +blip +footwork +outstrip +disjoint +pallor +catholic +clove +rout +outward +bagel +lope +divert +trivia +pharmacist +divers +clash +petrol +siphon +filch +fortieth +class +clasp +fang +dens +dent +pipe +vernacular +gain +son +stove +sonnet +utter +chicken +feat +winch +dandruff +rioter +herald +piano +local +counsellor +vigor +sued +skimp +plaintiff +spud +watercolor +barter +bronco +spur +rite +ghetto +bisect +compliment +ascertain +sediment +view +unison +workbench +ebb +expel +hymnal +distract +violet +still +closet +superb +favor +viper +crude +torpedo +avow +jot +exam +amen +joy +foetus +job +spoil +jog +swift +memento +lifeboat +april +grain +commando +wall +hyphen +walk +respect +unclean +decent +trademark +tutor +reindeer +mike +nickel +cypress +penmanship +dearth +overturn +present +kerchief +corset +wilt +vanilla +priesthood +will +fingertip +wild +whirlpool +layer +mutant +motif +apprehend +rooster +lightweight +thug +thud +whore +headlight +cross +member +pediatrist +inch +grandeur +slave +diploma +outcast +beast +student +pedal +whale +collar +gutter +masochist +overwork +scissor +twirl +flint +outgrow +bandanna +rocker +cameo +rocket +camel +boot +wren +obtain +replenish +biologist +daub +distend +smite +now +panther +drunk +smith +hall +book +ski +enact +knob +sick +myth +know +knot +press +redesign +doughnut +loser +cutlet +vortex +clutch +exceed +setter +flagrant +birthmark +demeanor +growth +export +leaf +lead +leak +miner +leap +belt +leader +trout +obey +slur +mitt +slut +slum +pasta +mite +slug +throne +pike +throng +rare +linger +column +biscuit +fear +swear +sweat +udder +emperor +owl +outset +own +owe +weather +champ +brush +billfold +gape +rowboat +van +platoon +transfer +spiral +grotto +cliff +vat +nourish +catsup +unwrap +saunter +mutter +brassier +assail +tomcat +daffodil +nightgown +record +cake +faggot +maroon +boardwalk +abbot +counteract +limb +squirrel +mutual +glint +boor +percent +other +boom +branch +cutthroat +junk +mulch +june +squeak +squeal +extort +jewel +gynecologist +vane +sash diff --git a/libs/pyutil/fileutil.py~ b/libs/pyutil/fileutil.py~ deleted file mode 100644 index e37eb792..00000000 --- a/libs/pyutil/fileutil.py~ +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn -# This file is part of pyutil; see README.rst for licensing terms. - -""" -Futz with files like a pro. -""" - -import errno, exceptions, os, stat, tempfile - -try: - import bsddb -except ImportError: - DBNoSuchFileError = None -else: - DBNoSuchFileError = bsddb.db.DBNoSuchFileError - -# read_file() and write_file() copied from Mark Seaborn's blog post. Please -# read it for complete rationale: -# http://lackingrhoticity.blogspot.com/2009/12/readfile-and-writefile-in-python.html - -def read_file(filename, mode='rb'): - """ Read the contents of the file named filename and return it in - a string. This function closes the file handle before it returns - (even if the underlying Python implementation's garbage collector - doesn't). """ - fh = open(filename, mode) - try: - return fh.read() - finally: - fh.close() - -def write_file(filename, data, mode='wb'): - """ Write the string data into a file named filename. This - function closes the file handle (ensuring that the written data is - flushed from the perspective of the Python implementation) before - it returns (even if the underlying Python implementation's garbage - collector doesn't).""" - fh = open(filename, mode) - try: - fh.write(data) - finally: - fh.close() - -# For backwards-compatibility in case someone is using these names. We used to -# have a superkludge in fileutil.py under these names. -def rename(src, dst, tries=4, basedelay=0.1): - return os.rename(src, dst) - -def remove(f, tries=4, basedelay=0.1): - return os.remove(f) - -def rmdir(f, tries=4, basedelay=0.1): - return os.rmdir(f) - -class _Dir(object): - """ - Hold a set of files and subdirs and clean them all up when asked to. - """ - def __init__(self, name, cleanup=True): - self.name = name - self.cleanup = cleanup - self.files = [] - self.subdirs = set() - - def file(self, fname, mode=None): - """ - Create a file in the tempdir and remember it so as to close() it - before attempting to cleanup the temp dir. - - @rtype: file - """ - ffn = os.path.join(self.name, fname) - if mode is not None: - fo = open(ffn, mode) - else: - fo = open(ffn) - self.register_file(fo) - return fo - - def subdir(self, dirname): - """ - Create a subdirectory in the tempdir and remember it so as to call - shutdown() on it before attempting to clean up. - - @rtype: _Dir instance - """ - ffn = os.path.join(self.name, dirname) - sd = _Dir(ffn, self.cleanup) - self.register_subdir(sd) - make_dirs(sd.name) - return sd - - def register_file(self, fileobj): - """ - Remember the file object and call close() on it before attempting to - clean up. - """ - self.files.append(fileobj) - - def register_subdir(self, dirobj): - """ - Remember the _Dir object and call shutdown() on it before attempting - to clean up. - """ - self.subdirs.add(dirobj) - - def shutdown(self): - if self.cleanup: - for subdir in hasattr(self, 'subdirs') and self.subdirs or []: - subdir.shutdown() - for fileobj in hasattr(self, 'files') and self.files or []: - if DBNoSuchFileError is None: - fileobj.close() # "close()" is idempotent so we don't need to catch exceptions here - else: - try: - fileobj.close() - except DBNoSuchFileError: - # Ah, except that the bsddb module's file-like object (a DB object) has a non-idempotent close... - pass - - if hasattr(self, 'name'): - rm_dir(self.name) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - try: - self.shutdown() - except: - import traceback - traceback.print_exc() - -class NamedTemporaryDirectory(_Dir): - """ - Call tempfile.mkdtemp(), store the name of the dir in self.name, and - rm_dir() when it gets garbage collected or "shutdown()". - - Also keep track of file objects for files within the tempdir and call - close() on them before rm_dir(). This is a convenient way to open temp - files within the directory, and it is very helpful on Windows because you - can't delete a directory which contains a file which is currently open. - """ - - def __init__(self, cleanup=True, *args, **kwargs): - """ If cleanup, then the directory will be rmrf'ed when the object is shutdown. """ - name = tempfile.mkdtemp(*args, **kwargs) - _Dir.__init__(self, name, cleanup) - -class ReopenableNamedTemporaryFile: - """ - This uses tempfile.mkstemp() to generate a secure temp file. It then closes - the file, leaving a zero-length file as a placeholder. You can get the - filename with ReopenableNamedTemporaryFile.name. When the - ReopenableNamedTemporaryFile instance is garbage collected or its shutdown() - method is called, it deletes the file. - """ - def __init__(self, *args, **kwargs): - fd, self.name = tempfile.mkstemp(*args, **kwargs) - os.close(fd) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - self.shutdown() - - def shutdown(self): - remove(self.name) - -def make_dirs(dirname, mode=0777): - """ - An idempotent version of os.makedirs(). If the dir already exists, do - nothing and return without raising an exception. If this call creates the - dir, return without raising an exception. If there is an error that - prevents creation or if the directory gets deleted after make_dirs() creates - it and before make_dirs() checks that it exists, raise an exception. - """ - tx = None - try: - os.makedirs(dirname, mode) - except OSError, x: - tx = x - - if not os.path.isdir(dirname): - if tx: - raise tx - raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning... - -def rmtree(dirname): - """ - A threadsafe and idempotent version of shutil.rmtree(). If the dir is - already gone, do nothing and return without raising an exception. If this - call removes the dir, return without raising an exception. If there is an - error that prevents deletion or if the directory gets created again after - rm_dir() deletes it and before rm_dir() checks that it is gone, raise an - exception. - """ - excs = [] - try: - os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - for f in os.listdir(dirname): - fullname = os.path.join(dirname, f) - if os.path.isdir(fullname): - rm_dir(fullname) - else: - remove(fullname) - os.rmdir(dirname) - except EnvironmentError, le: - # Ignore "No such file or directory", collect any other exception. - if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): - excs.append(le) - except Exception, le: - excs.append(le) - - # Okay, now we've recursively removed everything, ignoring any "No - # such file or directory" errors, and collecting any other errors. - - if os.path.exists(dirname): - if len(excs) == 1: - raise excs[0] - if len(excs) == 0: - raise OSError, "Failed to remove dir for unknown reason." - raise OSError, excs - -def rm_dir(dirname): - # Renamed to be like shutil.rmtree and unlike rmdir. - return rmtree(dirname) - -def remove_if_possible(f): - try: - remove(f) - except EnvironmentError: - pass - -def remove_if_present(f): - try: - remove(f) - except EnvironmentError, le: - # Ignore "No such file or directory", re-raise any other exception. - if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): - raise - -def rmdir_if_possible(f): - try: - rmdir(f) - except EnvironmentError: - pass - -def open_or_create(fname, binarymode=True): - try: - f = open(fname, binarymode and "r+b" or "r+") - except EnvironmentError: - f = open(fname, binarymode and "w+b" or "w+") - return f - -def du(basedir): - size = 0 - - for root, dirs, files in os.walk(basedir): - for f in files: - fn = os.path.join(root, f) - size += os.path.getsize(fn) - - return size diff --git a/libs/pyutil/iputil.py b/libs/pyutil/iputil.py index b8b96362..cb3e7c02 100644 --- a/libs/pyutil/iputil.py +++ b/libs/pyutil/iputil.py @@ -1,22 +1,12 @@ -# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith) -# portions adapted from nattraverso.ipdiscover -# portions authored by Brian Warner, working for Allmydata -# most recent version authored by Zooko O'Whielacronx, working for Allmydata - # from the Python Standard Library -import os, re, socket, sys +import os, re, socket, sys, subprocess # from Twisted -from twisted.internet import defer, reactor -from twisted.python import failure +from twisted.internet import defer, threads, reactor from twisted.internet.protocol import DatagramProtocol -from twisted.internet.utils import getProcessOutput from twisted.python.procutils import which from twisted.python import log -# from pyutil -import observer - try: import resource def increase_rlimits(): @@ -77,6 +67,7 @@ except ImportError: # since one might be shadowing the other. This hack appeases pyflakes. increase_rlimits = _increase_rlimits + def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad @@ -121,14 +112,16 @@ def get_local_ip_for(target): except socket.gaierror: # DNS isn't running, or somehow we encountered an error - # note: if an interface is configured and up, but nothing is connected to it, - # gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror - # . This is synchronous and occurs for each node being started, so users of certain unit - # tests will see something like 120s of delay, which may be enough to hit the default - # trial timeouts. For that reason, get_local_addresses_async() was changed to default to - # the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also - # makes node startup a tad faster. - + # note: if an interface is configured and up, but nothing is + # connected to it, gethostbyname("A.ROOT-SERVERS.NET") will take 20 + # seconds to raise socket.gaierror . This is synchronous and occurs + # for each node being started, so users of + # test.common.SystemTestMixin (like test_system) will see something + # like 120s of delay, which may be enough to hit the default trial + # timeouts. For that reason, get_local_addresses_async() was changed + # to default to the numerical ip address for A.ROOT-SERVERS.NET, to + # avoid this DNS lookup. This also makes node startup fractionally + # faster. return None udpprot = DatagramProtocol() port = reactor.listenUDP(0, udpprot) @@ -146,16 +139,29 @@ _platform_map = { "linux-i386": "linux", # redhat "linux-ppc": "linux", # redhat "linux2": "linux", # debian + "linux3": "linux", # debian "win32": "win32", "irix6-n32": "irix", "irix6-n64": "irix", "irix6": "irix", "openbsd2": "bsd", + "openbsd3": "bsd", + "openbsd4": "bsd", + "openbsd5": "bsd", "darwin": "bsd", # Mac OS X "freebsd4": "bsd", "freebsd5": "bsd", "freebsd6": "bsd", + "freebsd7": "bsd", + "freebsd8": "bsd", + "freebsd9": "bsd", "netbsd1": "bsd", + "netbsd2": "bsd", + "netbsd3": "bsd", + "netbsd4": "bsd", + "netbsd5": "bsd", + "netbsd6": "bsd", + "dragonfly2": "bsd", "sunos5": "sunos", "cygwin": "cygwin", } @@ -173,12 +179,12 @@ _win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\ # These work in Redhat 6.x and Debian 2.2 potato _linux_path = '/sbin/ifconfig' -_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) +_linux_re = re.compile('^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) -# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD -_bsd_path = '/sbin/ifconfig' -_bsd_args = ('-a',) -_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) +# NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X +_netbsd_path = '/sbin/ifconfig' +_netbsd_args = ('-a',) +_netbsd_re = re.compile('^\s+inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) # Irix 6.5 _irix_path = '/usr/etc/ifconfig' @@ -186,39 +192,6 @@ _irix_path = '/usr/etc/ifconfig' # Solaris 2.x _sunos_path = '/usr/sbin/ifconfig' -class SequentialTrier(object): - """ I hold a list of executables to try and try each one in turn - until one gives me a list of IP addresses.""" - - def __init__(self, exebasename, args, regex): - assert not os.path.isabs(exebasename) - self.exes_left_to_try = which(exebasename) - self.exes_left_to_try.reverse() - self.args = args - self.regex = regex - self.o = observer.OneShotObserverList() - self._try_next() - - def _try_next(self): - if not self.exes_left_to_try: - self.o.fire(None) - else: - exe = self.exes_left_to_try.pop() - d2 = _query(exe, self.args, self.regex) - - def cb(res): - if res: - self.o.fire(res) - else: - self._try_next() - - def eb(why): - self._try_next() - - d2.addCallbacks(cb, eb) - - def when_tried(self): - return self.o.when_fired() # k: platform string as provided in the value of _platform_map # v: tuple of (path_to_tool, args, regex,) @@ -226,19 +199,22 @@ _tool_map = { "linux": (_linux_path, (), _linux_re,), "win32": (_win32_path, _win32_args, _win32_re,), "cygwin": (_win32_path, _win32_args, _win32_re,), - "bsd": (_bsd_path, _bsd_args, _bsd_re,), - "irix": (_irix_path, _bsd_args, _bsd_re,), - "sunos": (_sunos_path, _bsd_args, _bsd_re,), + "bsd": (_netbsd_path, _netbsd_args, _netbsd_re,), + "irix": (_irix_path, _netbsd_args, _netbsd_re,), + "sunos": (_sunos_path, _netbsd_args, _netbsd_re,), } + def _find_addresses_via_config(): - # originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API. + return threads.deferToThread(_synchronously_find_addresses_via_config) + +def _synchronously_find_addresses_via_config(): + # originally by Greg Smith, hacked by Zooko to conform to Brian's API platform = _platform_map.get(sys.platform) - (pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,)) + if not platform: + raise UnsupportedPlatformError(sys.platform) - # If the platform isn't known then we attempt BSD-style ifconfig. If it - # turns out that we don't get anything resembling a dotted quad IPv4 address - # out of it, then we'll raise UnsupportedPlatformError. + (pathtotool, args, regex,) = _tool_map[platform] # If pathtotool is a fully qualified path then we just try that. # If it is merely an executable name then we use Twisted's @@ -246,34 +222,33 @@ def _find_addresses_via_config(): # gives us something that resembles a dotted-quad IPv4 address. if os.path.isabs(pathtotool): - d = _query(pathtotool, args, regex) + return _query(pathtotool, args, regex) else: - d = SequentialTrier(pathtotool, args, regex).when_tried() - - d.addCallback(_check_result) - return d - -def _check_result(result): - if not result and not _platform_map.has_key(sys.platform): - return failure.Failure(UnsupportedPlatformError(sys.platform)) - else: - return result + exes_to_try = which(pathtotool) + for exe in exes_to_try: + try: + addresses = _query(exe, args, regex) + except Exception: + addresses = [] + if addresses: + return addresses + return [] def _query(path, args, regex): - d = getProcessOutput(path, args) - def _parse(output): - addresses = [] - outputsplit = output.split('\n') - for outline in outputsplit: - m = regex.match(outline) - if m: - addr = m.groupdict()['address'] - if addr not in addresses: - addresses.append(addr) + env = {'LANG': 'en_US.UTF-8'} + p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + (output, err) = p.communicate() - return addresses - d.addCallback(_parse) - return d + addresses = [] + outputsplit = output.split('\n') + for outline in outputsplit: + m = regex.match(outline) + if m: + addr = m.groupdict()['address'] + if addr not in addresses: + addresses.append(addr) + + return addresses def _cygwin_hack_find_addresses(target): addresses = [] diff --git a/libs/pyutil/iputil.py~ b/libs/pyutil/iputil.py~ new file mode 100644 index 00000000..b8b96362 --- /dev/null +++ b/libs/pyutil/iputil.py~ @@ -0,0 +1,288 @@ +# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith) +# portions adapted from nattraverso.ipdiscover +# portions authored by Brian Warner, working for Allmydata +# most recent version authored by Zooko O'Whielacronx, working for Allmydata + +# from the Python Standard Library +import os, re, socket, sys + +# from Twisted +from twisted.internet import defer, reactor +from twisted.python import failure +from twisted.internet.protocol import DatagramProtocol +from twisted.internet.utils import getProcessOutput +from twisted.python.procutils import which +from twisted.python import log + +# from pyutil +import observer + +try: + import resource + def increase_rlimits(): + # We'd like to raise our soft resource.RLIMIT_NOFILE, since certain + # systems (OS-X, probably solaris) start with a relatively low limit + # (256), and some unit tests want to open up more sockets than this. + # Most linux systems start with both hard and soft limits at 1024, + # which is plenty. + + # unfortunately the values to pass to setrlimit() vary widely from + # one system to another. OS-X reports (256, HUGE), but the real hard + # limit is 10240, and accepts (-1,-1) to mean raise it to the + # maximum. Cygwin reports (256, -1), then ignores a request of + # (-1,-1): instead you have to guess at the hard limit (it appears to + # be 3200), so using (3200,-1) seems to work. Linux reports a + # sensible (1024,1024), then rejects (-1,-1) as trying to raise the + # maximum limit, so you could set it to (1024,1024) but you might as + # well leave it alone. + + try: + current = resource.getrlimit(resource.RLIMIT_NOFILE) + except AttributeError: + # we're probably missing RLIMIT_NOFILE + return + + if current[0] >= 1024: + # good enough, leave it alone + return + + try: + if current[1] > 0 and current[1] < 1000000: + # solaris reports (256, 65536) + resource.setrlimit(resource.RLIMIT_NOFILE, + (current[1], current[1])) + else: + # this one works on OS-X (bsd), and gives us 10240, but + # it doesn't work on linux (on which both the hard and + # soft limits are set to 1024 by default). + resource.setrlimit(resource.RLIMIT_NOFILE, (-1,-1)) + new = resource.getrlimit(resource.RLIMIT_NOFILE) + if new[0] == current[0]: + # probably cygwin, which ignores -1. Use a real value. + resource.setrlimit(resource.RLIMIT_NOFILE, (3200,-1)) + + except ValueError: + log.msg("unable to set RLIMIT_NOFILE: current value %s" + % (resource.getrlimit(resource.RLIMIT_NOFILE),)) + except: + # who knows what. It isn't very important, so log it and continue + log.err() +except ImportError: + def _increase_rlimits(): + # TODO: implement this for Windows. Although I suspect the + # solution might be "be running under the iocp reactor and + # make this function be a no-op". + pass + # pyflakes complains about two 'def FOO' statements in the same time, + # since one might be shadowing the other. This hack appeases pyflakes. + increase_rlimits = _increase_rlimits + +def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET + """ + Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad + strings) that are currently configured on this host, sorted in descending + order of how likely we think they are to work. + + @param target: we want to learn an IP address they could try using to + connect to us; The default value is fine, but it might help if you + pass the address of a host that you are actually trying to be + reachable to. + """ + addresses = [] + local_ip = get_local_ip_for(target) + if local_ip: + addresses.append(local_ip) + + if sys.platform == "cygwin": + d = _cygwin_hack_find_addresses(target) + else: + d = _find_addresses_via_config() + + def _collect(res): + for addr in res: + if addr != "0.0.0.0" and not addr in addresses: + addresses.append(addr) + return addresses + d.addCallback(_collect) + + return d + +def get_local_ip_for(target): + """Find out what our IP address is for use by a given target. + + @return: the IP address as a dotted-quad string which could be used by + to connect to us. It might work for them, it might not. If + there is no suitable address (perhaps we don't currently have an + externally-visible interface), this will return None. + """ + + try: + target_ipaddr = socket.gethostbyname(target) + except socket.gaierror: + # DNS isn't running, or somehow we encountered an error + + # note: if an interface is configured and up, but nothing is connected to it, + # gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror + # . This is synchronous and occurs for each node being started, so users of certain unit + # tests will see something like 120s of delay, which may be enough to hit the default + # trial timeouts. For that reason, get_local_addresses_async() was changed to default to + # the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also + # makes node startup a tad faster. + + return None + udpprot = DatagramProtocol() + port = reactor.listenUDP(0, udpprot) + try: + udpprot.transport.connect(target_ipaddr, 7) + localip = udpprot.transport.getHost().host + except socket.error: + # no route to that host + localip = None + port.stopListening() # note, this returns a Deferred + return localip + +# k: result of sys.platform, v: which kind of IP configuration reader we use +_platform_map = { + "linux-i386": "linux", # redhat + "linux-ppc": "linux", # redhat + "linux2": "linux", # debian + "win32": "win32", + "irix6-n32": "irix", + "irix6-n64": "irix", + "irix6": "irix", + "openbsd2": "bsd", + "darwin": "bsd", # Mac OS X + "freebsd4": "bsd", + "freebsd5": "bsd", + "freebsd6": "bsd", + "netbsd1": "bsd", + "sunos5": "sunos", + "cygwin": "cygwin", + } + +class UnsupportedPlatformError(Exception): + pass + +# Wow, I'm really amazed at home much mileage we've gotten out of calling +# the external route.exe program on windows... It appears to work on all +# versions so far. Still, the real system calls would much be preferred... +# ... thus wrote Greg Smith in time immemorial... +_win32_path = 'route.exe' +_win32_args = ('print',) +_win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S) + +# These work in Redhat 6.x and Debian 2.2 potato +_linux_path = '/sbin/ifconfig' +_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) + +# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD +_bsd_path = '/sbin/ifconfig' +_bsd_args = ('-a',) +_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) + +# Irix 6.5 +_irix_path = '/usr/etc/ifconfig' + +# Solaris 2.x +_sunos_path = '/usr/sbin/ifconfig' + +class SequentialTrier(object): + """ I hold a list of executables to try and try each one in turn + until one gives me a list of IP addresses.""" + + def __init__(self, exebasename, args, regex): + assert not os.path.isabs(exebasename) + self.exes_left_to_try = which(exebasename) + self.exes_left_to_try.reverse() + self.args = args + self.regex = regex + self.o = observer.OneShotObserverList() + self._try_next() + + def _try_next(self): + if not self.exes_left_to_try: + self.o.fire(None) + else: + exe = self.exes_left_to_try.pop() + d2 = _query(exe, self.args, self.regex) + + def cb(res): + if res: + self.o.fire(res) + else: + self._try_next() + + def eb(why): + self._try_next() + + d2.addCallbacks(cb, eb) + + def when_tried(self): + return self.o.when_fired() + +# k: platform string as provided in the value of _platform_map +# v: tuple of (path_to_tool, args, regex,) +_tool_map = { + "linux": (_linux_path, (), _linux_re,), + "win32": (_win32_path, _win32_args, _win32_re,), + "cygwin": (_win32_path, _win32_args, _win32_re,), + "bsd": (_bsd_path, _bsd_args, _bsd_re,), + "irix": (_irix_path, _bsd_args, _bsd_re,), + "sunos": (_sunos_path, _bsd_args, _bsd_re,), + } +def _find_addresses_via_config(): + # originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API. + + platform = _platform_map.get(sys.platform) + (pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,)) + + # If the platform isn't known then we attempt BSD-style ifconfig. If it + # turns out that we don't get anything resembling a dotted quad IPv4 address + # out of it, then we'll raise UnsupportedPlatformError. + + # If pathtotool is a fully qualified path then we just try that. + # If it is merely an executable name then we use Twisted's + # "which()" utility and try each executable in turn until one + # gives us something that resembles a dotted-quad IPv4 address. + + if os.path.isabs(pathtotool): + d = _query(pathtotool, args, regex) + else: + d = SequentialTrier(pathtotool, args, regex).when_tried() + + d.addCallback(_check_result) + return d + +def _check_result(result): + if not result and not _platform_map.has_key(sys.platform): + return failure.Failure(UnsupportedPlatformError(sys.platform)) + else: + return result + +def _query(path, args, regex): + d = getProcessOutput(path, args) + def _parse(output): + addresses = [] + outputsplit = output.split('\n') + for outline in outputsplit: + m = regex.match(outline) + if m: + addr = m.groupdict()['address'] + if addr not in addresses: + addresses.append(addr) + + return addresses + d.addCallback(_parse) + return d + +def _cygwin_hack_find_addresses(target): + addresses = [] + for h in [target, "localhost", "127.0.0.1",]: + try: + addr = get_local_ip_for(h) + if addr not in addresses: + addresses.append(addr) + except socket.gaierror: + pass + + return defer.succeed(addresses) diff --git a/libs/pyutil/mathutil.py b/libs/pyutil/mathutil.py index 46781b0c..9c169801 100644 --- a/libs/pyutil/mathutil.py +++ b/libs/pyutil/mathutil.py @@ -11,7 +11,7 @@ def div_ceil(n, d): """ The smallest integer k such that k*d >= n. """ - return (n/d) + (n%d != 0) + return int((n//d) + (n%d != 0)) def next_multiple(n, k): """ diff --git a/libs/pyutil/mathutil.py~ b/libs/pyutil/mathutil.py~ new file mode 100644 index 00000000..46781b0c --- /dev/null +++ b/libs/pyutil/mathutil.py~ @@ -0,0 +1,106 @@ +# Copyright (c) 2005-2010 Zooko Wilcox-O'Hearn +# This file is part of pyutil; see README.rst for licensing terms. + +""" +A few commonly needed functions. +""" + +import math + +def div_ceil(n, d): + """ + The smallest integer k such that k*d >= n. + """ + return (n/d) + (n%d != 0) + +def next_multiple(n, k): + """ + The smallest multiple of k which is >= n. Note that if n is 0 then the + answer is 0. + """ + return div_ceil(n, k) * k + +def pad_size(n, k): + """ + The smallest number that has to be added to n to equal a multiple of k. + """ + if n%k: + return k - n%k + else: + return 0 + +def is_power_of_k(n, k): + return k**int(math.log(n, k) + 0.5) == n + +def next_power_of_k(n, k): + p = 1 + while p < n: + p *= k + return p + +def ave(l): + return sum(l) / len(l) + +def log_ceil(n, b): + """ + The smallest integer k such that b^k >= n. + + log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. + the number of bits needed to store any of 128 possible values is 7. + """ + p = 1 + k = 0 + while p < n: + p *= b + k += 1 + return k + +def log_floor(n, b): + """ + The largest integer k such that b^k <= n. + """ + p = 1 + k = 0 + while p <= n: + p *= b + k += 1 + return k - 1 + +def linear_fit_slope(ps): + """ + Single-independent-variable linear regression -- least squares method. + + At least, I *think* this function computes that answer. I no longer + remember where I learned this trick and at the moment I can't prove to + myself that this is correct. + + @param ps a sequence of tuples of (x, y) + """ + avex = ave([x for (x, y) in ps]) + avey = ave([y for (x, y) in ps]) + sxy = sum([ (x - avex) * (y - avey) for (x, y) in ps ]) + sxx = sum([ (x - avex) ** 2 for (x, y) in ps ]) + if sxx == 0: + return None + return sxy / sxx + +def permute(l): + """ + Return all possible permutations of l. + + @type l: sequence + @rtype a set of sequences + """ + if len(l) == 1: + return [l,] + + res = [] + for i in range(len(l)): + l2 = list(l[:]) + x = l2.pop(i) + for l3 in permute(l2): + l3.append(x) + res.append(l3) + + return res + diff --git a/libs/pyutil/odict.py~ b/libs/pyutil/odict.py~ deleted file mode 100644 index 0ed5ce7b..00000000 --- a/libs/pyutil/odict.py~ +++ /dev/null @@ -1,552 +0,0 @@ -# Copyright (c) 2002-2009 Zooko "Zooko" Wilcox-O'Hearn - -""" -This module offers a Ordered Dict, which is a dict that preserves -insertion order. See PEP 372 for description of the problem. This -implementation uses a linked-list to get good O(1) asymptotic -performance. (Actually it is O(hashtable-update-cost), but whatever.) - -Warning: if -O optimizations are not turned on then OrderedDict performs -extensive self-analysis in every function call, which can take minutes -and minutes for a large cache. Turn on -O, or comment out assert -self._assert_invariants() -""" - -import operator - -from assertutil import _assert, precondition -from humanreadable import hr - -class OrderedDict: - """ - An efficient ordered dict. - - Adding an item that is already in the dict *does not* make it the - most- recently-added item although it may change the state of the - dict itself (if the value is different than the previous value). - - See also SmallOrderedDict (below), which is faster in some cases. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - k = self.i - precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) - (v, p, n,) = self.c.d[k] - self.i = p - return (k, v,) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - k = self.i - precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) - (v, p, n,) = self.c.d[k] - self.i = p - return k - - class ValIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - precondition(self.c.d.has_key(self.i), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - (v, p, n,) = self.c.d[self.i] - self.i = p - return v - - class Sentinel: - def __init__(self, msg): - self.msg = msg - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.msg,) - - def __init__(self, initialdata={}): - self.d = {} # k: k, v: [v, prev, next,] # the dict - self.hs = OrderedDict.Sentinel("hs") - self.ts = OrderedDict.Sentinel("ts") - self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.update(initialdata) - - assert self._assert_invariants() - - def __repr_n__(self, n=None): - s = ["{",] - try: - iter = self.iteritems() - x = iter.next() - s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - i = 1 - while (n is None) or (i < n): - x = iter.next() - s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - except StopIteration: - pass - s.append("}") - return ''.join(s) - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) - - def _assert_invariants(self): - _assert((len(self.d) > 2) == (self.d[self.hs][2] is not self.ts) == (self.d[self.ts][1] is not self.hs), "Head and tail point to something other than each other if and only if there is at least one element in the dictionary.", self.hs, self.ts, len(self.d)) - foundprevsentinel = 0 - foundnextsentinel = 0 - for (k, (v, p, n,)) in self.d.iteritems(): - _assert(v not in (self.hs, self.ts,)) - _assert(p is not self.ts, "A reference to the tail sentinel may not appear in prev.", k, v, p, n) - _assert(n is not self.hs, "A reference to the head sentinel may not appear in next.", k, v, p, n) - _assert(p in self.d, "Each prev is required to appear as a key in the dict.", k, v, p, n) - _assert(n in self.d, "Each next is required to appear as a key in the dict.", k, v, p, n) - if p is self.hs: - foundprevsentinel += 1 - _assert(foundprevsentinel <= 2, "No more than two references to the head sentinel may appear as a prev.", k, v, p, n) - if n is self.ts: - foundnextsentinel += 1 - _assert(foundnextsentinel <= 2, "No more than one reference to the tail sentinel may appear as a next.", k, v, p, n) - _assert(foundprevsentinel == 2, "A reference to the head sentinel is required appear as a prev (plus a self-referential reference).") - _assert(foundnextsentinel == 2, "A reference to the tail sentinel is required appear as a next (plus a self-referential reference).") - - count = 0 - for (k, v,) in self.iteritems(): - _assert(k not in (self.hs, self.ts,), k, self.hs, self.ts) - count += 1 - _assert(count == len(self.d)-2, count, len(self.d)) # -2 for the sentinels - - return True - - def move_to_most_recent(self, k, strictkey=False): - assert self._assert_invariants() - - if not self.d.has_key(k): - if strictkey: - raise KeyError, k - return - - node = self.d[k] - - # relink - self.d[node[1]][2] = node[2] - self.d[node[2]][1] = node[1] - - # move to front - hnode = self.d[self.hs] - - node[1] = self.hs - node[2] = hnode[2] - hnode[2] = k - self.d[node[2]][1] = k - - assert self._assert_invariants() - - def iteritems(self): - return OrderedDict.ItemIterator(self) - - def itervalues(self): - return OrderedDict.ValIterator(self) - - def iterkeys(self): - return self.__iter__() - - def __iter__(self): - return OrderedDict.KeyIterator(self) - - def __getitem__(self, key, default=None, strictkey=True): - node = self.d.get(key) - if not node: - if strictkey: - raise KeyError, key - return default - return node[0] - - def __setitem__(self, k, v=None): - assert self._assert_invariants() - - node = self.d.get(k) - if node: - node[0] = v - return - - hnode = self.d[self.hs] - n = hnode[2] - self.d[k] = [v, self.hs, n,] - hnode[2] = k - self.d[n][1] = k - - assert self._assert_invariants() - return v - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the value removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if self.d.has_key(key): - node = self.d[key] - # relink - self.d[node[1]][2] = node[2] - self.d[node[2]][1] = node[1] - del self.d[key] - assert self._assert_invariants() - return node[0] - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def has_key(self, key): - assert self._assert_invariants() - if self.d.has_key(key): - assert self._assert_invariants() - return True - else: - assert self._assert_invariants() - return False - - def clear(self): - assert self._assert_invariants() - self.d.clear() - self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - - for (k, v,) in otherdict.iteritems(): - assert self._assert_invariants() - self[k] = v - assert self._assert_invariants() - - def pop(self): - assert self._assert_invariants() - if len(self.d) < 2: # the +2 is for the sentinels - raise KeyError, 'popitem(): dictionary is empty' - k = self.d[self.hs][2] - self.remove(k) - assert self._assert_invariants() - return k - - def popitem(self): - assert self._assert_invariants() - if len(self.d) < 2: # the +2 is for the sentinels - raise KeyError, 'popitem(): dictionary is empty' - k = self.d[self.hs][2] - val = self.remove(k) - assert self._assert_invariants() - return (k, val,) - - def keys_unsorted(self): - assert self._assert_invariants() - t = self.d.copy() - del t[self.hs] - del t[self.ts] - assert self._assert_invariants() - return t.keys() - - def keys(self): - res = [None] * len(self) - i = 0 - for k in self.iterkeys(): - res[i] = k - i += 1 - return res - - def values_unsorted(self): - assert self._assert_invariants() - t = self.d.copy() - del t[self.hs] - del t[self.ts] - assert self._assert_invariants() - return map(operator.__getitem__, t.values(), [0]*len(t)) - - def values(self): - res = [None] * len(self) - i = 0 - for v in self.itervalues(): - res[i] = v - i += 1 - return res - - def items(self): - res = [None] * len(self) - i = 0 - for it in self.iteritems(): - res[i] = it - i += 1 - return res - - def __len__(self): - return len(self.d) - 2 - - def insert(self, key, val=None): - assert self._assert_invariants() - result = self.__setitem__(key, val) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def get(self, key, default=None): - return self.__getitem__(key, default, strictkey=False) - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - -class SmallOrderedDict(dict): - """ - SmallOrderedDict is faster than OrderedDict for small sets. How small? That - depends on your machine and which operations you use most often. Use - performance profiling to determine whether the cache class that you are - using makes any difference to the performance of your program, and if it - does, then run "quick_bench()" in test/test_cache.py to see which cache - implementation is faster for the size of your datasets. - - A simple least-recently-used cache. It keeps an LRU queue, and - when the number of items in the cache reaches maxsize, it removes - the least recently used item. - - "Looking" at an item or a key such as with "has_key()" makes that - item become the most recently used item. - - You can also use "refresh()" to explicitly make an item become the most - recently used item. - - Adding an item that is already in the dict *does* make it the - most- recently-used item although it does not change the state of - the dict itself. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return (k, dict.__getitem__(self.c, k),) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return k - - class ValueIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return dict.__getitem__(self.c, k) - - def __init__(self, initialdata={}, maxsize=128): - dict.__init__(self, initialdata) - self._lru = initialdata.keys() # contains keys - self._maxsize = maxsize - over = len(self) - self._maxsize - if over > 0: - map(dict.__delitem__, [self]*over, self._lru[:over]) - del self._lru[:over] - assert self._assert_invariants() - - def _assert_invariants(self): - _assert(len(self._lru) <= self._maxsize, "Size is required to be <= maxsize.") - _assert(len(filter(lambda x: dict.has_key(self, x), self._lru)) == len(self._lru), "Each key in self._lru is required to be in dict.", filter(lambda x: not dict.has_key(self, x), self._lru), len(self._lru), self._lru, len(self), self) - _assert(len(filter(lambda x: x in self._lru, self.keys())) == len(self), "Each key in dict is required to be in self._lru.", filter(lambda x: x not in self._lru, self.keys()), len(self._lru), self._lru, len(self), self) - _assert(len(self._lru) == len(self), "internal consistency", filter(lambda x: x not in self.keys(), self._lru), len(self._lru), self._lru, len(self), self) - _assert(len(self._lru) <= self._maxsize, "internal consistency", len(self._lru), self._lru, self._maxsize) - return True - - def insert(self, key, item=None): - assert self._assert_invariants() - result = self.__setitem__(key, item) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def __setitem__(self, key, item=None): - assert self._assert_invariants() - if dict.has_key(self, key): - self._lru.remove(key) - else: - if len(self._lru) == self._maxsize: - # If this insert is going to increase the size of the cache to bigger than maxsize: - killkey = self._lru.pop(0) - dict.__delitem__(self, killkey) - dict.__setitem__(self, key, item) - self._lru.append(key) - assert self._assert_invariants() - return item - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the object removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if dict.has_key(self, key): - val = dict.__getitem__(self, key) - dict.__delitem__(self, key) - self._lru.remove(key) - assert self._assert_invariants() - return val - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def clear(self): - assert self._assert_invariants() - dict.clear(self) - self._lru = [] - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - if len(otherdict) > self._maxsize: - # Handling this special case here makes it possible to implement the - # other more common cases faster below. - dict.clear(self) - self._lru = [] - if self._maxsize > (len(otherdict) - self._maxsize): - dict.update(self, otherdict) - while len(self) > self._maxsize: - dict.popitem(self) - else: - for k, v, in otherdict.iteritems(): - if len(self) == self._maxsize: - break - dict.__setitem__(self, k, v) - self._lru = dict.keys(self) - assert self._assert_invariants() - return self - - for k in otherdict.iterkeys(): - if dict.has_key(self, k): - self._lru.remove(k) - self._lru.extend(otherdict.keys()) - dict.update(self, otherdict) - - over = len(self) - self._maxsize - if over > 0: - map(dict.__delitem__, [self]*over, self._lru[:over]) - del self._lru[:over] - - assert self._assert_invariants() - return self - - def has_key(self, key): - assert self._assert_invariants() - if dict.has_key(self, key): - assert key in self._lru, "key: %s, self._lru: %s" % tuple(map(hr, (key, self._lru,))) - self._lru.remove(key) - self._lru.append(key) - assert self._assert_invariants() - return True - else: - assert self._assert_invariants() - return False - - def refresh(self, key, strictkey=True): - """ - @param strictkey: raise a KeyError exception if key isn't present - """ - assert self._assert_invariants() - if not dict.has_key(self, key): - if strictkey: - raise KeyError, key - return - self._lru.remove(key) - self._lru.append(key) - - def popitem(self): - if not self._lru: - raise KeyError, 'popitem(): dictionary is empty' - k = self._lru[-1] - obj = self.remove(k) - return (k, obj,) diff --git a/libs/pyutil/randutil.py b/libs/pyutil/randutil.py index a3efb74f..82eb3e18 100644 --- a/libs/pyutil/randutil.py +++ b/libs/pyutil/randutil.py @@ -80,6 +80,5 @@ seed = randobj.seed def randstr(n): return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) -import random as insecurerandom def insecurerandstr(n): - return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n))) + return os.urandom(n) diff --git a/libs/pyutil/randutil.py~ b/libs/pyutil/randutil.py~ deleted file mode 100644 index b0f1c4f9..00000000 --- a/libs/pyutil/randutil.py~ +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn -# This file is part of pyutil; see README.rst for licensing terms. - -import warnings -import os, random - -try: - import hashexpand - class SHA256Random(hashexpand.SHA256Expander, random.Random): - def __init__(self, seed=None, deterministic=True): - warnings.warn("deprecated", DeprecationWarning) - if not deterministic: - raise NotImplementedError, "SHA256Expander is always deterministic. For non-deterministic, try urandomRandom." - - hashexpand.SHA256Expander.__init__(self) - random.Random.__init__(self, seed) - self.seed(seed) - - def seed(self, seed=None): - if seed is None: - import increasing_timer - seed = repr(increasing_timer.time()) - hashexpand.SHA256Expander.seed(self, seed) - - - class SHA256Random(hashexpand.SHA256Expander, random.Random): - def __init__(self, seed=""): - warnings.warn("deprecated", DeprecationWarning) - hashexpand.SHA256Expander.__init__(self) - self.seed(seed) - - def seed(self, seed=None): - if seed is None: - seed = os.urandom(32) - hashexpand.SHA256Expander.seed(self, seed) -except ImportError, le: - class InsecureSHA256Random: - def __init__(self, seed=None): - raise ImportError, le - class SHA256Random: - def __init__(self, seed=""): - raise ImportError, le - -class devrandomRandom(random.Random): - """ The problem with using this one, of course, is that it blocks. This - is, of course, a security flaw. (On Linux and probably on other - systems.) --Zooko 2005-03-04 - - Not repeatable. - """ - def __init__(self): - warnings.warn("deprecated", DeprecationWarning) - self.dr = open("/dev/random", "r") - - def get(self, bytes): - return self.dr.read(bytes) - - -class devurandomRandom(random.Random): - """ The problem with using this one is that it gives answers even when it - has never been properly seeded, e.g. when you are booting from CD and have - just started up and haven't yet gathered enough entropy to actually be - unguessable. (On Linux and probably on other systems.) --Zooko 2005-03-04 - - Not repeatable. - """ - def get(self, bytes): - warnings.warn("deprecated", DeprecationWarning) - return os.urandom(bytes) - - -randobj = devurandomRandom() -get = randobj.get -random = randobj.random -randrange = randobj.randrange -shuffle = randobj.shuffle -choice = randobj.choice -seed = randobj.seed - -def randstr(n): - return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) - -import random as insecurerandom -def insecurerandstr(n): - return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n))) diff --git a/libs/pyutil/scripts/passphrase.py b/libs/pyutil/scripts/passphrase.py new file mode 100644 index 00000000..bed79c13 --- /dev/null +++ b/libs/pyutil/scripts/passphrase.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse, math, random + +from pyutil.mathutil import div_ceil + +from pkg_resources import resource_stream + +def recursive_subset_sum(entropy_needed, wordlists): + # Pick a minimalish set of numbers which sum to at least + # entropy_needed. + + # Okay now what's the smallest number of words which will give us + # at least this much entropy? + entropy_of_biggest_wordlist = wordlists[-1][0] + assert isinstance(entropy_of_biggest_wordlist, float), wordlists[-1] + needed_words = div_ceil(entropy_needed, entropy_of_biggest_wordlist) + # How much entropy do we need from each word? + needed_entropy_per_word = entropy_needed / needed_words + # What's the smallest wordlist that offers at least this much + # entropy per word? + for (wlentropy, wl) in wordlists: + if wlentropy >= needed_entropy_per_word: + break + assert wlentropy >= needed_entropy_per_word, (wlentropy, needed_entropy_per_word) + + result = [(wlentropy, wl)] + # If we need more, recurse... + if wlentropy < entropy_needed: + rest = recursive_subset_sum(entropy_needed - wlentropy, wordlists) + result.extend(rest) + return result + +def gen_passphrase(entropy, allwords): + maxlenwords = [] + i = 2 # The smallest set is words of length 1 or 2. + words = [x for x in allwords if len(x) <= i] + maxlenwords.append((math.log(len(words), 2), words)) + while len(maxlenwords[-1][1]) < len(allwords): + i += 1 + words = [x for x in allwords if len(x) <= i] + maxlenwords.append((math.log(len(words), 2), words)) + + sr = random.SystemRandom() + passphrase = [] + + wordlists_to_use = recursive_subset_sum(entropy, maxlenwords) + + passphraseentropy = 0.0 + for (wle, wl) in wordlists_to_use: + passphrase.append(sr.choice(wl)) + passphraseentropy += wle + + return (u".".join(passphrase), passphraseentropy) + +def main(): + parser = argparse.ArgumentParser(prog="chbs", description="Create a random passphrase by picking a few random words.") + + parser.add_argument('-d', '--dictionary', help="what file to read a list of words from (or omit this option to use chbs's bundled dictionary)", type=argparse.FileType('rU'), metavar="DICT") + parser.add_argument('bits', help="how many bits of entropy minimum", type=float, metavar="BITS") + args = parser.parse_args() + + dicti = args.dictionary + if not dicti: + dicti = resource_stream('pyutil', 'data/wordlist.txt') + allwords = set([x.decode('utf-8').strip().lower() for x in dicti.readlines()]) + + passphrase, bits = gen_passphrase(args.bits, allwords) + + print u"Your new password is: '%s'. It is worth about %s bits." % (passphrase, bits) diff --git a/libs/pyutil/scripts/time_comparisons.py b/libs/pyutil/scripts/time_comparisons.py new file mode 100644 index 00000000..15a38852 --- /dev/null +++ b/libs/pyutil/scripts/time_comparisons.py @@ -0,0 +1,209 @@ +# If you run this file, it will make up a random secret and then crack it +# using timing information from a string comparison function. Maybe--if it +# gets lucky. It takes a long, long time to work. + +# So, the thing I need help with is statistics. The way this thing works is +# extremely stupid. Suppose you want to know which function invocation takes +# longer: comparison(secret, guess1) or comparison(secret, guess2)? + +# If you can correctly determine that one of them takes longer than the +# other, then (a) you can use that to crack the secret, and (b) this is a +# unit test demonstrating that comparison() is not timing-safe. + +# So how does this script do it? Extremely stupidly. First of all, you can't +# reliably measure tiny times, so to measure the time that a function takes, +# we run that function 10,000 times in a row, measure how long that took, and +# divide by 10,000 to estimate how long any one run would have taken. + +# Then, we do that 100 times in a row, and take the fastest of 100 runs. (I +# also experimented with taking the mean of 100 runs instead of the fastest.) + +# Then, we just say whichever comparison took longer (for its fastest run of +# 100 runs of 10,000 executions per run) is the one we think is a closer +# guess to the secret. + +# Now I would *like* to think that there is some kind of statistical analysis +# more sophisticated than "take the slowest of the fastest of 100 runs of +# 10,000 executions". Such improved statistical analysis would hopefully be +# able to answer these two questions: + +# 1. Are these two function calls -- comparison(secret, guess1) and +# comparison(secret, guess2) -- drawing from the same distribution or +# different? If you can answer that question, then you've answered the +# question of whether "comparison" is timing-safe or not. + +# And, this would also allow the cracker to recover from a false step. If it +# incorrectly decides the the prefix of the secret is ABCX, when the real +# secret is ABCD, then after that every next step it takes will be the +# "drawing from the same distribution" kind -- any difference between ABCXQ +# and ABCXR will be just due to noise, since both are equally far from the +# correct answer, which startsw with ABCD. If it could realize that there is +# no real difference between the distributions, then it could back-track and +# recover. + +# 2. Giving the ability to measure, noisily, the time taken by comparison(), +# how can you most efficiently figure out which guess takes the longest? If +# you can do that more efficiently, you can crack secrets more efficiently. + +# The script takes two arguments. The first is how many symbols in the +# secret, and the second is how big the alphabet from which the symbols are +# drawn. To prove that this script can *ever* work, try passing length 5 and +# alphabet size 2. Also try editing the code to let is use sillycomp. That'll +# definitely make it work. If you can improve this script (as per the thing +# above about "needing better statistics") to the degree that it can crack a +# secret with length 32 and alphabet size 256, then that would be awesome. + +# See the result of this commandline: + +# $ python -c 'import time_comparisons ; time_comparisons.print_measurements()' + + +from pyutil import benchutil + +import hashlib, random, os + +from decimal import Decimal +D=Decimal + +p1 = 'a'*32 +p1a = 'a'*32 +p2 = 'a'*31+'b' # close, but no cigar +p3 = 'b'*32 # different in the first byte + +def randstr(n, alphabetsize): + alphabet = [ chr(x) for x in range(alphabetsize) ] + return ''.join([random.choice(alphabet) for i in range(n)]) + +def compare(n, f, a, b): + for i in xrange(n): + f(a, b) + +def eqeqcomp(a, b): + return a == b + +def sillycomp(a, b): + # This exposes a lot of information in its timing about how many leading bytes match. + for i in range(len(a)): + if a[i] != b[i]: + return False + for i in xrange(2**9): + pass + if len(a) == len(b): + return True + else: + return False + +def hashcomp(a, b): + # Brian Warner invented this for Tahoe-LAFS. It seems like it should be very safe agaist timing leakage of any kind, because of the inclusion of a new random randkey every time. Note that exposing the value of the hash (i.e. the output of md5(randkey+secret)) is *not* a security problem. You can post that on your web site and let all attackers have it, no problem. (Provided that the value of "randkey" remains secret.) + + randkey = os.urandom(32) + return hashlib.md5(randkey+ a).digest() == hashlib.md5(randkey+b).digest() + +def xorcomp(a, b): + # This appears to be the most popular timing-insensitive string comparison function. I'm not completely sure it is fully timing-insensitive. (There are all sorts of funny things inside Python, such as caching of integer objects < 100...) + if len(a) != len(b): + return False + result = 0 + for x, y in zip(a, b): + result |= ord(x) ^ ord(y) + return result == 0 + +def print_measurements(): + N=10**4 + REPS=10**2 + + print "all times are in nanoseconds per comparison (in scientific notation)" + print + + for comparator in [eqeqcomp, hashcomp, xorcomp, sillycomp]: + print "using comparator ", comparator + + # for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]: + trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')] + random.shuffle(trials) + for (a, b, desc) in trials: + print "comparing two strings that are %s to each other" % (desc,) + + def f(n): + compare(n, comparator, a, b) + + benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) + + print + +def try_to_crack_secret(cracker, comparator, secretlen, alphabetsize): + secret = randstr(secretlen, alphabetsize) + + def test_guess(x): + return comparator(secret, x) + + print "Giving cracker %s a chance to figure out the secret. Don't tell him, but the secret is %s. Whenever he makes a guess, we'll use comparator %s to decide if his guess is right ..." % (cracker, secret.encode('hex'), comparator,) + + guess = cracker(test_guess, secretlen, alphabetsize) + + print "Cracker %s guessed %r" % (cracker, guess,) + if guess == secret: + print "HE FIGURED IT OUT!? HOW DID HE DO THAT." + else: + print "HAHA. Our secret is safe." + +def byte_at_a_time_cracker(test_guess, secretlen, alphabetsize): + # If we were cleverer, we'd add some backtracking behaviour where, if we can't find any x such that ABCx stands out from the crowd as taking longer than all the other ABCy's, then we start to think that we've taken a wrong step and we go back to trying ABy's. Make sense? But we're not that clever. Once we take a step, we don't backtrack. + + print + + guess=[] + + while len(guess) < secretlen: + best_next_byte = None + best_next_byte_time = None + + # For each possible byte... + for next_byte in range(alphabetsize): + c = chr(next_byte) + + # Construct a guess with our best candidate so far... + candidate_guess = guess[:] + + # Plus that byte... + candidate_guess.append(c) + s = ''.join(candidate_guess) + + # Plus random bytes... + s += os.urandom(32 - len(s)) + + # And see how long it takes the test_guess to consider it... + def f(n): + for i in xrange(n): + test_guess(s) + + times = benchutil.rep_bench(f, 10**7, MAXREPS=10**3, quiet=True) + + fastesttime = times['mean'] + + print "%s..."%(c.encode('hex'),), + if best_next_byte is None or fastesttime > best_next_byte_time: + print "new candidate for slowest next-char: %s, took: %s" % (c.encode('hex'), fastesttime,), + + best_next_byte_time = fastesttime + best_next_byte = c + + # Okay we've tried all possible next bytes. Our guess is this one (the one that took longest to be tested by test_guess): + guess.append(best_next_byte) + print "SLOWEST next-char %s! Current guess at secret: %s" % (best_next_byte.encode('hex'), ''.join(guess).encode('hex'),) + + guess = ''.join(guess) + print "Our guess for the secret: %r" % (guess,) + return guess + +if __name__ == '__main__': + import sys + secretlen = int(sys.argv[1]) + alphabetsize = int(sys.argv[2]) + if alphabetsize > 256: + raise Exception("We assume we can fit one element of the alphabet into a byte.") + + print "secretlen: %d, alphabetsize: %d" % (secretlen, alphabetsize,) + + # try_to_crack_secret(byte_at_a_time_cracker, sillycomp, secretlen, alphabetsize) + try_to_crack_secret(byte_at_a_time_cracker, eqeqcomp, secretlen, alphabetsize) diff --git a/libs/pyutil/test/current/test_mathutil.py b/libs/pyutil/test/current/test_mathutil.py index 7c189dfb..da788758 100644 --- a/libs/pyutil/test/current/test_mathutil.py +++ b/libs/pyutil/test/current/test_mathutil.py @@ -42,6 +42,13 @@ class MathUtilTestCase(unittest.TestCase): self.failUnlessEqual(f(5, 3), 2) self.failUnlessEqual(f(6, 3), 2) self.failUnlessEqual(f(7, 3), 3) + self.failUnless(isinstance(f(0.0, 1), int)) + self.failUnlessEqual(f(7.0, 3.0), 3) + self.failUnlessEqual(f(7, 3.0), 3) + self.failUnlessEqual(f(7.0, 3), 3) + self.failUnlessEqual(f(6.0, 3.0), 2) + self.failUnlessEqual(f(6.0, 3), 2) + self.failUnlessEqual(f(6, 3.0), 2) def test_next_multiple(self): f = mathutil.next_multiple diff --git a/libs/pyutil/test/current/test_mathutil.py~ b/libs/pyutil/test/current/test_mathutil.py~ new file mode 100644 index 00000000..7c189dfb --- /dev/null +++ b/libs/pyutil/test/current/test_mathutil.py~ @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +import unittest + +from pyutil import mathutil +from pyutil.assertutil import _assert + +class MathUtilTestCase(unittest.TestCase): + def _help_test_is_power_of_k(self, k): + for i in range(2, 40): + _assert(mathutil.is_power_of_k(k**i, k), k, i) + + def test_is_power_of_k(self): + for i in range(2, 5): + self._help_test_is_power_of_k(i) + + def test_log_ceil(self): + f = mathutil.log_ceil + self.failUnlessEqual(f(1, 2), 0) + self.failUnlessEqual(f(1, 3), 0) + self.failUnlessEqual(f(2, 2), 1) + self.failUnlessEqual(f(2, 3), 1) + self.failUnlessEqual(f(3, 2), 2) + + def test_log_floor(self): + f = mathutil.log_floor + self.failUnlessEqual(f(1, 2), 0) + self.failUnlessEqual(f(1, 3), 0) + self.failUnlessEqual(f(2, 2), 1) + self.failUnlessEqual(f(2, 3), 0) + self.failUnlessEqual(f(3, 2), 1) + + def test_div_ceil(self): + f = mathutil.div_ceil + self.failUnlessEqual(f(0, 1), 0) + self.failUnlessEqual(f(0, 2), 0) + self.failUnlessEqual(f(0, 3), 0) + self.failUnlessEqual(f(1, 3), 1) + self.failUnlessEqual(f(2, 3), 1) + self.failUnlessEqual(f(3, 3), 1) + self.failUnlessEqual(f(4, 3), 2) + self.failUnlessEqual(f(5, 3), 2) + self.failUnlessEqual(f(6, 3), 2) + self.failUnlessEqual(f(7, 3), 3) + + def test_next_multiple(self): + f = mathutil.next_multiple + self.failUnlessEqual(f(5, 1), 5) + self.failUnlessEqual(f(5, 2), 6) + self.failUnlessEqual(f(5, 3), 6) + self.failUnlessEqual(f(5, 4), 8) + self.failUnlessEqual(f(5, 5), 5) + self.failUnlessEqual(f(5, 6), 6) + self.failUnlessEqual(f(32, 1), 32) + self.failUnlessEqual(f(32, 2), 32) + self.failUnlessEqual(f(32, 3), 33) + self.failUnlessEqual(f(32, 4), 32) + self.failUnlessEqual(f(32, 5), 35) + self.failUnlessEqual(f(32, 6), 36) + self.failUnlessEqual(f(32, 7), 35) + self.failUnlessEqual(f(32, 8), 32) + self.failUnlessEqual(f(32, 9), 36) + self.failUnlessEqual(f(32, 10), 40) + self.failUnlessEqual(f(32, 11), 33) + self.failUnlessEqual(f(32, 12), 36) + self.failUnlessEqual(f(32, 13), 39) + self.failUnlessEqual(f(32, 14), 42) + self.failUnlessEqual(f(32, 15), 45) + self.failUnlessEqual(f(32, 16), 32) + self.failUnlessEqual(f(32, 17), 34) + self.failUnlessEqual(f(32, 18), 36) + self.failUnlessEqual(f(32, 589), 589) + + def test_pad_size(self): + f = mathutil.pad_size + self.failUnlessEqual(f(0, 4), 0) + self.failUnlessEqual(f(1, 4), 3) + self.failUnlessEqual(f(2, 4), 2) + self.failUnlessEqual(f(3, 4), 1) + self.failUnlessEqual(f(4, 4), 0) + self.failUnlessEqual(f(5, 4), 3) + + def test_is_power_of_k_part_2(self): + f = mathutil.is_power_of_k + for i in range(1, 100): + if i in (1, 2, 4, 8, 16, 32, 64): + self.failUnless(f(i, 2), "but %d *is* a power of 2" % i) + else: + self.failIf(f(i, 2), "but %d is *not* a power of 2" % i) + for i in range(1, 100): + if i in (1, 3, 9, 27, 81): + self.failUnless(f(i, 3), "but %d *is* a power of 3" % i) + else: + self.failIf(f(i, 3), "but %d is *not* a power of 3" % i) + + def test_next_power_of_k(self): + f = mathutil.next_power_of_k + self.failUnlessEqual(f(0,2), 1) + self.failUnlessEqual(f(1,2), 1) + self.failUnlessEqual(f(2,2), 2) + self.failUnlessEqual(f(3,2), 4) + self.failUnlessEqual(f(4,2), 4) + for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i) + for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i) + for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i) + for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i) + for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i) + + self.failUnlessEqual(f(0,3), 1) + self.failUnlessEqual(f(1,3), 1) + self.failUnlessEqual(f(2,3), 3) + self.failUnlessEqual(f(3,3), 3) + for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i) + for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i) + for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i) + for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i) + + def test_ave(self): + f = mathutil.ave + self.failUnlessEqual(f([1,2,3]), 2) + self.failUnlessEqual(f([0,0,0,4]), 1) + self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) + + def failUnlessEqualContents(self, a, b): + self.failUnlessEqual(sorted(a), sorted(b)) + + def test_permute(self): + f = mathutil.permute + self.failUnlessEqualContents(f([]), []) + self.failUnlessEqualContents(f([1]), [[1]]) + self.failUnlessEqualContents(f([1,2]), [[1,2], [2,1]]) + self.failUnlessEqualContents(f([1,2,3]), + [[1,2,3], [1,3,2], + [2,1,3], [2,3,1], + [3,1,2], [3,2,1]]) diff --git a/libs/pyutil/time_comparisons.py b/libs/pyutil/time_comparisons.py deleted file mode 100644 index ee1bcfa0..00000000 --- a/libs/pyutil/time_comparisons.py +++ /dev/null @@ -1,44 +0,0 @@ -from pyutil import benchutil - -import hashlib, random, os - -from decimal import Decimal -D=Decimal - -p1 = 'a'*32 -p1a = 'a'*32 -p2 = 'a'*31+'b' # close, but no cigar -p3 = 'b'*32 # different in the first byte - -def compare(n, f, a, b): - for i in xrange(n): - f(a, b) - -def eqeqcomp(a, b): - return a == b - -def hashcomp(a, b): - salt = os.urandom(32) - return hashlib.md5(salt+ a).digest() == hashlib.md5(salt+b).digest() - -N=10**4 -REPS=10**2 - -print "all times are in nanoseconds per comparison (scientific notation)" -print - -for comparator in [eqeqcomp, hashcomp]: - print "using comparator ", comparator - - # for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]: - trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')] - random.shuffle(trials) - for (a, b, desc) in trials: - print "comparing two strings that are %s to each other" % (desc,) - - def f(n): - compare(n, comparator, a, b) - - benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - - print diff --git a/libs/pyutil/time_comparisons.py~ b/libs/pyutil/time_comparisons.py~ deleted file mode 100644 index abf151ad..00000000 --- a/libs/pyutil/time_comparisons.py~ +++ /dev/null @@ -1,72 +0,0 @@ -from pyutil import benchutil - -import hashlib -import os - -from decimal import Decimal -D=Decimal - -p1 = 'a'*32 -p1a = 'a'*32 -p2 = 'a'*31+'b' # close, but no cigar -p3 = 'b'*32 # different in the first byte - -def compare(n, f, a, b): - for i in xrange(n): - f(a, b) - -def eqeq(a, b): - return a == b - -def equalsequals_s(n): - # return compare(n, eqeq, - for i in xrange(n): - p1 == p1a - -def equalsequals_c(n): - for i in xrange(n): - p1 == p2 - -def equalsequals_f(n): - for i in xrange(n): - p1 == p3 - -def hash_s(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p1a).digest() - -def hash_c(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p2).digest() - -def hash_f(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p3).digest() - -N=10**4 -REPS=10**2 - -print "using '=='" - -print "same" -benchutil.rep_bench(equalsequals_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "close" -benchutil.rep_bench(equalsequals_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "far" -benchutil.rep_bench(equalsequals_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "using hash" - -print "same" -benchutil.rep_bench(hash_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "far" -benchutil.rep_bench(hash_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "close" -benchutil.rep_bench(hash_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index 0cbfa7c0..a2cef356 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -549,7 +549,7 @@ class OAuth2Mixin(object): @return_future def authorize_redirect(self, redirect_uri=None, client_id=None, client_secret=None, extra_params=None, - callback=None): + callback=None, scope=None, response_type="code"): """Redirects the user to obtain OAuth authorization for this service. Some providers require that you register a redirect URL with @@ -566,10 +566,13 @@ class OAuth2Mixin(object): """ args = { "redirect_uri": redirect_uri, - "client_id": client_id + "client_id": client_id, + "response_type": response_type } if extra_params: args.update(extra_params) + if scope: + args['scope'] = ' '.join(scope) self.redirect( url_concat(self._OAUTH_AUTHORIZE_URL, args)) callback() @@ -945,6 +948,67 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): return OpenIdMixin.get_authenticated_user(self) +class GoogleOAuth2Mixin(OAuth2Mixin): + """Google authentication using OAuth2.""" + _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth" + _OAUTH_ACCESS_TOKEN_URL = "https://accounts.google.com/o/oauth2/token" + _OAUTH_NO_CALLBACKS = False + _OAUTH_SETTINGS_KEY = 'google_oauth' + + @_auth_return_future + def get_authenticated_user(self, redirect_uri, code, callback): + """Handles the login for the Google user, returning a user object. + + Example usage:: + + class GoogleOAuth2LoginHandler(LoginHandler, tornado.auth.GoogleOAuth2Mixin): + @tornado.web.asynchronous + @tornado.gen.coroutine + def get(self): + if self.get_argument("code", False): + user = yield self.get_authenticated_user( + redirect_uri='http://your.site.com/auth/google', + code=self.get_argument("code")) + # Save the user with e.g. set_secure_cookie + else: + yield self.authorize_redirect( + redirect_uri='http://your.site.com/auth/google', + client_id=self.settings["google_consumer_key"], + scope=['openid', 'email'], + response_type='code', + extra_params={"approval_prompt": "auto"}) + """ + http = self.get_auth_http_client() + body = urllib_parse.urlencode({ + "redirect_uri": redirect_uri, + "code": code, + "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'], + "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'], + "grant_type": "authorization_code", + }) + + http.fetch(self._OAUTH_ACCESS_TOKEN_URL, + self.async_callback(self._on_access_token, callback), + method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body) + + def _on_access_token(self, future, response): + """Callback function for the exchange to the access token.""" + if response.error: + future.set_exception(AuthError('Google auth error: %s' % str(response))) + return + + args = escape.json_decode(response.body) + future.set_result(args) + + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() + + class FacebookMixin(object): """Facebook Connect authentication. diff --git a/libs/tornado/autoreload.py b/libs/tornado/autoreload.py index 05754299..79cccb49 100755 --- a/libs/tornado/autoreload.py +++ b/libs/tornado/autoreload.py @@ -16,11 +16,15 @@ """xAutomatically restart the server when a source file is modified. -Most applications should not access this module directly. Instead, pass the -keyword argument ``debug=True`` to the `tornado.web.Application` constructor. -This will enable autoreload mode as well as checking for changes to templates -and static resources. Note that restarting is a destructive operation -and any requests in progress will be aborted when the process restarts. +Most applications should not access this module directly. Instead, +pass the keyword argument ``autoreload=True`` to the +`tornado.web.Application` constructor (or ``debug=True``, which +enables this setting and several others). This will enable autoreload +mode as well as checking for changes to templates and static +resources. Note that restarting is a destructive operation and any +requests in progress will be aborted when the process restarts. (If +you want to disable autoreload while using other debug-mode features, +pass both ``debug=True`` and ``autoreload=False``). This module can also be used as a command-line wrapper around scripts such as unit test runners. See the `main` method for details. @@ -38,6 +42,7 @@ Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) because it re-executes Python using ``sys.executable`` and ``sys.argv``. Additionally, modifying these variables will cause reloading to behave incorrectly. + """ from __future__ import absolute_import, division, print_function, with_statement diff --git a/libs/tornado/curl_httpclient.py b/libs/tornado/curl_httpclient.py index e0900569..cb97710a 100755 --- a/libs/tornado/curl_httpclient.py +++ b/libs/tornado/curl_httpclient.py @@ -360,6 +360,7 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') + curl.unsetopt(pycurl.PROXYUSERPWD) if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) @@ -382,6 +383,8 @@ def _curl_setup_request(curl, request, buffer, headers): # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) + else: + curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) # Set the request method through curl's irritating interface which makes # up names for almost every single method @@ -404,6 +407,11 @@ def _curl_setup_request(curl, request, buffer, headers): # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): + if request.body is None: + raise AssertionError( + 'Body must not be empty for "%s" request' + % request.method) + request_buffer = BytesIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": @@ -414,6 +422,9 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) + elif request.method == "GET": + if request.body is not None: + raise AssertionError('Body must be empty for GET request') if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') diff --git a/libs/tornado/gen.py b/libs/tornado/gen.py index 92b7458e..217ebdf5 100755 --- a/libs/tornado/gen.py +++ b/libs/tornado/gen.py @@ -38,8 +38,8 @@ since it is both shorter and provides better exception handling):: def get(self): yield gen.Task(AsyncHTTPClient().fetch, "http://example.com") -You can also yield a list of ``Futures`` and/or ``Tasks``, which will be -started at the same time and run in parallel; a list of results will +You can also yield a list or dict of ``Futures`` and/or ``Tasks``, which will be +started at the same time and run in parallel; a list or dict of results will be returned when they are all finished:: @gen.coroutine @@ -47,6 +47,13 @@ be returned when they are all finished:: http_client = AsyncHTTPClient() response1, response2 = yield [http_client.fetch(url1), http_client.fetch(url2)] + response_dict = yield dict(response3=http_client.fetch(url3), + response4=http_client.fetch(url4)) + response3 = response_dict['response3'] + response4 = response_dict['response4'] + +.. versionchanged:: 3.2 + Dict support added. For more complicated interfaces, `Task` can be split into two parts: `Callback` and `Wait`:: @@ -404,6 +411,10 @@ class Multi(YieldPoint): a list of ``YieldPoints``. """ def __init__(self, children): + self.keys = None + if isinstance(children, dict): + self.keys = list(children.keys()) + children = children.values() self.children = [] for i in children: if isinstance(i, Future): @@ -423,7 +434,11 @@ class Multi(YieldPoint): return not self.unfinished_children def get_result(self): - return [i.get_result() for i in self.children] + result = (i.get_result() for i in self.children) + if self.keys is not None: + return dict(zip(self.keys, result)) + else: + return list(result) class _NullYieldPoint(YieldPoint): @@ -523,7 +538,7 @@ class Runner(object): self.finished = True self.yield_point = _null_yield_point raise - if isinstance(yielded, list): + if isinstance(yielded, (list, dict)): yielded = Multi(yielded) elif isinstance(yielded, Future): yielded = YieldFuture(yielded) diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index 67675894..b58a8348 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -282,7 +282,8 @@ class HTTPRequest(object): :arg int max_redirects: Limit for ``follow_redirects`` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server - :arg string network_interface: Network interface to use for request + :arg string network_interface: Network interface to use for request. + ``curl_httpclient`` only; see note below. :arg callable streaming_callback: If set, ``streaming_callback`` will be run with each chunk of data as it is received, and ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in @@ -310,14 +311,26 @@ class HTTPRequest(object): :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, - or None to use defaults. Note that in ``curl_httpclient``, if - any request uses a custom ``ca_certs`` file, they all must (they - don't have to all use the same ``ca_certs``, but it's not possible - to mix requests with ``ca_certs`` and requests that use the defaults. + or None to use defaults. See note below when used with + ``curl_httpclient``. :arg bool allow_ipv6: Use IPv6 when available? Default is false in ``simple_httpclient`` and true in ``curl_httpclient`` - :arg string client_key: Filename for client SSL key, if any - :arg string client_cert: Filename for client SSL certificate, if any + :arg string client_key: Filename for client SSL key, if any. See + note below when used with ``curl_httpclient``. + :arg string client_cert: Filename for client SSL certificate, if any. + See note below when used with ``curl_httpclient``. + + .. note:: + + When using ``curl_httpclient`` certain options may be + inherited by subsequent fetches because ``pycurl`` does + not allow them to be cleanly reset. This applies to the + ``ca_certs``, ``client_key``, ``client_cert``, and + ``network_interface`` arguments. If you use these + options, you should pass them on every request (you don't + have to always use the same values, but it's not possible + to mix requests that specify these options with ones that + use the defaults). .. versionadded:: 3.1 The ``auth_mode`` argument. @@ -372,6 +385,9 @@ class HTTPResponse(object): * headers: `tornado.httputil.HTTPHeaders` object + * effective_url: final location of the resource after following any + redirects + * buffer: ``cStringIO`` object for response body * body: response body as string (created on demand from ``self.buffer``) diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index d005545e..34e7b768 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -29,6 +29,7 @@ from __future__ import absolute_import, division, print_function, with_statement import socket import ssl import time +import copy from tornado.escape import native_str, parse_qs_bytes from tornado import httputil @@ -326,8 +327,8 @@ class HTTPConnection(object): self.request_callback(self._request) except _BadRequestException as e: - gen_log.info("Malformed HTTP request from %s: %s", - self.address[0], e) + gen_log.info("Malformed HTTP request from %r: %s", + self.address, e) self.close() return @@ -336,7 +337,10 @@ class HTTPConnection(object): if self._request.method in ("POST", "PATCH", "PUT"): httputil.parse_body_arguments( self._request.headers.get("Content-Type", ""), data, - self._request.arguments, self._request.files) + self._request.body_arguments, self._request.files) + + for k, v in self._request.body_arguments.items(): + self._request.arguments.setdefault(k, []).extend(v) self.request_callback(self._request) @@ -403,6 +407,20 @@ class HTTPRequest(object): `.RequestHandler.get_argument`, which returns argument values as unicode strings. + .. attribute:: query_arguments + + Same format as ``arguments``, but contains only arguments extracted + from the query string. + + .. versionadded:: 3.2 + + .. attribute:: body_arguments + + Same format as ``arguments``, but contains only arguments extracted + from the request body. + + .. versionadded:: 3.2 + .. attribute:: files File uploads are available in the files property, which maps file @@ -457,6 +475,8 @@ class HTTPRequest(object): self.path, sep, self.query = uri.partition('?') self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) + self.query_arguments = copy.deepcopy(self.arguments) + self.body_arguments = {} def supports_http_1_1(self): """Returns True if this request supports HTTP/1.1 semantics""" diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index 3e7337d9..2575bc56 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -320,7 +320,11 @@ def parse_body_arguments(content_type, body, arguments, files): with the parsed contents. """ if content_type.startswith("application/x-www-form-urlencoded"): - uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + try: + uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + except Exception as e: + gen_log.warning('Invalid x-www-form-urlencoded body: %s', e) + uri_arguments = {} for name, values in uri_arguments.items(): if values: arguments.setdefault(name, []).extend(values) diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 91ee2c5b..a36ab7a5 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -676,8 +676,7 @@ class PollIOLoop(IOLoop): while self._events: fd, events = self._events.popitem() try: - if self._handlers.has_key(fd): - self._handlers[fd](fd, events) + self._handlers[fd](fd, events) except (OSError, IOError) as e: if e.args[0] == errno.EPIPE: # Happens when the client closes the connection diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index 6bdc6397..08430cea 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -774,7 +774,7 @@ class IOStream(BaseIOStream): # Sometimes setsockopt will fail if the socket is closed # at the wrong time. This can happen with HTTPServer # resetting the value to false between requests. - if e.errno != errno.EINVAL: + if e.errno not in (errno.EINVAL, errno.ECONNRESET): raise diff --git a/libs/tornado/log.py b/libs/tornado/log.py index fa11f379..648db5c6 100755 --- a/libs/tornado/log.py +++ b/libs/tornado/log.py @@ -51,7 +51,7 @@ gen_log = logging.getLogger("tornado.general") def _stderr_supports_color(): color = False - if curses and sys.stderr.isatty(): + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): try: curses.setupterm() if curses.tigetnum("colors") > 0: diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index 9dc8506e..21db4755 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -20,7 +20,6 @@ from __future__ import absolute_import, division, print_function, with_statement import errno import os -import re import socket import ssl import stat @@ -30,6 +29,13 @@ from tornado.ioloop import IOLoop from tornado.platform.auto import set_close_exec from tornado.util import Configurable +if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ + ssl_match_hostname = ssl.match_hostname + SSLCertificateError = ssl.CertificateError +else: + import backports.ssl_match_hostname + ssl_match_hostname = backports.ssl_match_hostname.match_hostname + SSLCertificateError = backports.ssl_match_hostname.CertificateError def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): """Creates listening sockets bound to the given port and address. @@ -391,73 +397,3 @@ def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): return context.wrap_socket(socket, **kwargs) else: return ssl.wrap_socket(socket, **dict(context, **kwargs)) - -if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ - ssl_match_hostname = ssl.match_hostname - SSLCertificateError = ssl.CertificateError -else: - # match_hostname was added to the standard library ssl module in python 3.2. - # The following code was backported for older releases and copied from - # https://bitbucket.org/brandon/backports.ssl_match_hostname - class SSLCertificateError(ValueError): - pass - - def _dnsname_to_pat(dn, max_wildcards=1): - pats = [] - for frag in dn.split(r'.'): - if frag.count('*') > max_wildcards: - # Issue #17980: avoid denials of service by refusing more - # than one wildcard per fragment. A survery of established - # policy among SSL implementations showed it to be a - # reasonable choice. - raise SSLCertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) - if frag == '*': - # When '*' is a fragment by itself, it matches a non-empty dotless - # fragment. - pats.append('[^.]+') - else: - # Otherwise, '*' matches any dotless fragment. - frag = re.escape(frag) - pats.append(frag.replace(r'\*', '[^.]*')) - return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - - def ssl_match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules - are mostly followed, but IP addresses are not accepted for *hostname*. - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise SSLCertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise SSLCertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise SSLCertificateError("no appropriate commonName or " - "subjectAltName fields were found") diff --git a/libs/tornado/platform/asyncio.py b/libs/tornado/platform/asyncio.py new file mode 100644 index 00000000..a8f5bad4 --- /dev/null +++ b/libs/tornado/platform/asyncio.py @@ -0,0 +1,134 @@ +"""Bridges between the `asyncio` module and Tornado IOLoop. + +This is a work in progress and interfaces are subject to change. + +To test: +python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOLoop +python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOMainLoop +(the tests log a few warnings with AsyncIOMainLoop because they leave some +unfinished callbacks on the event loop that fail when it resumes) +""" +import asyncio +import datetime +import functools +import os + +from tornado.ioloop import IOLoop +from tornado import stack_context + +class BaseAsyncIOLoop(IOLoop): + def initialize(self, asyncio_loop, close_loop=False): + self.asyncio_loop = asyncio_loop + self.close_loop = close_loop + self.asyncio_loop.call_soon(self.make_current) + # Maps fd to handler function (as in IOLoop.add_handler) + self.handlers = {} + # Set of fds listening for reads/writes + self.readers = set() + self.writers = set() + self.closing = False + + def close(self, all_fds=False): + self.closing = True + for fd in list(self.handlers): + self.remove_handler(fd) + if all_fds: + os.close(fd) + if self.close_loop: + self.asyncio_loop.close() + + def add_handler(self, fd, handler, events): + if fd in self.handlers: + raise ValueError("fd %d added twice" % fd) + self.handlers[fd] = stack_context.wrap(handler) + if events & IOLoop.READ: + self.asyncio_loop.add_reader( + fd, self._handle_events, fd, IOLoop.READ) + self.readers.add(fd) + if events & IOLoop.WRITE: + self.asyncio_loop.add_writer( + fd, self._handle_events, fd, IOLoop.WRITE) + self.writers.add(fd) + + def update_handler(self, fd, events): + if events & IOLoop.READ: + if fd not in self.readers: + self.asyncio_loop.add_reader( + fd, self._handle_events, fd, IOLoop.READ) + self.readers.add(fd) + else: + if fd in self.readers: + self.asyncio_loop.remove_reader(fd) + self.readers.remove(fd) + if events & IOLoop.WRITE: + if fd not in self.writers: + self.asyncio_loop.add_writer( + fd, self._handle_events, fd, IOLoop.WRITE) + self.writers.add(fd) + else: + if fd in self.writers: + self.asyncio_loop.remove_writer(fd) + self.writers.remove(fd) + + def remove_handler(self, fd): + if fd not in self.handlers: + return + if fd in self.readers: + self.asyncio_loop.remove_reader(fd) + self.readers.remove(fd) + if fd in self.writers: + self.asyncio_loop.remove_writer(fd) + self.writers.remove(fd) + del self.handlers[fd] + + def _handle_events(self, fd, events): + self.handlers[fd](fd, events) + + def start(self): + self.asyncio_loop.run_forever() + + def stop(self): + self.asyncio_loop.stop() + + def _run_callback(self, callback, *args, **kwargs): + try: + callback(*args, **kwargs) + except Exception: + self.handle_callback_exception(callback) + + def add_timeout(self, deadline, callback): + if isinstance(deadline, (int, float)): + delay = max(deadline - self.time(), 0) + elif isinstance(deadline, datetime.timedelta): + delay = deadline.total_seconds() + else: + raise TypeError("Unsupported deadline %r", deadline) + return self.asyncio_loop.call_later(delay, self._run_callback, + stack_context.wrap(callback)) + + def remove_timeout(self, timeout): + timeout.cancel() + + def add_callback(self, callback, *args, **kwargs): + if self.closing: + raise RuntimeError("IOLoop is closing") + if kwargs: + self.asyncio_loop.call_soon_threadsafe(functools.partial( + self._run_callback, stack_context.wrap(callback), + *args, **kwargs)) + else: + self.asyncio_loop.call_soon_threadsafe( + self._run_callback, stack_context.wrap(callback), *args) + + add_callback_from_signal = add_callback + + +class AsyncIOMainLoop(BaseAsyncIOLoop): + def initialize(self): + super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), + close_loop=False) + +class AsyncIOLoop(BaseAsyncIOLoop): + def initialize(self): + super(AsyncIOLoop, self).initialize(asyncio.new_event_loop(), + close_loop=True) diff --git a/libs/tornado/process.py b/libs/tornado/process.py index ffd2d29d..942c5c3f 100755 --- a/libs/tornado/process.py +++ b/libs/tornado/process.py @@ -92,7 +92,8 @@ def fork_processes(num_processes, max_restarts=100): between any server code. Note that multiple processes are not compatible with the autoreload - module (or the debug=True option to `tornado.web.Application`). + module (or the ``autoreload=True`` option to `tornado.web.Application` + which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``fork_processes``. diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index d8dbb271..2558ada8 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -72,6 +72,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.max_clients = max_clients self.queue = collections.deque() self.active = {} + self.waiting = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver @@ -89,7 +90,16 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.resolver.close() def fetch_impl(self, request, callback): - self.queue.append((request, callback)) + key = object() + self.queue.append((key, request, callback)) + if not len(self.active) < self.max_clients: + timeout_handle = self.io_loop.add_timeout( + self.io_loop.time() + min(request.connect_timeout, + request.request_timeout), + functools.partial(self._on_timeout, key)) + else: + timeout_handle = None + self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " @@ -99,8 +109,10 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: - request, callback = self.queue.popleft() - key = object() + key, request, callback = self.queue.popleft() + if key not in self.waiting: + continue + self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) @@ -113,6 +125,22 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): del self.active[key] self._process_queue() + def _remove_timeout(self, key): + if key in self.waiting: + request, callback, timeout_handle = self.waiting[key] + if timeout_handle is not None: + self.io_loop.remove_timeout(timeout_handle) + del self.waiting[key] + + def _on_timeout(self, key): + request, callback, timeout_handle = self.waiting[key] + self.queue.remove((key, request, callback)) + timeout_response = HTTPResponse( + request, 599, error=HTTPError(599, "Timeout"), + request_time=self.io_loop.time() - request.start_time) + self.io_loop.add_callback(callback, timeout_response) + del self.waiting[key] + class _HTTPConnection(object): _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) @@ -162,15 +190,18 @@ class _HTTPConnection(object): # so restrict to ipv4 by default. af = socket.AF_INET + timeout = min(self.request.connect_timeout, self.request.request_timeout) + if timeout: + self._timeout = self.io_loop.add_timeout( + self.start_time + timeout, + stack_context.wrap(self._on_timeout)) self.resolver.resolve(host, port, af, callback=self._on_resolve) def _on_resolve(self, addrinfo): + if self.final_callback is None: + # final_callback is cleared if we've hit our timeout + return self.stream = self._create_stream(addrinfo) - timeout = min(self.request.connect_timeout, self.request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - stack_context.wrap(self._on_timeout)) self.stream.set_close_callback(self._on_close) # ipv6 addresses are broken (in self.parsed.hostname) until # 2.7, here is correctly parsed value calculated in __init__ @@ -199,10 +230,10 @@ class _HTTPConnection(object): # the SSL_OP_NO_SSLv2, but that wasn't exposed to python # until 3.2. Python 2.7 adds the ciphers argument, which # can also be used to disable SSLv2. As a last resort - # on python 2.6, we set ssl_version to SSLv3. This is + # on python 2.6, we set ssl_version to TLSv1. This is # more narrow than we'd like since it also breaks - # compatibility with servers configured for TLSv1 only, - # but nearly all servers support SSLv3: + # compatibility with servers configured for SSLv3 only, + # but nearly all servers support both SSLv3 and TLSv1: # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html if sys.version_info >= (2, 7): ssl_options["ciphers"] = "DEFAULT:!SSLv2" @@ -210,7 +241,7 @@ class _HTTPConnection(object): # This is really only necessary for pre-1.0 versions # of openssl, but python 2.6 doesn't expose version # information. - ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 + ssl_options["ssl_version"] = ssl.PROTOCOL_TLSv1 return SSLIOStream(socket.socket(af), io_loop=self.io_loop, @@ -233,6 +264,8 @@ class _HTTPConnection(object): def _on_connect(self): self._remove_timeout() + if self.final_callback is None: + return if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, @@ -269,9 +302,15 @@ class _HTTPConnection(object): self.request.headers["User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: if self.request.method in ("POST", "PATCH", "PUT"): - assert self.request.body is not None + if self.request.body is None: + raise AssertionError( + 'Body must not be empty for "%s" request' + % self.request.method) else: - assert self.request.body is None + if self.request.body is not None: + raise AssertionError( + 'Body must be empty for "%s" request' + % self.request.method) if self.request.body is not None: self.request.headers["Content-Length"] = str(len( self.request.body)) diff --git a/libs/tornado/speedups.c b/libs/tornado/speedups.c new file mode 100644 index 00000000..8a316c58 --- /dev/null +++ b/libs/tornado/speedups.c @@ -0,0 +1,49 @@ +#include <Python.h> + +static PyObject* websocket_mask(PyObject* self, PyObject* args) { + const char* mask; + int mask_len; + const char* data; + int data_len; + int i; + + if (!PyArg_ParseTuple(args, "s#s#", &mask, &mask_len, &data, &data_len)) { + return NULL; + } + + PyObject* result = PyBytes_FromStringAndSize(NULL, data_len); + if (!result) { + return NULL; + } + char* buf = PyBytes_AsString(result); + for (i = 0; i < data_len; i++) { + buf[i] = data[i] ^ mask[i % 4]; + } + + return result; +} + +static PyMethodDef methods[] = { + {"websocket_mask", websocket_mask, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef speedupsmodule = { + PyModuleDef_HEAD_INIT, + "speedups", + NULL, + -1, + methods +}; + +PyMODINIT_FUNC +PyInit_speedups() { + return PyModule_Create(&speedupsmodule); +} +#else // Python 2.x +PyMODINIT_FUNC +initspeedups() { + Py_InitModule("tornado.speedups", methods); +} +#endif diff --git a/libs/tornado/tcpserver.py b/libs/tornado/tcpserver.py index 8473a21a..c0773732 100755 --- a/libs/tornado/tcpserver.py +++ b/libs/tornado/tcpserver.py @@ -180,7 +180,8 @@ class TCPServer(object): between any server code. Note that multiple processes are not compatible with the autoreload - module (or the ``debug=True`` option to `tornado.web.Application`). + module (or the ``autoreload=True`` option to `tornado.web.Application` + which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``TCPServer.start(n)``. """ diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 5f8d6091..b6d7e97e 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -250,7 +250,7 @@ class RequestHandler(object): not self.request.connection.no_keep_alive): conn_header = self.request.headers.get("Connection") if conn_header and (conn_header.lower() == "keep-alive"): - self.set_header("Connection", "Keep-Alive") + self._headers["Connection"] = "Keep-Alive" self._write_buffer = [] self._status_code = 200 self._reason = httputil.responses[200] @@ -348,12 +348,7 @@ class RequestHandler(object): The returned value is always unicode. """ - args = self.get_arguments(name, strip=strip) - if not args: - if default is self._ARG_DEFAULT: - raise MissingArgumentError(name) - return default - return args[-1] + return self._get_argument(name, default, self.request.arguments, strip) def get_arguments(self, name, strip=True): """Returns a list of the arguments with the given name. @@ -362,9 +357,73 @@ class RequestHandler(object): The returned values are always unicode. """ + return self._get_arguments(name, self.request.arguments, strip) + def get_body_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name + from the request body. + + If default is not provided, the argument is considered to be + required, and we raise a `MissingArgumentError` if it is missing. + + If the argument appears in the url more than once, we return the + last value. + + The returned value is always unicode. + + .. versionadded:: 3.2 + """ + return self._get_argument(name, default, self.request.body_arguments, strip) + + def get_body_arguments(self, name, strip=True): + """Returns a list of the body arguments with the given name. + + If the argument is not present, returns an empty list. + + The returned values are always unicode. + + .. versionadded:: 3.2 + """ + return self._get_arguments(name, self.request.body_arguments, strip) + + def get_query_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name + from the request query string. + + If default is not provided, the argument is considered to be + required, and we raise a `MissingArgumentError` if it is missing. + + If the argument appears in the url more than once, we return the + last value. + + The returned value is always unicode. + + .. versionadded:: 3.2 + """ + return self._get_argument(name, default, self.request.query_arguments, strip) + + def get_query_arguments(self, name, strip=True): + """Returns a list of the query arguments with the given name. + + If the argument is not present, returns an empty list. + + The returned values are always unicode. + + .. versionadded:: 3.2 + """ + return self._get_arguments(name, self.request.query_arguments, strip) + + def _get_argument(self, name, default, source, strip=True): + args = self._get_arguments(name, source, strip=strip) + if not args: + if default is self._ARG_DEFAULT: + raise MissingArgumentError(name) + return default + return args[-1] + + def _get_arguments(self, name, source, strip=True): values = [] - for v in self.request.arguments.get(name, []): + for v in source.get(name, []): v = self.decode_argument(v, name=name) if isinstance(v, unicode_type): # Get rid of any weird control chars (unless decoding gave @@ -838,7 +897,7 @@ class RequestHandler(object): else: self.finish(self.get_error_html(status_code, **kwargs)) return - if self.settings.get("debug") and "exc_info" in kwargs: + if self.settings.get("serve_traceback") and "exc_info" in kwargs: # in debug mode, try to send a traceback self.set_header('Content-Type', 'text/plain') for line in traceback.format_exception(*kwargs["exc_info"]): @@ -1318,6 +1377,12 @@ def asynchronous(method): if not self._finished: self.finish() IOLoop.current().add_future(result, future_complete) + # Once we have done this, hide the Future from our + # caller (i.e. RequestHandler._when_complete), which + # would otherwise set up its own callback and + # exception handler (resulting in exceptions being + # logged twice). + return None return result return wrapper @@ -1383,10 +1448,16 @@ class Application(object): or (regexp, request_class) tuples. When we receive requests, we iterate over the list in order and instantiate an instance of the first request class whose regexp matches the request path. + The request class can be specified as either a class object or a + (fully-qualified) name. - Each tuple can contain an optional third element, which should be - a dictionary if it is present. That dictionary is passed as - keyword arguments to the contructor of the handler. This pattern + Each tuple can contain additional elements, which correspond to the + arguments to the `URLSpec` constructor. (Prior to Tornado 3.2, this + only tuples of two or three elements were allowed). + + A dictionary may be passed as the third element of the tuple, + which will be used as keyword arguments to the handler's + constructor and `~RequestHandler.initialize` method. This pattern is used for the `StaticFileHandler` in this example (note that a `StaticFileHandler` can be installed automatically with the static_path setting described below):: @@ -1409,6 +1480,7 @@ class Application(object): and ``/robots.txt`` from the same directory. A custom subclass of `StaticFileHandler` can be specified with the ``static_handler_class`` setting. + """ def __init__(self, handlers=None, default_host="", transforms=None, wsgi=False, **settings): @@ -1447,8 +1519,14 @@ class Application(object): if handlers: self.add_handlers(".*$", handlers) + if self.settings.get('debug'): + self.settings.setdefault('autoreload', True) + self.settings.setdefault('compiled_template_cache', False) + self.settings.setdefault('static_hash_cache', False) + self.settings.setdefault('serve_traceback', True) + # Automatically reload modified modules - if self.settings.get("debug") and not wsgi: + if self.settings.get('autoreload') and not wsgi: from tornado import autoreload autoreload.start() @@ -1493,20 +1571,8 @@ class Application(object): for spec in host_handlers: if isinstance(spec, (tuple, list)): - assert len(spec) in (2, 3) - pattern = spec[0] - handler = spec[1] - - if isinstance(handler, str): - # import the Module and instantiate the class - # Must be a fully qualified name (module.ClassName) - handler = import_object(handler) - - if len(spec) == 3: - kwargs = spec[2] - else: - kwargs = {} - spec = URLSpec(pattern, handler, kwargs) + assert len(spec) in (2, 3, 4) + spec = URLSpec(*spec) handlers.append(spec) if spec.name: if spec.name in self.named_handlers: @@ -1597,14 +1663,23 @@ class Application(object): args = [unquote(s) for s in match.groups()] break if not handler: - handler = ErrorHandler(self, request, status_code=404) + if self.settings.get('default_handler_class'): + handler_class = self.settings['default_handler_class'] + handler_args = self.settings.get( + 'default_handler_args', {}) + else: + handler_class = ErrorHandler + handler_args = dict(status_code=404) + handler = handler_class(self, request, **handler_args) - # In debug mode, re-compile templates and reload static files on every + # If template cache is disabled (usually in the debug mode), + # re-compile templates and reload static files on every # request so you don't need to restart to see changes - if self.settings.get("debug"): + if not self.settings.get("compiled_template_cache", True): with RequestHandler._template_loader_lock: for loader in RequestHandler._template_loaders.values(): loader.reset() + if not self.settings.get('static_hash_cache', True): StaticFileHandler.reset() handler._execute(transforms, *args, **kwargs) @@ -2454,7 +2529,7 @@ class _UIModuleNamespace(object): class URLSpec(object): """Specifies mappings between URLs and handlers.""" - def __init__(self, pattern, handler_class, kwargs=None, name=None): + def __init__(self, pattern, handler, kwargs=None, name=None): """Parameters: * ``pattern``: Regular expression to be matched. Any groups @@ -2475,7 +2550,13 @@ class URLSpec(object): assert len(self.regex.groupindex) in (0, self.regex.groups), \ ("groups in url regexes must either be all named or all " "positional: %r" % self.regex.pattern) - self.handler_class = handler_class + + if isinstance(handler, str): + # import the Module and instantiate the class + # Must be a fully qualified name (module.ClassName) + handler = import_object(handler) + + self.handler_class = handler self.kwargs = kwargs or {} self.name = name self._path, self._group_count = self._find_groups() diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index 676d21bf..8c2f5a64 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -33,7 +33,7 @@ import tornado.web from tornado.concurrent import TracebackFuture from tornado.escape import utf8, native_str -from tornado import httpclient +from tornado import httpclient, httputil from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from tornado.log import gen_log, app_log @@ -52,6 +52,10 @@ class WebSocketError(Exception): class WebSocketClosedError(WebSocketError): + """Raised by operations on a closed connection. + + .. versionadded:: 3.2 + """ pass @@ -163,6 +167,12 @@ class WebSocketHandler(tornado.web.RequestHandler): encoded as json). If the ``binary`` argument is false, the message will be sent as utf8; in binary mode any byte string is allowed. + + If the connection is already closed, raises `WebSocketClosedError`. + + .. versionchanged:: 3.2 + `WebSocketClosedError` was added (previously a closed connection + would raise an `AttributeError`) """ if self.ws_connection is None: raise WebSocketClosedError() @@ -586,7 +596,7 @@ class WebSocketProtocol13(WebSocketProtocol): frame += struct.pack("!BQ", 127 | mask_bit, l) if self.mask_outgoing: mask = os.urandom(4) - data = mask + self._apply_mask(mask, data) + data = mask + _websocket_mask(mask, data) frame += data self.stream.write(frame) @@ -671,21 +681,8 @@ class WebSocketProtocol13(WebSocketProtocol): except StreamClosedError: self._abort() - def _apply_mask(self, mask, data): - mask = array.array("B", mask) - unmasked = array.array("B", data) - for i in xrange(len(data)): - unmasked[i] = unmasked[i] ^ mask[i % 4] - if hasattr(unmasked, 'tobytes'): - # tostring was deprecated in py32. It hasn't been removed, - # but since we turn on deprecation warnings in our tests - # we need to use the right one. - return unmasked.tobytes() - else: - return unmasked.tostring() - def _on_masked_frame_data(self, data): - self._on_frame_data(self._apply_mask(self._frame_mask, data)) + self._on_frame_data(_websocket_mask(self._frame_mask, data)) def _on_frame_data(self, data): if self._frame_opcode_is_control: @@ -771,7 +768,11 @@ class WebSocketProtocol13(WebSocketProtocol): class WebSocketClientConnection(simple_httpclient._HTTPConnection): - """WebSocket client connection.""" + """WebSocket client connection. + + This class should not be instantiated directly; use the + `websocket_connect` function instead. + """ def __init__(self, io_loop, request): self.connect_future = TracebackFuture() self.read_future = None @@ -793,9 +794,19 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.resolver) + def close(self): + """Closes the websocket connection. + + .. versionadded:: 3.2 + """ + if self.protocol is not None: + self.protocol.close() + self.protocol = None + def _on_close(self): self.on_message(None) self.resolver.close() + super(WebSocketClientConnection, self)._on_close() def _on_http_response(self, response): if not self.connect_future.done(): @@ -859,13 +870,54 @@ def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None): Takes a url and returns a Future whose result is a `WebSocketClientConnection`. + + .. versionchanged:: 3.2 + Also accepts ``HTTPRequest`` objects in place of urls. """ if io_loop is None: io_loop = IOLoop.current() - request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) + if isinstance(url, httpclient.HTTPRequest): + assert connect_timeout is None + request = url + # Copy and convert the headers dict/object (see comments in + # AsyncHTTPClient.fetch) + request.headers = httputil.HTTPHeaders(request.headers) + else: + request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) request = httpclient._RequestProxy( request, httpclient.HTTPRequest._DEFAULTS) conn = WebSocketClientConnection(io_loop, request) if callback is not None: io_loop.add_future(conn.connect_future, callback) return conn.connect_future + +def _websocket_mask_python(mask, data): + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length. + Returns a `bytes` object of the same length as `data` with the mask applied + as specified in section 5.3 of RFC 6455. + + This pure-python implementation may be replaced by an optimized version when available. + """ + mask = array.array("B", mask) + unmasked = array.array("B", data) + for i in xrange(len(data)): + unmasked[i] = unmasked[i] ^ mask[i % 4] + if hasattr(unmasked, 'tobytes'): + # tostring was deprecated in py32. It hasn't been removed, + # but since we turn on deprecation warnings in our tests + # we need to use the right one. + return unmasked.tobytes() + else: + return unmasked.tostring() + +if os.environ.get('TORNADO_NO_EXTENSION'): + # This environment variable exists to make it easier to do performance comparisons; + # it's not guaranteed to remain supported in the future. + _websocket_mask = _websocket_mask_python +else: + try: + from tornado.speedups import websocket_mask as _websocket_mask + except ImportError: + _websocket_mask = _websocket_mask_python diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index 5e25a564..8e5ddedb 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -33,6 +33,7 @@ from __future__ import absolute_import, division, print_function, with_statement import sys import time +import copy import tornado from tornado import escape @@ -142,11 +143,14 @@ class HTTPRequest(object): self.path += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", ""))) self.uri = self.path self.arguments = {} + self.query_arguments = {} + self.body_arguments = {} self.query = environ.get("QUERY_STRING", "") if self.query: self.uri += "?" + self.query self.arguments = parse_qs_bytes(native_str(self.query), keep_blank_values=True) + self.query_arguments = copy.deepcopy(self.arguments) self.version = "HTTP/1.1" self.headers = httputil.HTTPHeaders() if environ.get("CONTENT_TYPE"): @@ -171,7 +175,10 @@ class HTTPRequest(object): # Parse request body self.files = {} httputil.parse_body_arguments(self.headers.get("Content-Type", ""), - self.body, self.arguments, self.files) + self.body, self.body_arguments, self.files) + + for k, v in self.body_arguments.items(): + self.arguments.setdefault(k, []).extend(v) self._start_time = time.time() self._finish_time = None
self.processSpaceCharactersNonPre = self.processSpaceCharacters self.startTagHandler = utils.MethodDispatcher([ ("html", self.startTagHtml), - (("base", "basefont", "bgsound", "command", "link", "meta", - "noframes", "script", "style", "title"), + (("base", "basefont", "bgsound", "command", "link", "meta", + "noframes", "script", "style", "title"), self.startTagProcessInHead), ("body", self.startTagBody), ("frameset", self.startTagFrameset), (("address", "article", "aside", "blockquote", "center", "details", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", - "footer", "header", "hgroup", "menu", "nav", "ol", "p", + "footer", "header", "hgroup", "main", "menu", "nav", "ol", "p", "section", "summary", "ul"), - self.startTagCloseP), + self.startTagCloseP), (headingElements, self.startTagHeading), (("pre", "listing"), self.startTagPreListing), ("form", self.startTagForm), (("li", "dd", "dt"), self.startTagListItem), - ("plaintext",self.startTagPlaintext), + ("plaintext", self.startTagPlaintext), ("a", self.startTagA), - (("b", "big", "code", "em", "font", "i", "s", "small", "strike", - "strong", "tt", "u"),self.startTagFormatting), + (("b", "big", "code", "em", "font", "i", "s", "small", "strike", + "strong", "tt", "u"), self.startTagFormatting), ("nobr", self.startTagNobr), ("button", self.startTagButton), (("applet", "marquee", "object"), self.startTagAppletMarqueeObject), @@ -961,21 +912,21 @@ def getPhases(debug): self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([ - ("body",self.endTagBody), - ("html",self.endTagHtml), - (("address", "article", "aside", "blockquote", "center", - "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", - "footer", "header", "hgroup", "listing", "menu", "nav", "ol", "pre", + ("body", self.endTagBody), + ("html", self.endTagHtml), + (("address", "article", "aside", "blockquote", "button", "center", + "details", "dialog", "dir", "div", "dl", "fieldset", "figcaption", "figure", + "footer", "header", "hgroup", "listing", "main", "menu", "nav", "ol", "pre", "section", "summary", "ul"), self.endTagBlock), ("form", self.endTagForm), - ("p",self.endTagP), + ("p", self.endTagP), (("dd", "dt", "li"), self.endTagListItem), (headingElements, self.endTagHeading), (("a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u"), self.endTagFormatting), - (("applet", "marquee", "object"), self.endTagAppletMarqueeObject), + (("applet", "marquee", "object"), self.endTagAppletMarqueeObject), ("br", self.endTagBr), - ]) + ]) self.endTagHandler.default = self.endTagOther def isMatchingFormattingElement(self, node1, node2): @@ -995,14 +946,14 @@ def getPhases(debug): def addFormattingElement(self, token): self.tree.insertElement(token) element = self.tree.openElements[-1] - + matchingElements = [] for node in self.tree.activeFormattingElements[::-1]: if node is Marker: break elif self.isMatchingFormattingElement(node, element): matchingElements.append(node) - + assert len(matchingElements) <= 3 if len(matchingElements) == 3: self.tree.activeFormattingElements.remove(matchingElements[-1]) @@ -1017,7 +968,7 @@ def getPhases(debug): if node.name not in allowed_elements: self.parser.parseError("expected-closing-tag-but-got-eof") break - #Stop parsing + # Stop parsing def processSpaceCharactersDropNewline(self, token): # Sometimes (start of , , and blocks) we @@ -1026,19 +977,19 @@ def getPhases(debug): self.processSpaceCharacters = self.processSpaceCharactersNonPre if (data.startswith("\n") and self.tree.openElements[-1].name in ("pre", "listing", "textarea") - and not self.tree.openElements[-1].hasContent()): + and not self.tree.openElements[-1].hasContent()): data = data[1:] if data: self.tree.reconstructActiveFormattingElements() self.tree.insertText(data) def processCharacters(self, token): - if token["data"] == u"\u0000": - #The tokenizer should always emit null on its own + if token["data"] == "\u0000": + # The tokenizer should always emit null on its own return self.tree.reconstructActiveFormattingElements() self.tree.insertText(token["data"]) - #This must be bad for performance + # This must be bad for performance if (self.parser.framesetOK and any([char not in spaceCharacters for char in token["data"]])): @@ -1054,11 +1005,11 @@ def getPhases(debug): def startTagBody(self, token): self.parser.parseError("unexpected-start-tag", {"name": "body"}) if (len(self.tree.openElements) == 1 - or self.tree.openElements[1].name != "body"): + or self.tree.openElements[1].name != "body"): assert self.parser.innerHTML else: self.parser.framesetOK = False - for attr, value in token["data"].iteritems(): + for attr, value in token["data"].items(): if attr not in self.tree.openElements[1].attributes: self.tree.openElements[1].attributes[attr] = value @@ -1090,7 +1041,7 @@ def getPhases(debug): def startTagForm(self, token): if self.tree.formPointer: - self.parser.parseError(u"unexpected-start-tag", {"name": "form"}) + self.parser.parseError("unexpected-start-tag", {"name": "form"}) else: if self.tree.elementInScope("p", variant="button"): self.endTagP(impliedTagToken("p")) @@ -1100,9 +1051,9 @@ def getPhases(debug): def startTagListItem(self, token): self.parser.framesetOK = False - stopNamesMap = {"li":["li"], - "dt":["dt", "dd"], - "dd":["dt", "dd"]} + stopNamesMap = {"li": ["li"], + "dt": ["dt", "dd"], + "dd": ["dt", "dd"]} stopNames = stopNamesMap[token["name"]] for node in reversed(self.tree.openElements): if node.name in stopNames: @@ -1110,7 +1061,7 @@ def getPhases(debug): impliedTagToken(node.name, "EndTag")) break if (node.nameTuple in specialElements and - node.name not in ("address", "div", "p")): + node.name not in ("address", "div", "p")): break if self.tree.elementInScope("p", variant="button"): @@ -1137,7 +1088,7 @@ def getPhases(debug): afeAElement = self.tree.elementInActiveFormattingElements("a") if afeAElement: self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "a", "endName": "a"}) + {"startName": "a", "endName": "a"}) self.endTagFormatting(impliedTagToken("a")) if afeAElement in self.tree.openElements: self.tree.openElements.remove(afeAElement) @@ -1154,7 +1105,7 @@ def getPhases(debug): self.tree.reconstructActiveFormattingElements() if self.tree.elementInScope("nobr"): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "nobr", "endName": "nobr"}) + {"startName": "nobr", "endName": "nobr"}) self.processEndTag(impliedTagToken("nobr")) # XXX Need tests that trigger the following self.tree.reconstructActiveFormattingElements() @@ -1163,7 +1114,7 @@ def getPhases(debug): def startTagButton(self, token): if self.tree.elementInScope("button"): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "button", "endName": "button"}) + {"startName": "button", "endName": "button"}) self.processEndTag(impliedTagToken("button")) return token else: @@ -1203,8 +1154,8 @@ def getPhases(debug): framesetOK = self.parser.framesetOK self.startTagVoidFormatting(token) if ("type" in token["data"] and - token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): - #input type=hidden doesn't change framesetOK + token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): + # input type=hidden doesn't change framesetOK self.parser.framesetOK = framesetOK def startTagParamSource(self, token): @@ -1223,7 +1174,7 @@ def getPhases(debug): def startTagImage(self, token): # No really... self.parser.parseError("unexpected-start-tag-treated-as", - {"originalName": "image", "newName": "img"}) + {"originalName": "image", "newName": "img"}) self.processStartTag(impliedTagToken("img", "StartTag", attributes=token["data"], selfClosing=token["selfClosing"])) @@ -1243,18 +1194,18 @@ def getPhases(debug): if "prompt" in token["data"]: prompt = token["data"]["prompt"] else: - prompt = u"This is a searchable index. Enter search keywords: " + prompt = "This is a searchable index. Enter search keywords: " self.processCharacters( - {"type":tokenTypes["Characters"], "data":prompt}) + {"type": tokenTypes["Characters"], "data": prompt}) attributes = token["data"].copy() if "action" in attributes: del attributes["action"] if "prompt" in attributes: del attributes["prompt"] attributes["name"] = "isindex" - self.processStartTag(impliedTagToken("input", "StartTag", - attributes = attributes, - selfClosing = + self.processStartTag(impliedTagToken("input", "StartTag", + attributes=attributes, + selfClosing= token["selfClosing"])) self.processEndTag(impliedTagToken("label")) self.processStartTag(impliedTagToken("hr", "StartTag")) @@ -1287,7 +1238,7 @@ def getPhases(debug): if self.parser.phase in (self.parser.phases["inTable"], self.parser.phases["inCaption"], self.parser.phases["inColumnGroup"], - self.parser.phases["inTableBody"], + self.parser.phases["inTableBody"], self.parser.phases["inRow"], self.parser.phases["inCell"]): self.parser.phase = self.parser.phases["inSelectInTable"] @@ -1307,8 +1258,8 @@ def getPhases(debug): self.parser.adjustForeignAttributes(token) token["namespace"] = namespaces["mathml"] self.tree.insertElement(token) - #Need to get the parse error right for the case where the token - #has a namespace not equal to the xmlns attribute + # Need to get the parse error right for the case where the token + # has a namespace not equal to the xmlns attribute if token["selfClosing"]: self.tree.openElements.pop() token["selfClosingAcknowledged"] = True @@ -1319,8 +1270,8 @@ def getPhases(debug): self.parser.adjustForeignAttributes(token) token["namespace"] = namespaces["svg"] self.tree.insertElement(token) - #Need to get the parse error right for the case where the token - #has a namespace not equal to the xmlns attribute + # Need to get the parse error right for the case where the token + # has a namespace not equal to the xmlns attribute if token["selfClosing"]: self.tree.openElements.pop() token["selfClosingAcknowledged"] = True @@ -1362,7 +1313,7 @@ def getPhases(debug): "tbody", "td", "tfoot", "th", "thead", "tr", "body", "html")): - #Not sure this is the correct name for the parse error + # Not sure this is the correct name for the parse error self.parser.parseError( "expected-one-end-tag-but-got-another", {"expectedName": "body", "gotName": node.name}) @@ -1370,20 +1321,20 @@ def getPhases(debug): self.parser.phase = self.parser.phases["afterBody"] def endTagHtml(self, token): - #We repeat the test for the body end tag token being ignored here + # We repeat the test for the body end tag token being ignored here if self.tree.elementInScope("body"): self.endTagBody(impliedTagToken("body")) return token def endTagBlock(self, token): - #Put us back in the right whitespace handling mode + # Put us back in the right whitespace handling mode if token["name"] == "pre": self.processSpaceCharacters = self.processSpaceCharactersNonPre inScope = self.tree.elementInScope(token["name"]) if inScope: self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != token["name"]: - self.parser.parseError("end-tag-too-early", {"name": token["name"]}) + self.parser.parseError("end-tag-too-early", {"name": token["name"]}) if inScope: node = self.tree.openElements.pop() while node.name != token["name"]: @@ -1394,7 +1345,7 @@ def getPhases(debug): self.tree.formPointer = None if node is None or not self.tree.elementInScope(node): self.parser.parseError("unexpected-end-tag", - {"name":"form"}) + {"name": "form"}) else: self.tree.generateImpliedEndTags() if self.tree.openElements[-1] != node: @@ -1410,7 +1361,7 @@ def getPhases(debug): if not self.tree.elementInScope(token["name"], variant=variant): self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) else: - self.tree.generateImpliedEndTags(exclude = token["name"]) + self.tree.generateImpliedEndTags(exclude=token["name"]) if self.tree.openElements[-1].name != token["name"]: self.parser.parseError( "end-tag-too-early", @@ -1436,65 +1387,105 @@ def getPhases(debug): def endTagFormatting(self, token): """The much-feared adoption agency algorithm""" - # http://www.whatwg.org/specs/web-apps/current-work/#adoptionAgency + # http://svn.whatwg.org/webapps/complete.html#adoptionAgency revision 7867 # XXX Better parseError messages appreciated. - name = token["name"] + # Step 1 outerLoopCounter = 0 + + # Step 2 while outerLoopCounter < 8: + + # Step 3 outerLoopCounter += 1 - # Step 1 paragraph 1 + # Step 4: + + # Let the formatting element be the last element in + # the list of active formatting elements that: + # - is between the end of the list and the last scope + # marker in the list, if any, or the start of the list + # otherwise, and + # - has the same tag name as the token. formattingElement = self.tree.elementInActiveFormattingElements( token["name"]) - if (not formattingElement or + if (not formattingElement or (formattingElement in self.tree.openElements and not self.tree.elementInScope(formattingElement.name))): - self.parser.parseError("adoption-agency-1.1", {"name": token["name"]}) + # If there is no such node, then abort these steps + # and instead act as described in the "any other + # end tag" entry below. + self.endTagOther(token) return - # Step 1 paragraph 2 + # Otherwise, if there is such a node, but that node is + # not in the stack of open elements, then this is a + # parse error; remove the element from the list, and + # abort these steps. elif formattingElement not in self.tree.openElements: self.parser.parseError("adoption-agency-1.2", {"name": token["name"]}) self.tree.activeFormattingElements.remove(formattingElement) return - # Step 1 paragraph 3 - if formattingElement != self.tree.openElements[-1]: - self.parser.parseError("adoption-agency-1.3", {"name": token["name"]}) + # Otherwise, if there is such a node, and that node is + # also in the stack of open elements, but the element + # is not in scope, then this is a parse error; ignore + # the token, and abort these steps. + elif not self.tree.elementInScope(formattingElement.name): + self.parser.parseError("adoption-agency-4.4", {"name": token["name"]}) + return - # Step 2 - # Start of the adoption agency algorithm proper + # Otherwise, there is a formatting element and that + # element is in the stack and is in scope. If the + # element is not the current node, this is a parse + # error. In any case, proceed with the algorithm as + # written in the following steps. + else: + if formattingElement != self.tree.openElements[-1]: + self.parser.parseError("adoption-agency-1.3", {"name": token["name"]}) + + # Step 5: + + # Let the furthest block be the topmost node in the + # stack of open elements that is lower in the stack + # than the formatting element, and is an element in + # the special category. There might not be one. afeIndex = self.tree.openElements.index(formattingElement) furthestBlock = None for element in self.tree.openElements[afeIndex:]: if element.nameTuple in specialElements: furthestBlock = element break - # Step 3 + + # Step 6: + + # If there is no furthest block, then the UA must + # first pop all the nodes from the bottom of the stack + # of open elements, from the current node up to and + # including the formatting element, then remove the + # formatting element from the list of active + # formatting elements, and finally abort these steps. if furthestBlock is None: element = self.tree.openElements.pop() while element != formattingElement: element = self.tree.openElements.pop() self.tree.activeFormattingElements.remove(element) return - commonAncestor = self.tree.openElements[afeIndex-1] - # Step 5 - #if furthestBlock.parent: - # furthestBlock.parent.removeChild(furthestBlock) + # Step 7 + commonAncestor = self.tree.openElements[afeIndex - 1] - # Step 5 + # Step 8: # The bookmark is supposed to help us identify where to reinsert - # nodes in step 12. We have to ensure that we reinsert nodes after + # nodes in step 15. We have to ensure that we reinsert nodes after # the node before the active formatting element. Note the bookmark - # can move in step 7.4 + # can move in step 9.7 bookmark = self.tree.activeFormattingElements.index(formattingElement) - # Step 6 + # Step 9 lastNode = node = furthestBlock innerLoopCounter = 0 - + index = self.tree.openElements.index(node) while innerLoopCounter < 3: innerLoopCounter += 1 @@ -1504,15 +1495,13 @@ def getPhases(debug): if node not in self.tree.activeFormattingElements: self.tree.openElements.remove(node) continue - # Step 6.3 + # Step 9.6 if node == formattingElement: break - # Step 6.4 + # Step 9.7 if lastNode == furthestBlock: - bookmark = (self.tree.activeFormattingElements.index(node) - + 1) - # Step 6.5 - #cite = node.parent + bookmark = self.tree.activeFormattingElements.index(node) + 1 + # Step 9.8 clone = node.cloneNode() # Replace node with clone self.tree.activeFormattingElements[ @@ -1520,20 +1509,18 @@ def getPhases(debug): self.tree.openElements[ self.tree.openElements.index(node)] = clone node = clone - - # Step 6.6 + # Step 9.9 # Remove lastNode from its parents, if any if lastNode.parent: lastNode.parent.removeChild(lastNode) node.appendChild(lastNode) - # Step 7.7 + # Step 9.10 lastNode = node - # End of inner loop - # Step 7 + # Step 10 # Foster parent lastNode if commonAncestor is a - # table, tbody, tfoot, thead, or tr we need to foster parent the - # lastNode + # table, tbody, tfoot, thead, or tr we need to foster + # parent the lastNode if lastNode.parent: lastNode.parent.removeChild(lastNode) @@ -1543,23 +1530,23 @@ def getPhases(debug): else: commonAncestor.appendChild(lastNode) - # Step 8 + # Step 11 clone = formattingElement.cloneNode() - # Step 9 + # Step 12 furthestBlock.reparentChildren(clone) - # Step 10 + # Step 13 furthestBlock.appendChild(clone) - # Step 11 + # Step 14 self.tree.activeFormattingElements.remove(formattingElement) self.tree.activeFormattingElements.insert(bookmark, clone) - # Step 12 + # Step 15 self.tree.openElements.remove(formattingElement) self.tree.openElements.insert( - self.tree.openElements.index(furthestBlock) + 1, clone) + self.tree.openElements.index(furthestBlock) + 1, clone) def endTagAppletMarqueeObject(self, token): if self.tree.elementInScope(token["name"]): @@ -1575,7 +1562,7 @@ def getPhases(debug): def endTagBr(self, token): self.parser.parseError("unexpected-end-tag-treated-as", - {"originalName": "br", "newName": "br element"}) + {"originalName": "br", "newName": "br element"}) self.tree.reconstructActiveFormattingElements() self.tree.insertElement(impliedTagToken("br", "StartTag")) self.tree.openElements.pop() @@ -1600,31 +1587,31 @@ def getPhases(debug): self.startTagHandler = utils.MethodDispatcher([]) self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([ - ("script", self.endTagScript)]) + ("script", self.endTagScript)]) self.endTagHandler.default = self.endTagOther def processCharacters(self, token): self.tree.insertText(token["data"]) def processEOF(self): - self.parser.parseError("expected-named-closing-tag-but-got-eof", - self.tree.openElements[-1].name) + self.parser.parseError("expected-named-closing-tag-but-got-eof", + {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() self.parser.phase = self.parser.originalPhase return True def startTagOther(self, token): - assert False, "Tried to process start tag %s in RCDATA/RAWTEXT mode"%token['name'] + assert False, "Tried to process start tag %s in RCDATA/RAWTEXT mode" % token['name'] def endTagScript(self, token): node = self.tree.openElements.pop() assert node.name == "script" self.parser.phase = self.parser.originalPhase - #The rest of this method is all stuff that only happens if - #document.write works + # The rest of this method is all stuff that only happens if + # document.write works def endTagOther(self, token): - node = self.tree.openElements.pop() + self.tree.openElements.pop() self.parser.phase = self.parser.originalPhase class InTablePhase(Phase): @@ -1656,7 +1643,7 @@ def getPhases(debug): def clearStackToTableContext(self): # "clear the stack back to a table context" while self.tree.openElements[-1].name not in ("table", "html"): - #self.parser.parseError("unexpected-implied-end-tag-in-table", + # self.parser.parseError("unexpected-implied-end-tag-in-table", # {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() # When the current node is <html> it's an innerHTML case @@ -1667,7 +1654,7 @@ def getPhases(debug): self.parser.parseError("eof-in-table") else: assert self.parser.innerHTML - #Stop parsing + # Stop parsing def processSpaceCharacters(self, token): originalPhase = self.parser.phase @@ -1682,7 +1669,7 @@ def getPhases(debug): self.parser.phase.processCharacters(token) def insertText(self, token): - #If we get here there must be at least one non-whitespace character + # If we get here there must be at least one non-whitespace character # Do the table magic! self.tree.insertFromTable = True self.parser.phases["inBody"].processCharacters(token) @@ -1714,7 +1701,7 @@ def getPhases(debug): def startTagTable(self, token): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "table", "endName": "table"}) + {"startName": "table", "endName": "table"}) self.parser.phase.processEndTag(impliedTagToken("table")) if not self.parser.innerHTML: return token @@ -1723,8 +1710,8 @@ def getPhases(debug): return self.parser.phases["inHead"].processStartTag(token) def startTagInput(self, token): - if ("type" in token["data"] and - token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): + if ("type" in token["data"] and + token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): self.parser.parseError("unexpected-hidden-input-in-table") self.tree.insertElement(token) # XXX associate with form @@ -1751,8 +1738,8 @@ def getPhases(debug): self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != "table": self.parser.parseError("end-tag-too-early-named", - {"gotName": "table", - "expectedName": self.tree.openElements[-1].name}) + {"gotName": "table", + "expectedName": self.tree.openElements[-1].name}) while self.tree.openElements[-1].name != "table": self.tree.openElements.pop() self.tree.openElements.pop() @@ -1781,7 +1768,7 @@ def getPhases(debug): def flushCharacters(self): data = "".join([item["data"] for item in self.characterTokens]) if any([item not in spaceCharacters for item in data]): - token = {"type":tokenTypes["Characters"], "data":data} + token = {"type": tokenTypes["Characters"], "data": data} self.parser.phases["inTable"].insertText(token) elif data: self.tree.insertText(data) @@ -1798,12 +1785,12 @@ def getPhases(debug): return True def processCharacters(self, token): - if token["data"] == u"\u0000": + if token["data"] == "\u0000": return self.characterTokens.append(token) def processSpaceCharacters(self, token): - #pretty sure we should never reach here + # pretty sure we should never reach here self.characterTokens.append(token) # assert False @@ -1817,7 +1804,6 @@ def getPhases(debug): self.parser.phase = self.originalPhase return token - class InCaptionPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-caption def __init__(self, parser, tree): @@ -1849,7 +1835,7 @@ def getPhases(debug): def startTagTableElement(self, token): self.parser.parseError() - #XXX Have to duplicate logic here to find out if the tag is ignored + # XXX Have to duplicate logic here to find out if the tag is ignored ignoreEndTag = self.ignoreEndTagCaption() self.parser.phase.processEndTag(impliedTagToken("caption")) if not ignoreEndTag: @@ -1864,8 +1850,8 @@ def getPhases(debug): self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != "caption": self.parser.parseError("expected-one-end-tag-but-got-another", - {"gotName": "caption", - "expectedName": self.tree.openElements[-1].name}) + {"gotName": "caption", + "expectedName": self.tree.openElements[-1].name}) while self.tree.openElements[-1].name != "caption": self.tree.openElements.pop() self.tree.openElements.pop() @@ -1889,7 +1875,6 @@ def getPhases(debug): def endTagOther(self, token): return self.parser.phases["inBody"].processEndTag(token) - class InColumnGroupPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-column @@ -1955,7 +1940,6 @@ def getPhases(debug): if not ignoreEndTag: return token - class InTableBodyPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-table0 def __init__(self, parser, tree): @@ -1980,8 +1964,8 @@ def getPhases(debug): # helper methods def clearStackToTableBodyContext(self): while self.tree.openElements[-1].name not in ("tbody", "tfoot", - "thead", "html"): - #self.parser.parseError("unexpected-implied-end-tag-in-table", + "thead", "html"): + # self.parser.parseError("unexpected-implied-end-tag-in-table", # {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() if self.tree.openElements[-1].name == "html": @@ -2003,7 +1987,7 @@ def getPhases(debug): self.parser.phase = self.parser.phases["inRow"] def startTagTableCell(self, token): - self.parser.parseError("unexpected-cell-in-table-body", + self.parser.parseError("unexpected-cell-in-table-body", {"name": token["name"]}) self.startTagTr(impliedTagToken("tr", "StartTag")) return token @@ -2012,7 +1996,7 @@ def getPhases(debug): # XXX AT Any ideas on how to share this with endTagTable? if (self.tree.elementInScope("tbody", variant="table") or self.tree.elementInScope("thead", variant="table") or - self.tree.elementInScope("tfoot", variant="table")): + self.tree.elementInScope("tfoot", variant="table")): self.clearStackToTableBodyContext() self.endTagTableRowGroup( impliedTagToken(self.tree.openElements[-1].name)) @@ -2032,12 +2016,12 @@ def getPhases(debug): self.parser.phase = self.parser.phases["inTable"] else: self.parser.parseError("unexpected-end-tag-in-table-body", - {"name": token["name"]}) + {"name": token["name"]}) def endTagTable(self, token): if (self.tree.elementInScope("tbody", variant="table") or self.tree.elementInScope("thead", variant="table") or - self.tree.elementInScope("tfoot", variant="table")): + self.tree.elementInScope("tfoot", variant="table")): self.clearStackToTableBodyContext() self.endTagTableRowGroup( impliedTagToken(self.tree.openElements[-1].name)) @@ -2049,12 +2033,11 @@ def getPhases(debug): def endTagIgnore(self, token): self.parser.parseError("unexpected-end-tag-in-table-body", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOther(self, token): return self.parser.phases["inTable"].processEndTag(token) - class InRowPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-row def __init__(self, parser, tree): @@ -2072,7 +2055,7 @@ def getPhases(debug): ("table", self.endTagTable), (("tbody", "tfoot", "thead"), self.endTagTableRowGroup), (("body", "caption", "col", "colgroup", "html", "td", "th"), - self.endTagIgnore) + self.endTagIgnore) ]) self.endTagHandler.default = self.endTagOther @@ -2080,7 +2063,7 @@ def getPhases(debug): def clearStackToTableRowContext(self): while self.tree.openElements[-1].name not in ("tr", "html"): self.parser.parseError("unexpected-implied-end-tag-in-table-row", - {"name": self.tree.openElements[-1].name}) + {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() def ignoreEndTagTr(self): @@ -2091,7 +2074,7 @@ def getPhases(debug): self.parser.phases["inTable"].processEOF() def processSpaceCharacters(self, token): - return self.parser.phases["inTable"].processSpaceCharacters(token) + return self.parser.phases["inTable"].processSpaceCharacters(token) def processCharacters(self, token): return self.parser.phases["inTable"].processCharacters(token) @@ -2139,7 +2122,7 @@ def getPhases(debug): def endTagIgnore(self, token): self.parser.parseError("unexpected-end-tag-in-table-row", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOther(self, token): return self.parser.phases["inTable"].processEndTag(token) @@ -2178,7 +2161,7 @@ def getPhases(debug): def startTagTableOther(self, token): if (self.tree.elementInScope("td", variant="table") or - self.tree.elementInScope("th", variant="table")): + self.tree.elementInScope("th", variant="table")): self.closeCell() return token else: @@ -2194,7 +2177,7 @@ def getPhases(debug): self.tree.generateImpliedEndTags(token["name"]) if self.tree.openElements[-1].name != token["name"]: self.parser.parseError("unexpected-cell-end-tag", - {"name": token["name"]}) + {"name": token["name"]}) while True: node = self.tree.openElements.pop() if node.name == token["name"]: @@ -2249,7 +2232,7 @@ def getPhases(debug): assert self.parser.innerHTML def processCharacters(self, token): - if token["data"] == u"\u0000": + if token["data"] == "\u0000": return self.tree.insertText(token["data"]) @@ -2283,19 +2266,19 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-in-select", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOption(self, token): if self.tree.openElements[-1].name == "option": self.tree.openElements.pop() else: self.parser.parseError("unexpected-end-tag-in-select", - {"name": "option"}) + {"name": "option"}) def endTagOptgroup(self, token): # </optgroup> implicitly closes <option> if (self.tree.openElements[-1].name == "option" and - self.tree.openElements[-2].name == "optgroup"): + self.tree.openElements[-2].name == "optgroup"): self.tree.openElements.pop() # It also closes </optgroup> if self.tree.openElements[-1].name == "optgroup": @@ -2303,7 +2286,7 @@ def getPhases(debug): # But nothing else else: self.parser.parseError("unexpected-end-tag-in-select", - {"name": "optgroup"}) + {"name": "optgroup"}) def endTagSelect(self, token): if self.tree.elementInScope("select", variant="select"): @@ -2318,8 +2301,7 @@ def getPhases(debug): def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-in-select", - {"name": token["name"]}) - + {"name": token["name"]}) class InSelectInTablePhase(Phase): def __init__(self, parser, tree): @@ -2360,64 +2342,64 @@ def getPhases(debug): def endTagOther(self, token): return self.parser.phases["inSelect"].processEndTag(token) - class InForeignContentPhase(Phase): - breakoutElements = frozenset(["b", "big", "blockquote", "body", "br", + breakoutElements = frozenset(["b", "big", "blockquote", "body", "br", "center", "code", "dd", "div", "dl", "dt", - "em", "embed", "h1", "h2", "h3", + "em", "embed", "h1", "h2", "h3", "h4", "h5", "h6", "head", "hr", "i", "img", - "li", "listing", "menu", "meta", "nobr", - "ol", "p", "pre", "ruby", "s", "small", - "span", "strong", "strike", "sub", "sup", + "li", "listing", "menu", "meta", "nobr", + "ol", "p", "pre", "ruby", "s", "small", + "span", "strong", "strike", "sub", "sup", "table", "tt", "u", "ul", "var"]) + def __init__(self, parser, tree): Phase.__init__(self, parser, tree) def adjustSVGTagNames(self, token): - replacements = {u"altglyph":u"altGlyph", - u"altglyphdef":u"altGlyphDef", - u"altglyphitem":u"altGlyphItem", - u"animatecolor":u"animateColor", - u"animatemotion":u"animateMotion", - u"animatetransform":u"animateTransform", - u"clippath":u"clipPath", - u"feblend":u"feBlend", - u"fecolormatrix":u"feColorMatrix", - u"fecomponenttransfer":u"feComponentTransfer", - u"fecomposite":u"feComposite", - u"feconvolvematrix":u"feConvolveMatrix", - u"fediffuselighting":u"feDiffuseLighting", - u"fedisplacementmap":u"feDisplacementMap", - u"fedistantlight":u"feDistantLight", - u"feflood":u"feFlood", - u"fefunca":u"feFuncA", - u"fefuncb":u"feFuncB", - u"fefuncg":u"feFuncG", - u"fefuncr":u"feFuncR", - u"fegaussianblur":u"feGaussianBlur", - u"feimage":u"feImage", - u"femerge":u"feMerge", - u"femergenode":u"feMergeNode", - u"femorphology":u"feMorphology", - u"feoffset":u"feOffset", - u"fepointlight":u"fePointLight", - u"fespecularlighting":u"feSpecularLighting", - u"fespotlight":u"feSpotLight", - u"fetile":u"feTile", - u"feturbulence":u"feTurbulence", - u"foreignobject":u"foreignObject", - u"glyphref":u"glyphRef", - u"lineargradient":u"linearGradient", - u"radialgradient":u"radialGradient", - u"textpath":u"textPath"} + replacements = {"altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath"} if token["name"] in replacements: token["name"] = replacements[token["name"]] def processCharacters(self, token): - if token["data"] == u"\u0000": - token["data"] = u"\uFFFD" - elif (self.parser.framesetOK and + if token["data"] == "\u0000": + token["data"] = "\uFFFD" + elif (self.parser.framesetOK and any(char not in spaceCharacters for char in token["data"])): self.parser.framesetOK = False Phase.processCharacters(self, token) @@ -2428,9 +2410,9 @@ def getPhases(debug): (token["name"] == "font" and set(token["data"].keys()) & set(["color", "face", "size"]))): self.parser.parseError("unexpected-html-element-in-foreign-content", - token["name"]) + {"name": token["name"]}) while (self.tree.openElements[-1].namespace != - self.tree.defaultNamespace and + self.tree.defaultNamespace and not self.parser.isHTMLIntegrationPoint(self.tree.openElements[-1]) and not self.parser.isMathMLTextIntegrationPoint(self.tree.openElements[-1])): self.tree.openElements.pop() @@ -2453,11 +2435,11 @@ def getPhases(debug): nodeIndex = len(self.tree.openElements) - 1 node = self.tree.openElements[-1] if node.name != token["name"]: - self.parser.parseError("unexpected-end-tag", token["name"]) + self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) while True: if node.name.translate(asciiUpper2Lower) == token["name"]: - #XXX this isn't in the spec but it seems necessary + # XXX this isn't in the spec but it seems necessary if self.parser.phase == self.parser.phases["inTableText"]: self.parser.phase.flushCharacters() self.parser.phase = self.parser.phase.originalPhase @@ -2475,21 +2457,20 @@ def getPhases(debug): break return new_token - class AfterBodyPhase(Phase): def __init__(self, parser, tree): Phase.__init__(self, parser, tree) self.startTagHandler = utils.MethodDispatcher([ - ("html", self.startTagHtml) - ]) + ("html", self.startTagHtml) + ]) self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([("html", self.endTagHtml)]) self.endTagHandler.default = self.endTagOther def processEOF(self): - #Stop parsing + # Stop parsing pass def processComment(self, token): @@ -2507,11 +2488,11 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-after-body", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token - def endTagHtml(self,name): + def endTagHtml(self, name): if self.parser.innerHTML: self.parser.parseError("unexpected-end-tag-after-body-innerhtml") else: @@ -2519,7 +2500,7 @@ def getPhases(debug): def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-after-body", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token @@ -2562,7 +2543,7 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-in-frameset", - {"name": token["name"]}) + {"name": token["name"]}) def endTagFrameset(self, token): if self.tree.openElements[-1].name == "html": @@ -2571,15 +2552,14 @@ def getPhases(debug): else: self.tree.openElements.pop() if (not self.parser.innerHTML and - self.tree.openElements[-1].name != "frameset"): + self.tree.openElements[-1].name != "frameset"): # If we're not in innerHTML mode and the the current node is not a # "frameset" element (anymore) then switch. self.parser.phase = self.parser.phases["afterFrameset"] def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-in-frameset", - {"name": token["name"]}) - + {"name": token["name"]}) class AfterFramesetPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#after3 @@ -2598,7 +2578,7 @@ def getPhases(debug): self.endTagHandler.default = self.endTagOther def processEOF(self): - #Stop parsing + # Stop parsing pass def processCharacters(self, token): @@ -2609,15 +2589,14 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-after-frameset", - {"name": token["name"]}) + {"name": token["name"]}) def endTagHtml(self, token): self.parser.phase = self.parser.phases["afterAfterFrameset"] def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-after-frameset", - {"name": token["name"]}) - + {"name": token["name"]}) class AfterAfterBodyPhase(Phase): def __init__(self, parser, tree): @@ -2647,13 +2626,13 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("expected-eof-but-got-start-tag", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token def processEndTag(self, token): self.parser.parseError("expected-eof-but-got-end-tag", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token @@ -2687,12 +2666,11 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("expected-eof-but-got-start-tag", - {"name": token["name"]}) + {"name": token["name"]}) def processEndTag(self, token): self.parser.parseError("expected-eof-but-got-end-tag", - {"name": token["name"]}) - + {"name": token["name"]}) return { "initial": InitialPhase, @@ -2719,14 +2697,16 @@ def getPhases(debug): "afterAfterBody": AfterAfterBodyPhase, "afterAfterFrameset": AfterAfterFramesetPhase, # XXX after after frameset - } + } -def impliedTagToken(name, type="EndTag", attributes = None, - selfClosing = False): + +def impliedTagToken(name, type="EndTag", attributes=None, + selfClosing=False): if attributes is None: attributes = {} - return {"type":tokenTypes[type], "name":unicode(name), "data":attributes, - "selfClosing":selfClosing} + return {"type": tokenTypes[type], "name": name, "data": attributes, + "selfClosing": selfClosing} + class ParseError(Exception): """Error in parsed document""" diff --git a/libs/html5lib/ihatexml.py b/libs/html5lib/ihatexml.py index dd785639..0fc79308 100644 --- a/libs/html5lib/ihatexml.py +++ b/libs/html5lib/ihatexml.py @@ -1,25 +1,105 @@ -import re +from __future__ import absolute_import, division, unicode_literals -baseChar = """[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | [#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | [#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | [#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | [#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | [#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | [#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | [#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | [#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | [#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | [#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | [#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | [#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | [#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | [#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | [#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | [#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | [#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | [#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | [#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | [#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | [#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | [#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | [#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | [#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | [#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | [#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | [#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | [#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | #x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | #x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | #x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | [#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | [#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | #x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | [#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | [#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | [#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | [#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | [#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | #x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | [#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | [#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | [#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | [#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]""" +import re +import warnings + +from .constants import DataLossWarning + +baseChar = """ +[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | +[#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | +[#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | +[#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | +[#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | +[#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | +[#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | +[#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | +[#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | +[#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | +[#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | +[#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | +[#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | +[#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | +[#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | +[#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | +[#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | +[#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | +[#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | +[#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | +[#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | +[#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | +[#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | +[#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | +[#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | +[#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | +[#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | +[#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | +[#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | +[#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | +#x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | +#x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | +#x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | +[#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | +[#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | +#x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | +[#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | +[#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | +[#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | +[#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | +[#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | +#x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | +[#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | +[#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | +[#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | +[#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]""" ideographic = """[#x4E00-#x9FA5] | #x3007 | [#x3021-#x3029]""" -combiningCharacter = """[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | [#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | [#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | [#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | #x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | [#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | [#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | #x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | [#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | [#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | #x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | [#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | [#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | [#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | [#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | [#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | #x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | [#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | #x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | [#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | [#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | #x3099 | #x309A""" +combiningCharacter = """ +[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | +[#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | +[#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | +[#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | +#x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | +[#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | +[#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | +#x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | +[#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | +[#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | +#x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | +[#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | +[#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | +[#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | +[#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | +[#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | +#x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | +[#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | +#x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | +[#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | +[#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | +#x3099 | #x309A""" -digit = """[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | [#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | [#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | [#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]""" +digit = """ +[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | +[#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | +[#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | +[#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]""" -extender = """#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | [#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]""" +extender = """ +#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | +#[#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]""" letter = " | ".join([baseChar, ideographic]) -#Without the -name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, - extender]) +# Without the +name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, + extender]) nameFirst = " | ".join([letter, "_"]) reChar = re.compile(r"#x([\d|A-F]{4,4})") reCharRange = re.compile(r"\[#x([\d|A-F]{4,4})-#x([\d|A-F]{4,4})\]") + def charStringToList(chars): charRanges = [item.strip() for item in chars.split(" | ")] rv = [] @@ -30,16 +110,17 @@ def charStringToList(chars): if match is not None: rv.append([hexToInt(item) for item in match.groups()]) if len(rv[-1]) == 1: - rv[-1] = rv[-1]*2 + rv[-1] = rv[-1] * 2 foundMatch = True break if not foundMatch: assert len(item) == 1 - + rv.append([ord(item)] * 2) rv = normaliseCharList(rv) return rv + def normaliseCharList(charList): charList = sorted(charList) for item in charList: @@ -49,61 +130,69 @@ def normaliseCharList(charList): while i < len(charList): j = 1 rv.append(charList[i]) - while i + j < len(charList) and charList[i+j][0] <= rv[-1][1] + 1: - rv[-1][1] = charList[i+j][1] + while i + j < len(charList) and charList[i + j][0] <= rv[-1][1] + 1: + rv[-1][1] = charList[i + j][1] j += 1 i += j return rv -#We don't really support characters above the BMP :( +# We don't really support characters above the BMP :( max_unicode = int("FFFF", 16) - + + def missingRanges(charList): rv = [] if charList[0] != 0: rv.append([0, charList[0][0] - 1]) for i, item in enumerate(charList[:-1]): - rv.append([item[1]+1, charList[i+1][0] - 1]) + rv.append([item[1] + 1, charList[i + 1][0] - 1]) if charList[-1][1] != max_unicode: rv.append([charList[-1][1] + 1, max_unicode]) return rv + def listToRegexpStr(charList): rv = [] for item in charList: if item[0] == item[1]: - rv.append(escapeRegexp(unichr(item[0]))) + rv.append(escapeRegexp(chr(item[0]))) else: - rv.append(escapeRegexp(unichr(item[0])) + "-" + - escapeRegexp(unichr(item[1]))) - return "[%s]"%"".join(rv) + rv.append(escapeRegexp(chr(item[0])) + "-" + + escapeRegexp(chr(item[1]))) + return "[%s]" % "".join(rv) + def hexToInt(hex_str): return int(hex_str, 16) + def escapeRegexp(string): specialCharacters = (".", "^", "$", "*", "+", "?", "{", "}", - "[", "]", "|", "(", ")", "-") + "[", "]", "|", "(", ")", "-") for char in specialCharacters: string = string.replace(char, "\\" + char) - if char in string: - print string return string -#output from the above -nonXmlNameBMPRegexp = re.compile(u'[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') +# output from the above +nonXmlNameBMPRegexp = re.compile('[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') + +nonXmlNameFirstBMPRegexp = re.compile('[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') + +# Simpler things +nonPubidCharRegexp = re.compile("[^\x20\x0D\x0Aa-zA-Z0-9\-\'()+,./:=?;!*#@$_%]") -nonXmlNameFirstBMPRegexp = re.compile(u'[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') class InfosetFilter(object): replacementRegexp = re.compile(r"U[\dA-F]{5,5}") - def __init__(self, replaceChars = None, - dropXmlnsLocalName = False, - dropXmlnsAttrNs = False, - preventDoubleDashComments = False, - preventDashAtCommentEnd = False, - replaceFormFeedCharacters = True): + + def __init__(self, replaceChars=None, + dropXmlnsLocalName=False, + dropXmlnsAttrNs=False, + preventDoubleDashComments=False, + preventDashAtCommentEnd=False, + replaceFormFeedCharacters=True, + preventSingleQuotePubid=False): self.dropXmlnsLocalName = dropXmlnsLocalName self.dropXmlnsAttrNs = dropXmlnsAttrNs @@ -113,14 +202,17 @@ class InfosetFilter(object): self.replaceFormFeedCharacters = replaceFormFeedCharacters + self.preventSingleQuotePubid = preventSingleQuotePubid + self.replaceCache = {} def coerceAttribute(self, name, namespace=None): if self.dropXmlnsLocalName and name.startswith("xmlns:"): - #Need a datalosswarning here + warnings.warn("Attributes cannot begin with xmlns", DataLossWarning) return None - elif (self.dropXmlnsAttrNs and + elif (self.dropXmlnsAttrNs and namespace == "http://www.w3.org/2000/xmlns/"): + warnings.warn("Attributes cannot be in the xml namespace", DataLossWarning) return None else: return self.toXmlName(name) @@ -131,20 +223,35 @@ class InfosetFilter(object): def coerceComment(self, data): if self.preventDoubleDashComments: while "--" in data: + warnings.warn("Comments cannot contain adjacent dashes", DataLossWarning) data = data.replace("--", "- -") return data - + def coerceCharacters(self, data): if self.replaceFormFeedCharacters: + for i in range(data.count("\x0C")): + warnings.warn("Text cannot contain U+000C", DataLossWarning) data = data.replace("\x0C", " ") - #Other non-xml characters + # Other non-xml characters return data + def coercePubid(self, data): + dataOutput = data + for char in nonPubidCharRegexp.findall(data): + warnings.warn("Coercing non-XML pubid", DataLossWarning) + replacement = self.getReplacementCharacter(char) + dataOutput = dataOutput.replace(char, replacement) + if self.preventSingleQuotePubid and dataOutput.find("'") >= 0: + warnings.warn("Pubid cannot contain single quote", DataLossWarning) + dataOutput = dataOutput.replace("'", self.getReplacementCharacter("'")) + return dataOutput + def toXmlName(self, name): nameFirst = name[0] nameRest = name[1:] m = nonXmlNameFirstBMPRegexp.match(nameFirst) if m: + warnings.warn("Coercing non-XML name", DataLossWarning) nameFirstOutput = self.getReplacementCharacter(nameFirst) else: nameFirstOutput = nameFirst @@ -152,10 +259,11 @@ class InfosetFilter(object): nameRestOutput = nameRest replaceChars = set(nonXmlNameBMPRegexp.findall(nameRest)) for char in replaceChars: + warnings.warn("Coercing non-XML name", DataLossWarning) replacement = self.getReplacementCharacter(char) nameRestOutput = nameRestOutput.replace(char, replacement) return nameFirstOutput + nameRestOutput - + def getReplacementCharacter(self, char): if char in self.replaceCache: replacement = self.replaceCache[char] @@ -169,9 +277,9 @@ class InfosetFilter(object): return name def escapeChar(self, char): - replacement = "U" + hex(ord(char))[2:].upper().rjust(5, "0") + replacement = "U%05X" % ord(char) self.replaceCache[char] = replacement return replacement def unescapeChar(self, charcode): - return unichr(int(charcode[1:], 16)) + return chr(int(charcode[1:], 16)) diff --git a/libs/html5lib/inputstream.py b/libs/html5lib/inputstream.py index edec1329..004bdd4a 100644 --- a/libs/html5lib/inputstream.py +++ b/libs/html5lib/inputstream.py @@ -1,19 +1,33 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type + import codecs import re -import types -import sys -from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase -from constants import encodings, ReparseException -import utils +from .constants import EOF, spaceCharacters, asciiLetters, asciiUppercase +from .constants import encodings, ReparseException +from . import utils -#Non-unicode versions of constants for use in the pre-parser -spaceCharactersBytes = frozenset([str(item) for item in spaceCharacters]) -asciiLettersBytes = frozenset([str(item) for item in asciiLetters]) -asciiUppercaseBytes = frozenset([str(item) for item in asciiUppercase]) -spacesAngleBrackets = spaceCharactersBytes | frozenset([">", "<"]) +from io import StringIO -invalid_unicode_re = re.compile(u"[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]") +try: + from io import BytesIO +except ImportError: + BytesIO = StringIO + +try: + from io import BufferedIOBase +except ImportError: + class BufferedIOBase(object): + pass + +# Non-unicode versions of constants for use in the pre-parser +spaceCharactersBytes = frozenset([item.encode("ascii") for item in spaceCharacters]) +asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters]) +asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase]) +spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"]) + +invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]") non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, @@ -23,22 +37,23 @@ non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF]) -ascii_punctuation_re = re.compile(ur"[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]") +ascii_punctuation_re = re.compile("[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]") # Cache for charsUntil() charsUntilRegEx = {} - -class BufferedStream: + + +class BufferedStream(object): """Buffering for streams that do not have buffering of their own - The buffer is implemented as a list of chunks on the assumption that + The buffer is implemented as a list of chunks on the assumption that joining many strings will be slow since it is O(n**2) """ - + def __init__(self, stream): self.stream = stream self.buffer = [] - self.position = [-1,0] #chunk number, offset + self.position = [-1, 0] # chunk number, offset def tell(self): pos = 0 @@ -48,11 +63,11 @@ class BufferedStream: return pos def seek(self, pos): - assert pos < self._bufferedBytes() + assert pos <= self._bufferedBytes() offset = pos i = 0 while len(self.buffer[i]) < offset: - offset -= pos + offset -= len(self.buffer[i]) i += 1 self.position = [i, offset] @@ -64,7 +79,7 @@ class BufferedStream: return self._readStream(bytes) else: return self._readFromBuffer(bytes) - + def _bufferedBytes(self): return sum([len(item) for item in self.buffer]) @@ -83,7 +98,7 @@ class BufferedStream: while bufferIndex < len(self.buffer) and remainingBytes != 0: assert remainingBytes > 0 bufferedData = self.buffer[bufferIndex] - + if remainingBytes <= len(bufferedData) - bufferOffset: bytesToRead = remainingBytes self.position = [bufferIndex, bufferOffset + bytesToRead] @@ -91,20 +106,33 @@ class BufferedStream: bytesToRead = len(bufferedData) - bufferOffset self.position = [bufferIndex, len(bufferedData)] bufferIndex += 1 - data = rv.append(bufferedData[bufferOffset: - bufferOffset + bytesToRead]) + rv.append(bufferedData[bufferOffset:bufferOffset + bytesToRead]) remainingBytes -= bytesToRead bufferOffset = 0 if remainingBytes: rv.append(self._readStream(remainingBytes)) - - return "".join(rv) - + + return b"".join(rv) -class HTMLInputStream: +def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True): + if hasattr(source, "read"): + isUnicode = isinstance(source.read(0), text_type) + else: + isUnicode = isinstance(source, text_type) + + if isUnicode: + if encoding is not None: + raise TypeError("Cannot explicitly set an encoding with a unicode string") + + return HTMLUnicodeInputStream(source) + else: + return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) + + +class HTMLUnicodeInputStream(object): """Provides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing @@ -114,7 +142,7 @@ class HTMLInputStream: _defaultChunkSize = 10240 - def __init__(self, source, encoding=None, parseMeta=True, chardet=True): + def __init__(self, source): """Initialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source @@ -126,49 +154,29 @@ class HTMLInputStream: the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) - + parseMeta - Look for a <meta> element containing encoding information """ - #Craziness - if len(u"\U0010FFFF") == 1: + # Craziness + if len("\U0010FFFF") == 1: self.reportCharacterErrors = self.characterErrorsUCS4 - self.replaceCharactersRegexp = re.compile(u"[\uD800-\uDFFF]") + self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]") else: self.reportCharacterErrors = self.characterErrorsUCS2 - self.replaceCharactersRegexp = re.compile(u"([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])") + self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])") # List of where new lines occur self.newLines = [0] - self.charEncoding = (codecName(encoding), "certain") - - # Raw Stream - for unicode objects this will encode to utf-8 and set - # self.charEncoding as appropriate - self.rawStream = self.openStream(source) - - # Encoding Information - #Number of bytes to use when looking for a meta element with - #encoding information - self.numBytesMeta = 512 - #Number of bytes to use when using detecting encoding using chardet - self.numBytesChardet = 100 - #Encoding to use if no other information can be found - self.defaultEncoding = "windows-1252" - - #Detect encoding iff no explicit "transport level" encoding is supplied - if (self.charEncoding[0] is None): - self.charEncoding = self.detectEncoding(parseMeta, chardet) - + self.charEncoding = ("utf-8", "certain") + self.dataStream = self.openStream(source) self.reset() def reset(self): - self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream, - 'replace') - - self.chunk = u"" + self.chunk = "" self.chunkSize = 0 self.chunkOffset = 0 self.errors = [] @@ -177,8 +185,8 @@ class HTMLInputStream: self.prevNumLines = 0 # number of columns in the last line of the previous chunk self.prevNumCols = 0 - - #Deal with CR LF and surrogates split over chunk boundaries + + # Deal with CR LF and surrogates split over chunk boundaries self._bufferedCharacter = None def openStream(self, source): @@ -191,126 +199,15 @@ class HTMLInputStream: if hasattr(source, 'read'): stream = source else: - # Otherwise treat source as a string and convert to a file object - if isinstance(source, unicode): - source = source.encode('utf-8') - self.charEncoding = ("utf-8", "certain") - try: - from io import BytesIO - except: - # 2to3 converts this line to: from io import StringIO - from cStringIO import StringIO as BytesIO - stream = BytesIO(source) - - if (not(hasattr(stream, "tell") and hasattr(stream, "seek")) or - stream is sys.stdin): - stream = BufferedStream(stream) + stream = StringIO(source) return stream - def detectEncoding(self, parseMeta=True, chardet=True): - #First look for a BOM - #This will also read past the BOM if present - encoding = self.detectBOM() - confidence = "certain" - #If there is no BOM need to look for meta elements with encoding - #information - if encoding is None and parseMeta: - encoding = self.detectEncodingMeta() - confidence = "tentative" - #Guess with chardet, if avaliable - if encoding is None and chardet: - confidence = "tentative" - try: - from chardet.universaldetector import UniversalDetector - buffers = [] - detector = UniversalDetector() - while not detector.done: - buffer = self.rawStream.read(self.numBytesChardet) - if not buffer: - break - buffers.append(buffer) - detector.feed(buffer) - detector.close() - encoding = detector.result['encoding'] - self.rawStream.seek(0) - except ImportError: - pass - # If all else fails use the default encoding - if encoding is None: - confidence="tentative" - encoding = self.defaultEncoding - - #Substitute for equivalent encodings: - encodingSub = {"iso-8859-1":"windows-1252"} - - if encoding.lower() in encodingSub: - encoding = encodingSub[encoding.lower()] - - return encoding, confidence - - def changeEncoding(self, newEncoding): - newEncoding = codecName(newEncoding) - if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"): - newEncoding = "utf-8" - if newEncoding is None: - return - elif newEncoding == self.charEncoding[0]: - self.charEncoding = (self.charEncoding[0], "certain") - else: - self.rawStream.seek(0) - self.reset() - self.charEncoding = (newEncoding, "certain") - raise ReparseException, "Encoding changed from %s to %s"%(self.charEncoding[0], newEncoding) - - def detectBOM(self): - """Attempts to detect at BOM at the start of the stream. If - an encoding can be determined from the BOM return the name of the - encoding otherwise return None""" - bomDict = { - codecs.BOM_UTF8: 'utf-8', - codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be', - codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be' - } - - # Go to beginning of file and read in 4 bytes - string = self.rawStream.read(4) - - # Try detecting the BOM using bytes from the string - encoding = bomDict.get(string[:3]) # UTF-8 - seek = 3 - if not encoding: - # Need to detect UTF-32 before UTF-16 - encoding = bomDict.get(string) # UTF-32 - seek = 4 - if not encoding: - encoding = bomDict.get(string[:2]) # UTF-16 - seek = 2 - - # Set the read position past the BOM if one was found, otherwise - # set it to the start of the stream - self.rawStream.seek(encoding and seek or 0) - - return encoding - - def detectEncodingMeta(self): - """Report the encoding declared by the meta element - """ - buffer = self.rawStream.read(self.numBytesMeta) - parser = EncodingParser(buffer) - self.rawStream.seek(0) - encoding = parser.getEncoding() - - if encoding in ("utf-16", "utf-16-be", "utf-16-le"): - encoding = "utf-8" - - return encoding - def _position(self, offset): chunk = self.chunk - nLines = chunk.count(u'\n', 0, offset) + nLines = chunk.count('\n', 0, offset) positionLine = self.prevNumLines + nLines - lastLinePos = chunk.rfind(u'\n', 0, offset) + lastLinePos = chunk.rfind('\n', 0, offset) if lastLinePos == -1: positionColumn = self.prevNumCols + offset else: @@ -320,7 +217,7 @@ class HTMLInputStream: def position(self): """Returns (line, col) of the current position in the stream.""" line, col = self._position(self.chunkOffset) - return (line+1, col) + return (line + 1, col) def char(self): """ Read one character from the stream or queue if available. Return @@ -343,34 +240,34 @@ class HTMLInputStream: self.prevNumLines, self.prevNumCols = self._position(self.chunkSize) - self.chunk = u"" + self.chunk = "" self.chunkSize = 0 self.chunkOffset = 0 data = self.dataStream.read(chunkSize) - - #Deal with CR LF and surrogates broken across chunks + + # Deal with CR LF and surrogates broken across chunks if self._bufferedCharacter: data = self._bufferedCharacter + data self._bufferedCharacter = None elif not data: # We have no more data, bye-bye stream return False - + if len(data) > 1: lastv = ord(data[-1]) if lastv == 0x0D or 0xD800 <= lastv <= 0xDBFF: self._bufferedCharacter = data[-1] data = data[:-1] - + self.reportCharacterErrors(data) - + # Replace invalid characters # Note U+0000 is dealt with in the tokenizer - data = self.replaceCharactersRegexp.sub(u"\ufffd", data) - - data = data.replace(u"\r\n", u"\n") - data = data.replace(u"\r", u"\n") + data = self.replaceCharactersRegexp.sub("\ufffd", data) + + data = data.replace("\r\n", "\n") + data = data.replace("\r", "\n") self.chunk = data self.chunkSize = len(data) @@ -378,23 +275,22 @@ class HTMLInputStream: return True def characterErrorsUCS4(self, data): - for i in xrange(len(invalid_unicode_re.findall(data))): + for i in range(len(invalid_unicode_re.findall(data))): self.errors.append("invalid-codepoint") def characterErrorsUCS2(self, data): - #Someone picked the wrong compile option - #You lose + # Someone picked the wrong compile option + # You lose skip = False - import sys for match in invalid_unicode_re.finditer(data): if skip: continue codepoint = ord(match.group()) pos = match.start() - #Pretty sure there should be endianness issues here - if utils.isSurrogatePair(data[pos:pos+2]): - #We have a surrogate pair! - char_val = utils.surrogatePairToCodepoint(data[pos:pos+2]) + # Pretty sure there should be endianness issues here + if utils.isSurrogatePair(data[pos:pos + 2]): + # We have a surrogate pair! + char_val = utils.surrogatePairToCodepoint(data[pos:pos + 2]) if char_val in non_bmp_invalid_codepoints: self.errors.append("invalid-codepoint") skip = True @@ -405,7 +301,7 @@ class HTMLInputStream: skip = False self.errors.append("invalid-codepoint") - def charsUntil(self, characters, opposite = False): + def charsUntil(self, characters, opposite=False): """ Returns a string of characters from the stream up to but not including any character in 'characters' or EOF. 'characters' must be a container that supports the 'in' method and iteration over its @@ -417,12 +313,12 @@ class HTMLInputStream: chars = charsUntilRegEx[(characters, opposite)] except KeyError: if __debug__: - for c in characters: + for c in characters: assert(ord(c) < 128) - regex = u"".join([u"\\x%02x" % ord(c) for c in characters]) + regex = "".join(["\\x%02x" % ord(c) for c in characters]) if not opposite: - regex = u"^%s" % regex - chars = charsUntilRegEx[(characters, opposite)] = re.compile(u"[%s]+" % regex) + regex = "^%s" % regex + chars = charsUntilRegEx[(characters, opposite)] = re.compile("[%s]+" % regex) rv = [] @@ -449,7 +345,7 @@ class HTMLInputStream: # Reached EOF break - r = u"".join(rv) + r = "".join(rv) return r def unget(self, char): @@ -468,26 +364,210 @@ class HTMLInputStream: self.chunkOffset -= 1 assert self.chunk[self.chunkOffset] == char -class EncodingBytes(str): + +class HTMLBinaryInputStream(HTMLUnicodeInputStream): + """Provides a unicode stream of characters to the HTMLTokenizer. + + This class takes care of character encoding and removing or replacing + incorrect byte-sequences and also provides column and line tracking. + + """ + + def __init__(self, source, encoding=None, parseMeta=True, chardet=True): + """Initialises the HTMLInputStream. + + HTMLInputStream(source, [encoding]) -> Normalized stream from source + for use by html5lib. + + source can be either a file-object, local filename or a string. + + The optional encoding parameter must be a string that indicates + the encoding. If specified, that encoding will be used, + regardless of any BOM or later declaration (such as in a meta + element) + + parseMeta - Look for a <meta> element containing encoding information + + """ + # Raw Stream - for unicode objects this will encode to utf-8 and set + # self.charEncoding as appropriate + self.rawStream = self.openStream(source) + + HTMLUnicodeInputStream.__init__(self, self.rawStream) + + self.charEncoding = (codecName(encoding), "certain") + + # Encoding Information + # Number of bytes to use when looking for a meta element with + # encoding information + self.numBytesMeta = 512 + # Number of bytes to use when using detecting encoding using chardet + self.numBytesChardet = 100 + # Encoding to use if no other information can be found + self.defaultEncoding = "windows-1252" + + # Detect encoding iff no explicit "transport level" encoding is supplied + if (self.charEncoding[0] is None): + self.charEncoding = self.detectEncoding(parseMeta, chardet) + + # Call superclass + self.reset() + + def reset(self): + self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream, + 'replace') + HTMLUnicodeInputStream.reset(self) + + def openStream(self, source): + """Produces a file object from source. + + source can be either a file object, local filename or a string. + + """ + # Already a file object + if hasattr(source, 'read'): + stream = source + else: + stream = BytesIO(source) + + try: + stream.seek(stream.tell()) + except: + stream = BufferedStream(stream) + + return stream + + def detectEncoding(self, parseMeta=True, chardet=True): + # First look for a BOM + # This will also read past the BOM if present + encoding = self.detectBOM() + confidence = "certain" + # If there is no BOM need to look for meta elements with encoding + # information + if encoding is None and parseMeta: + encoding = self.detectEncodingMeta() + confidence = "tentative" + # Guess with chardet, if avaliable + if encoding is None and chardet: + confidence = "tentative" + try: + try: + from charade.universaldetector import UniversalDetector + except ImportError: + from chardet.universaldetector import UniversalDetector + buffers = [] + detector = UniversalDetector() + while not detector.done: + buffer = self.rawStream.read(self.numBytesChardet) + assert isinstance(buffer, bytes) + if not buffer: + break + buffers.append(buffer) + detector.feed(buffer) + detector.close() + encoding = detector.result['encoding'] + self.rawStream.seek(0) + except ImportError: + pass + # If all else fails use the default encoding + if encoding is None: + confidence = "tentative" + encoding = self.defaultEncoding + + # Substitute for equivalent encodings: + encodingSub = {"iso-8859-1": "windows-1252"} + + if encoding.lower() in encodingSub: + encoding = encodingSub[encoding.lower()] + + return encoding, confidence + + def changeEncoding(self, newEncoding): + assert self.charEncoding[1] != "certain" + newEncoding = codecName(newEncoding) + if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"): + newEncoding = "utf-8" + if newEncoding is None: + return + elif newEncoding == self.charEncoding[0]: + self.charEncoding = (self.charEncoding[0], "certain") + else: + self.rawStream.seek(0) + self.reset() + self.charEncoding = (newEncoding, "certain") + raise ReparseException("Encoding changed from %s to %s" % (self.charEncoding[0], newEncoding)) + + def detectBOM(self): + """Attempts to detect at BOM at the start of the stream. If + an encoding can be determined from the BOM return the name of the + encoding otherwise return None""" + bomDict = { + codecs.BOM_UTF8: 'utf-8', + codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be', + codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be' + } + + # Go to beginning of file and read in 4 bytes + string = self.rawStream.read(4) + assert isinstance(string, bytes) + + # Try detecting the BOM using bytes from the string + encoding = bomDict.get(string[:3]) # UTF-8 + seek = 3 + if not encoding: + # Need to detect UTF-32 before UTF-16 + encoding = bomDict.get(string) # UTF-32 + seek = 4 + if not encoding: + encoding = bomDict.get(string[:2]) # UTF-16 + seek = 2 + + # Set the read position past the BOM if one was found, otherwise + # set it to the start of the stream + self.rawStream.seek(encoding and seek or 0) + + return encoding + + def detectEncodingMeta(self): + """Report the encoding declared by the meta element + """ + buffer = self.rawStream.read(self.numBytesMeta) + assert isinstance(buffer, bytes) + parser = EncodingParser(buffer) + self.rawStream.seek(0) + encoding = parser.getEncoding() + + if encoding in ("utf-16", "utf-16-be", "utf-16-le"): + encoding = "utf-8" + + return encoding + + +class EncodingBytes(bytes): """String-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raised""" def __new__(self, value): - return str.__new__(self, value.lower()) + assert isinstance(value, bytes) + return bytes.__new__(self, value.lower()) def __init__(self, value): - self._position=-1 - + self._position = -1 + def __iter__(self): return self - - def next(self): + + def __next__(self): p = self._position = self._position + 1 if p >= len(self): raise StopIteration elif p < 0: raise TypeError - return self[p] + return self[p:p + 1] + + def next(self): + # Py2 compat + return self.__next__() def previous(self): p = self._position @@ -496,13 +576,13 @@ class EncodingBytes(str): elif p < 0: raise TypeError self._position = p = p - 1 - return self[p] - + return self[p:p + 1] + def setPosition(self, position): if self._position >= len(self): raise StopIteration self._position = position - + def getPosition(self): if self._position >= len(self): raise StopIteration @@ -510,19 +590,19 @@ class EncodingBytes(str): return self._position else: return None - + position = property(getPosition, setPosition) def getCurrentByte(self): - return self[self.position] - + return self[self.position:self.position + 1] + currentByte = property(getCurrentByte) def skip(self, chars=spaceCharactersBytes): """Skip past a list of characters""" p = self.position # use property for the error-checking while p < len(self): - c = self[p] + c = self[p:p + 1] if c not in chars: self._position = p return c @@ -533,7 +613,7 @@ class EncodingBytes(str): def skipUntil(self, chars): p = self.position while p < len(self): - c = self[p] + c = self[p:p + 1] if c in chars: self._position = p return c @@ -542,16 +622,16 @@ class EncodingBytes(str): return None def matchBytes(self, bytes): - """Look for a sequence of bytes at the start of a string. If the bytes - are found return True and advance the position to the byte after the + """Look for a sequence of bytes at the start of a string. If the bytes + are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone""" p = self.position - data = self[p:p+len(bytes)] + data = self[p:p + len(bytes)] rv = data.startswith(bytes) if rv: self.position += len(bytes) return rv - + def jumpTo(self, bytes): """Look for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the match""" @@ -560,11 +640,12 @@ class EncodingBytes(str): # XXX: This is ugly, but I can't see a nicer way to fix this. if self._position == -1: self._position = 0 - self._position += (newPosition + len(bytes)-1) + self._position += (newPosition + len(bytes) - 1) return True else: raise StopIteration + class EncodingParser(object): """Mini parser for detecting character encoding from meta elements""" @@ -575,147 +656,158 @@ class EncodingParser(object): def getEncoding(self): methodDispatch = ( - ("<!--",self.handleComment), - ("<meta",self.handleMeta), - ("</",self.handlePossibleEndTag), - ("<!",self.handleOther), - ("<?",self.handleOther), - ("<",self.handlePossibleStartTag)) + (b"<!--", self.handleComment), + (b"<meta", self.handleMeta), + (b"</", self.handlePossibleEndTag), + (b"<!", self.handleOther), + (b"<?", self.handleOther), + (b"<", self.handlePossibleStartTag)) for byte in self.data: keepParsing = True for key, method in methodDispatch: if self.data.matchBytes(key): try: - keepParsing = method() + keepParsing = method() break except StopIteration: - keepParsing=False + keepParsing = False break if not keepParsing: break - + return self.encoding def handleComment(self): """Skip over comments""" - return self.data.jumpTo("-->") + return self.data.jumpTo(b"-->") def handleMeta(self): if self.data.currentByte not in spaceCharactersBytes: - #if we have <meta not followed by a space so just keep going + # if we have <meta not followed by a space so just keep going return True - #We have a valid meta element we want to search for attributes + # We have a valid meta element we want to search for attributes + hasPragma = False + pendingEncoding = None while True: - #Try to find the next attribute after the current position + # Try to find the next attribute after the current position attr = self.getAttribute() if attr is None: return True else: - if attr[0] == "charset": + if attr[0] == b"http-equiv": + hasPragma = attr[1] == b"content-type" + if hasPragma and pendingEncoding is not None: + self.encoding = pendingEncoding + return False + elif attr[0] == b"charset": tentativeEncoding = attr[1] codec = codecName(tentativeEncoding) if codec is not None: self.encoding = codec return False - elif attr[0] == "content": + elif attr[0] == b"content": contentParser = ContentAttrParser(EncodingBytes(attr[1])) tentativeEncoding = contentParser.parse() - codec = codecName(tentativeEncoding) - if codec is not None: - self.encoding = codec - return False + if tentativeEncoding is not None: + codec = codecName(tentativeEncoding) + if codec is not None: + if hasPragma: + self.encoding = codec + return False + else: + pendingEncoding = codec def handlePossibleStartTag(self): return self.handlePossibleTag(False) def handlePossibleEndTag(self): - self.data.next() + next(self.data) return self.handlePossibleTag(True) def handlePossibleTag(self, endTag): data = self.data if data.currentByte not in asciiLettersBytes: - #If the next byte is not an ascii letter either ignore this - #fragment (possible start tag case) or treat it according to - #handleOther + # If the next byte is not an ascii letter either ignore this + # fragment (possible start tag case) or treat it according to + # handleOther if endTag: data.previous() self.handleOther() return True - + c = data.skipUntil(spacesAngleBrackets) - if c == "<": - #return to the first step in the overall "two step" algorithm - #reprocessing the < byte + if c == b"<": + # return to the first step in the overall "two step" algorithm + # reprocessing the < byte data.previous() else: - #Read all attributes + # Read all attributes attr = self.getAttribute() while attr is not None: attr = self.getAttribute() return True def handleOther(self): - return self.data.jumpTo(">") + return self.data.jumpTo(b">") def getAttribute(self): - """Return a name,value pair for the next attribute in the stream, + """Return a name,value pair for the next attribute in the stream, if one is found, or None""" data = self.data # Step 1 (skip chars) - c = data.skip(spaceCharactersBytes | frozenset("/")) + c = data.skip(spaceCharactersBytes | frozenset([b"/"])) + assert c is None or len(c) == 1 # Step 2 - if c in (">", None): + if c in (b">", None): return None # Step 3 attrName = [] attrValue = [] - #Step 4 attribute name + # Step 4 attribute name while True: - if c == "=" and attrName: + if c == b"=" and attrName: break elif c in spaceCharactersBytes: - #Step 6! + # Step 6! c = data.skip() - c = data.next() break - elif c in ("/", ">"): - return "".join(attrName), "" + elif c in (b"/", b">"): + return b"".join(attrName), b"" elif c in asciiUppercaseBytes: attrName.append(c.lower()) - elif c == None: + elif c is None: return None else: attrName.append(c) - #Step 5 - c = data.next() - #Step 7 - if c != "=": + # Step 5 + c = next(data) + # Step 7 + if c != b"=": data.previous() - return "".join(attrName), "" - #Step 8 - data.next() - #Step 9 + return b"".join(attrName), b"" + # Step 8 + next(data) + # Step 9 c = data.skip() - #Step 10 - if c in ("'", '"'): - #10.1 + # Step 10 + if c in (b"'", b'"'): + # 10.1 quoteChar = c while True: - #10.2 - c = data.next() - #10.3 + # 10.2 + c = next(data) + # 10.3 if c == quoteChar: - data.next() - return "".join(attrName), "".join(attrValue) - #10.4 + next(data) + return b"".join(attrName), b"".join(attrValue) + # 10.4 elif c in asciiUppercaseBytes: attrValue.append(c.lower()) - #10.5 + # 10.5 else: attrValue.append(c) - elif c == ">": - return "".join(attrName), "" + elif c == b">": + return b"".join(attrName), b"" elif c in asciiUppercaseBytes: attrValue.append(c.lower()) elif c is None: @@ -724,9 +816,9 @@ class EncodingParser(object): attrValue.append(c) # Step 11 while True: - c = data.next() + c = next(data) if c in spacesAngleBrackets: - return "".join(attrName), "".join(attrValue) + return b"".join(attrName), b"".join(attrValue) elif c in asciiUppercaseBytes: attrValue.append(c.lower()) elif c is None: @@ -737,21 +829,23 @@ class EncodingParser(object): class ContentAttrParser(object): def __init__(self, data): + assert isinstance(data, bytes) self.data = data + def parse(self): try: - #Check if the attr name is charset - #otherwise return - self.data.jumpTo("charset") + # Check if the attr name is charset + # otherwise return + self.data.jumpTo(b"charset") self.data.position += 1 self.data.skip() - if not self.data.currentByte == "=": - #If there is no = sign keep looking for attrs + if not self.data.currentByte == b"=": + # If there is no = sign keep looking for attrs return None self.data.position += 1 self.data.skip() - #Look for an encoding between matching quote marks - if self.data.currentByte in ('"', "'"): + # Look for an encoding between matching quote marks + if self.data.currentByte in (b'"', b"'"): quoteMark = self.data.currentByte self.data.position += 1 oldPosition = self.data.position @@ -760,13 +854,13 @@ class ContentAttrParser(object): else: return None else: - #Unquoted value + # Unquoted value oldPosition = self.data.position try: self.data.skipUntil(spaceCharactersBytes) return self.data[oldPosition:self.data.position] except StopIteration: - #Return the whole remaining value + # Return the whole remaining value return self.data[oldPosition:] except StopIteration: return None @@ -775,7 +869,12 @@ class ContentAttrParser(object): def codecName(encoding): """Return the python codec name corresponding to an encoding or None if the string doesn't correspond to a valid encoding.""" - if (encoding is not None and type(encoding) in types.StringTypes): + if isinstance(encoding, bytes): + try: + encoding = encoding.decode("ascii") + except UnicodeDecodeError: + return None + if encoding: canonicalName = ascii_punctuation_re.sub("", encoding).lower() return encodings.get(canonicalName, None) else: diff --git a/libs/html5lib/sanitizer.py b/libs/html5lib/sanitizer.py index ae4c7d83..71dc5212 100644 --- a/libs/html5lib/sanitizer.py +++ b/libs/html5lib/sanitizer.py @@ -1,142 +1,145 @@ +from __future__ import absolute_import, division, unicode_literals + import re from xml.sax.saxutils import escape, unescape -from tokenizer import HTMLTokenizer -from constants import tokenTypes +from .tokenizer import HTMLTokenizer +from .constants import tokenTypes + class HTMLSanitizerMixin(object): """ sanitization of XHTML+MathML+SVG and of inline style attributes.""" acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', - 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', - 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', - 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', - 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', - 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', - 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', - 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', - 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', - 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', - 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', - 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] - + 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', + 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', + 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', + 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', + 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', + 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', + 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', + 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', + 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', + 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', + 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', + 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] + mathml_elements = ['maction', 'math', 'merror', 'mfrac', 'mi', - 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', - 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', - 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', - 'munderover', 'none'] - + 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', + 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', + 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', + 'munderover', 'none'] + svg_elements = ['a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', - 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', - 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] - + 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', + 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', + 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', + 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', + 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] + acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', - 'background', 'balance', 'bgcolor', 'bgproperties', 'border', - 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', - 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', - 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', - 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', - 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', - 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', - 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', - 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', - 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', - 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', - 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', - 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', - 'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg', - 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', - 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', - 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', - 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', - 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', - 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', - 'width', 'wrap', 'xml:lang'] + 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', + 'background', 'balance', 'bgcolor', 'bgproperties', 'border', + 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', + 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', + 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', + 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', + 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', + 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', + 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', + 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', + 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', + 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', + 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', + 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', + 'optimum', 'pattern', 'ping', 'point-size', 'poster', 'pqg', 'preload', + 'prompt', 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', + 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', + 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', + 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', + 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', + 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', + 'width', 'wrap', 'xml:lang'] mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign', - 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', - 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', - 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', - 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', - 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', - 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', - 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', - 'xlink:type', 'xmlns', 'xmlns:xlink'] - - svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', - 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', - 'fill-opacity', 'fill-rule', 'font-family', 'font-size', - 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', - 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', - 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', - 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', - 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', - 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', - 'opacity', 'orient', 'origin', 'overline-position', - 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', - 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', - 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', - 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', - 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', - 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', - 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', - 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', - 'transform', 'type', 'u1', 'u2', 'underline-position', - 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', - 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', - 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', - 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', - 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', - 'y1', 'y2', 'zoomAndPan'] + 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', + 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', + 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', + 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', + 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', + 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', + 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', + 'xlink:type', 'xmlns', 'xmlns:xlink'] - attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', - 'xlink:href', 'xml:base'] + svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', + 'arabic-form', 'ascent', 'attributeName', 'attributeType', + 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', + 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', + 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', + 'fill-opacity', 'fill-rule', 'font-family', 'font-size', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', + 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', + 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', + 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', + 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', + 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', + 'opacity', 'orient', 'origin', 'overline-position', + 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', + 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', + 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', + 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', + 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', + 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', + 'transform', 'type', 'u1', 'u2', 'underline-position', + 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', + 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', + 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', + 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', + 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', + 'y1', 'y2', 'zoomAndPan'] + + attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', + 'xlink:href', 'xml:base'] svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill', - 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', - 'mask', 'stroke'] + 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', + 'mask', 'stroke'] svg_allow_local_href = ['altGlyph', 'animate', 'animateColor', - 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', - 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', - 'set', 'use'] - + 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', + 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', + 'set', 'use'] + acceptable_css_properties = ['azimuth', 'background-color', - 'border-bottom-color', 'border-collapse', 'border-color', - 'border-left-color', 'border-right-color', 'border-top-color', 'clear', - 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', - 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', - 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', - 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', - 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', - 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', - 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', - 'white-space', 'width'] - + 'border-bottom-color', 'border-collapse', 'border-color', + 'border-left-color', 'border-right-color', 'border-top-color', 'clear', + 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', + 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', + 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', + 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', + 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', + 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', + 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', + 'white-space', 'width'] + acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue', - 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', - 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', - 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', - 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', - 'transparent', 'underline', 'white', 'yellow'] - - acceptable_svg_properties = [ 'fill', 'fill-opacity', 'fill-rule', - 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', - 'stroke-opacity'] - - acceptable_protocols = [ 'ed2k', 'ftp', 'http', 'https', 'irc', - 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', - 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', - 'ssh', 'sftp', 'rtsp', 'afs' ] - + 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', + 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', + 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', + 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', + 'transparent', 'underline', 'white', 'yellow'] + + acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule', + 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', + 'stroke-opacity'] + + acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc', + 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', + 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', + 'ssh', 'sftp', 'rtsp', 'afs'] + # subclasses may define their own versions of these constants allowed_elements = acceptable_elements + mathml_elements + svg_elements allowed_attributes = acceptable_attributes + mathml_attributes + svg_attributes @@ -160,94 +163,104 @@ class HTMLSanitizerMixin(object): # accommodate filters which use token_type differently token_type = token["type"] - if token_type in tokenTypes.keys(): - token_type = tokenTypes[token_type] + if token_type in list(tokenTypes.keys()): + token_type = tokenTypes[token_type] - if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], - tokenTypes["EmptyTag"]): + if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], + tokenTypes["EmptyTag"]): if token["name"] in self.allowed_elements: - if token.has_key("data"): - attrs = dict([(name,val) for name,val in - token["data"][::-1] - if name in self.allowed_attributes]) - for attr in self.attr_val_is_uri: - if not attrs.has_key(attr): - continue - val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', - unescape(attrs[attr])).lower() - #remove replacement characters from unescaped characters - val_unescaped = val_unescaped.replace(u"\ufffd", "") - if (re.match("^[a-z0-9][-+.a-z0-9]*:",val_unescaped) and - (val_unescaped.split(':')[0] not in - self.allowed_protocols)): - del attrs[attr] - for attr in self.svg_attr_val_allows_ref: - if attr in attrs: - attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', - ' ', - unescape(attrs[attr])) - if (token["name"] in self.svg_allow_local_href and - 'xlink:href' in attrs and re.search('^\s*[^#\s].*', - attrs['xlink:href'])): - del attrs['xlink:href'] - if attrs.has_key('style'): - attrs['style'] = self.sanitize_css(attrs['style']) - token["data"] = [[name,val] for name,val in attrs.items()] - return token + return self.allowed_token(token, token_type) else: - if token_type == tokenTypes["EndTag"]: - token["data"] = "</%s>" % token["name"] - elif token["data"]: - attrs = ''.join([' %s="%s"' % (k,escape(v)) for k,v in token["data"]]) - token["data"] = "<%s%s>" % (token["name"],attrs) - else: - token["data"] = "<%s>" % token["name"] - if token.get("selfClosing"): - token["data"]=token["data"][:-1] + "/>" - - if token["type"] in tokenTypes.keys(): - token["type"] = "Characters" - else: - token["type"] = tokenTypes["Characters"] - - del token["name"] - return token + return self.disallowed_token(token, token_type) elif token_type == tokenTypes["Comment"]: pass else: return token + def allowed_token(self, token, token_type): + if "data" in token: + attrs = dict([(name, val) for name, val in + token["data"][::-1] + if name in self.allowed_attributes]) + for attr in self.attr_val_is_uri: + if attr not in attrs: + continue + val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', + unescape(attrs[attr])).lower() + # remove replacement characters from unescaped characters + val_unescaped = val_unescaped.replace("\ufffd", "") + if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and + (val_unescaped.split(':')[0] not in + self.allowed_protocols)): + del attrs[attr] + for attr in self.svg_attr_val_allows_ref: + if attr in attrs: + attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', + ' ', + unescape(attrs[attr])) + if (token["name"] in self.svg_allow_local_href and + 'xlink:href' in attrs and re.search('^\s*[^#\s].*', + attrs['xlink:href'])): + del attrs['xlink:href'] + if 'style' in attrs: + attrs['style'] = self.sanitize_css(attrs['style']) + token["data"] = [[name, val] for name, val in list(attrs.items())] + return token + + def disallowed_token(self, token, token_type): + if token_type == tokenTypes["EndTag"]: + token["data"] = "</%s>" % token["name"] + elif token["data"]: + attrs = ''.join([' %s="%s"' % (k, escape(v)) for k, v in token["data"]]) + token["data"] = "<%s%s>" % (token["name"], attrs) + else: + token["data"] = "<%s>" % token["name"] + if token.get("selfClosing"): + token["data"] = token["data"][:-1] + "/>" + + if token["type"] in list(tokenTypes.keys()): + token["type"] = "Characters" + else: + token["type"] = tokenTypes["Characters"] + + del token["name"] + return token + def sanitize_css(self, style): # disallow urls - style=re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ',style) + style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style) # gauntlet - if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): return '' - if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): return '' + if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): + return '' + if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): + return '' clean = [] - for prop,value in re.findall("([-\w]+)\s*:\s*([^:;]*)",style): - if not value: continue - if prop.lower() in self.allowed_css_properties: - clean.append(prop + ': ' + value + ';') - elif prop.split('-')[0].lower() in ['background','border','margin', - 'padding']: - for keyword in value.split(): - if not keyword in self.acceptable_css_keywords and \ - not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$",keyword): - break - else: - clean.append(prop + ': ' + value + ';') - elif prop.lower() in self.allowed_svg_properties: - clean.append(prop + ': ' + value + ';') + for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style): + if not value: + continue + if prop.lower() in self.allowed_css_properties: + clean.append(prop + ': ' + value + ';') + elif prop.split('-')[0].lower() in ['background', 'border', 'margin', + 'padding']: + for keyword in value.split(): + if not keyword in self.acceptable_css_keywords and \ + not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword): + break + else: + clean.append(prop + ': ' + value + ';') + elif prop.lower() in self.allowed_svg_properties: + clean.append(prop + ': ' + value + ';') return ' '.join(clean) + class HTMLSanitizer(HTMLTokenizer, HTMLSanitizerMixin): def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True, lowercaseElementName=False, lowercaseAttrName=False, parser=None): - #Change case matching defaults as we only output lowercase html anyway - #This solution doesn't seem ideal... + # Change case matching defaults as we only output lowercase html anyway + # This solution doesn't seem ideal... HTMLTokenizer.__init__(self, stream, encoding, parseMeta, useChardet, lowercaseElementName, lowercaseAttrName, parser=parser) diff --git a/libs/html5lib/serializer/__init__.py b/libs/html5lib/serializer/__init__.py index 1b746655..8380839a 100644 --- a/libs/html5lib/serializer/__init__.py +++ b/libs/html5lib/serializer/__init__.py @@ -1,17 +1,16 @@ +from __future__ import absolute_import, division, unicode_literals -from html5lib import treewalkers +from .. import treewalkers -from htmlserializer import HTMLSerializer -from xhtmlserializer import XHTMLSerializer +from .htmlserializer import HTMLSerializer -def serialize(input, tree="simpletree", format="html", encoding=None, + +def serialize(input, tree="etree", format="html", encoding=None, **serializer_opts): # XXX: Should we cache this? - walker = treewalkers.getTreeWalker(tree) + walker = treewalkers.getTreeWalker(tree) if format == "html": s = HTMLSerializer(**serializer_opts) - elif format == "xhtml": - s = XHTMLSerializer(**serializer_opts) else: - raise ValueError, "type must be either html or xhtml" + raise ValueError("type must be html") return s.render(walker(input), encoding) diff --git a/libs/html5lib/serializer/htmlserializer.py b/libs/html5lib/serializer/htmlserializer.py index 8dd0a815..412a5a22 100644 --- a/libs/html5lib/serializer/htmlserializer.py +++ b/libs/html5lib/serializer/htmlserializer.py @@ -1,18 +1,20 @@ -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import ImmutableSet as frozenset +from __future__ import absolute_import, division, unicode_literals +from six import text_type import gettext _ = gettext.gettext -from html5lib.constants import voidElements, booleanAttributes, spaceCharacters -from html5lib.constants import rcdataElements, entities, xmlEntities -from html5lib import utils +try: + from functools import reduce +except ImportError: + pass + +from ..constants import voidElements, booleanAttributes, spaceCharacters +from ..constants import rcdataElements, entities, xmlEntities +from .. import utils from xml.sax.saxutils import escape -spaceCharacters = u"".join(spaceCharacters) +spaceCharacters = "".join(spaceCharacters) try: from codecs import register_error, xmlcharrefreplace_errors @@ -21,24 +23,18 @@ except ImportError: else: unicode_encode_errors = "htmlentityreplace" - from html5lib.constants import entities - encode_entity_map = {} - is_ucs4 = len(u"\U0010FFFF") == 1 - for k, v in entities.items(): - #skip multi-character entities + is_ucs4 = len("\U0010FFFF") == 1 + for k, v in list(entities.items()): + # skip multi-character entities if ((is_ucs4 and len(v) > 1) or - (not is_ucs4 and len(v) > 2)): + (not is_ucs4 and len(v) > 2)): continue if v != "&": if len(v) == 2: v = utils.surrogatePairToCodepoint(v) else: - try: - v = ord(v) - except: - print v - raise + v = ord(v) if not v in encode_entity_map or k.islower(): # prefer < over < and similarly for &, >, etc. encode_entity_map[v] = k @@ -53,8 +49,8 @@ else: skip = False continue index = i + exc.start - if utils.isSurrogatePair(exc.object[index:min([exc.end, index+2])]): - codepoint = utils.surrogatePairToCodepoint(exc.object[index:index+2]) + if utils.isSurrogatePair(exc.object[index:min([exc.end, index + 2])]): + codepoint = utils.surrogatePairToCodepoint(exc.object[index:index + 2]) skip = True else: codepoint = ord(c) @@ -67,8 +63,8 @@ else: if not e.endswith(";"): res.append(";") else: - res.append("&#x%s;"%(hex(cp)[2:])) - return (u"".join(res), exc.end) + res.append("&#x%s;" % (hex(cp)[2:])) + return ("".join(res), exc.end) else: return xmlcharrefreplace_errors(exc) @@ -81,7 +77,7 @@ class HTMLSerializer(object): # attribute quoting options quote_attr_values = False - quote_char = u'"' + quote_char = '"' use_best_quote_char = True # tag syntax options @@ -96,15 +92,17 @@ class HTMLSerializer(object): resolve_entities = True # miscellaneous options + alphabetical_attributes = False inject_meta_charset = True strip_whitespace = False sanitize = False options = ("quote_attr_values", "quote_char", "use_best_quote_char", - "minimize_boolean_attributes", "use_trailing_solidus", - "space_before_trailing_solidus", "omit_optional_tags", - "strip_whitespace", "inject_meta_charset", "escape_lt_in_attrs", - "escape_rcdata", "resolve_entities", "sanitize") + "omit_optional_tags", "minimize_boolean_attributes", + "use_trailing_solidus", "space_before_trailing_solidus", + "escape_lt_in_attrs", "escape_rcdata", "resolve_entities", + "alphabetical_attributes", "inject_meta_charset", + "strip_whitespace", "sanitize") def __init__(self, **kwargs): """Initialize HTMLSerializer. @@ -147,10 +145,12 @@ class HTMLSerializer(object): See `html5lib user documentation`_ omit_optional_tags=True|False Omit start/end tags that are optional. + alphabetical_attributes=False|True + Reorder attributes to be in alphabetical order. .. _html5lib user documentation: http://code.google.com/p/html5lib/wiki/UserDocumentation """ - if kwargs.has_key('quote_char'): + if 'quote_char' in kwargs: self.use_best_quote_char = False for attr in self.options: setattr(self, attr, kwargs.get(attr, getattr(self, attr))) @@ -158,14 +158,14 @@ class HTMLSerializer(object): self.strict = False def encode(self, string): - assert(isinstance(string, unicode)) + assert(isinstance(string, text_type)) if self.encoding: return string.encode(self.encoding, unicode_encode_errors) else: return string def encodeStrict(self, string): - assert(isinstance(string, unicode)) + assert(isinstance(string, text_type)) if self.encoding: return string.encode(self.encoding, "strict") else: @@ -175,39 +175,46 @@ class HTMLSerializer(object): self.encoding = encoding in_cdata = False self.errors = [] + if encoding and self.inject_meta_charset: - from html5lib.filters.inject_meta_charset import Filter + from ..filters.inject_meta_charset import Filter treewalker = Filter(treewalker, encoding) - # XXX: WhitespaceFilter should be used before OptionalTagFilter + # WhitespaceFilter should be used before OptionalTagFilter # for maximum efficiently of this latter filter if self.strip_whitespace: - from html5lib.filters.whitespace import Filter + from ..filters.whitespace import Filter treewalker = Filter(treewalker) if self.sanitize: - from html5lib.filters.sanitizer import Filter + from ..filters.sanitizer import Filter treewalker = Filter(treewalker) if self.omit_optional_tags: - from html5lib.filters.optionaltags import Filter + from ..filters.optionaltags import Filter treewalker = Filter(treewalker) + # Alphabetical attributes must be last, as other filters + # could add attributes and alter the order + if self.alphabetical_attributes: + from ..filters.alphabeticalattributes import Filter + treewalker = Filter(treewalker) + for token in treewalker: type = token["type"] if type == "Doctype": - doctype = u"<!DOCTYPE %s" % token["name"] - + doctype = "<!DOCTYPE %s" % token["name"] + if token["publicId"]: - doctype += u' PUBLIC "%s"' % token["publicId"] + doctype += ' PUBLIC "%s"' % token["publicId"] elif token["systemId"]: - doctype += u" SYSTEM" - if token["systemId"]: - if token["systemId"].find(u'"') >= 0: - if token["systemId"].find(u"'") >= 0: + doctype += " SYSTEM" + if token["systemId"]: + if token["systemId"].find('"') >= 0: + if token["systemId"].find("'") >= 0: self.serializeError(_("System identifer contains both single and double quote characters")) - quote_char = u"'" + quote_char = "'" else: - quote_char = u'"' - doctype += u" %s%s%s" % (quote_char, token["systemId"], quote_char) - - doctype += u">" + quote_char = '"' + doctype += " %s%s%s" % (quote_char, token["systemId"], quote_char) + + doctype += ">" yield self.encodeStrict(doctype) elif type in ("Characters", "SpaceCharacters"): @@ -220,41 +227,41 @@ class HTMLSerializer(object): elif type in ("StartTag", "EmptyTag"): name = token["name"] - yield self.encodeStrict(u"<%s" % name) + yield self.encodeStrict("<%s" % name) if name in rcdataElements and not self.escape_rcdata: in_cdata = True elif in_cdata: self.serializeError(_("Unexpected child element of a CDATA element")) - attributes = [] - for (attr_namespace,attr_name),attr_value in sorted(token["data"].items()): - #TODO: Add namespace support here + for (attr_namespace, attr_name), attr_value in token["data"].items(): + # TODO: Add namespace support here k = attr_name v = attr_value - yield self.encodeStrict(u' ') + yield self.encodeStrict(' ') yield self.encodeStrict(k) if not self.minimize_boolean_attributes or \ - (k not in booleanAttributes.get(name, tuple()) \ - and k not in booleanAttributes.get("", tuple())): - yield self.encodeStrict(u"=") + (k not in booleanAttributes.get(name, tuple()) + and k not in booleanAttributes.get("", tuple())): + yield self.encodeStrict("=") if self.quote_attr_values or not v: quote_attr = True else: - quote_attr = reduce(lambda x,y: x or (y in v), - spaceCharacters + u">\"'=", False) - v = v.replace(u"&", u"&") - if self.escape_lt_in_attrs: v = v.replace(u"<", u"<") + quote_attr = reduce(lambda x, y: x or (y in v), + spaceCharacters + ">\"'=", False) + v = v.replace("&", "&") + if self.escape_lt_in_attrs: + v = v.replace("<", "<") if quote_attr: quote_char = self.quote_char if self.use_best_quote_char: - if u"'" in v and u'"' not in v: - quote_char = u'"' - elif u'"' in v and u"'" not in v: - quote_char = u"'" - if quote_char == u"'": - v = v.replace(u"'", u"'") + if "'" in v and '"' not in v: + quote_char = '"' + elif '"' in v and "'" not in v: + quote_char = "'" + if quote_char == "'": + v = v.replace("'", "'") else: - v = v.replace(u'"', u""") + v = v.replace('"', """) yield self.encodeStrict(quote_char) yield self.encode(v) yield self.encodeStrict(quote_char) @@ -262,10 +269,10 @@ class HTMLSerializer(object): yield self.encode(v) if name in voidElements and self.use_trailing_solidus: if self.space_before_trailing_solidus: - yield self.encodeStrict(u" /") + yield self.encodeStrict(" /") else: - yield self.encodeStrict(u"/") - yield self.encode(u">") + yield self.encodeStrict("/") + yield self.encode(">") elif type == "EndTag": name = token["name"] @@ -273,13 +280,13 @@ class HTMLSerializer(object): in_cdata = False elif in_cdata: self.serializeError(_("Unexpected child element of a CDATA element")) - yield self.encodeStrict(u"</%s>" % name) + yield self.encodeStrict("</%s>" % name) elif type == "Comment": data = token["data"] if data.find("--") >= 0: self.serializeError(_("Comment contains --")) - yield self.encodeStrict(u"<!--%s-->" % token["data"]) + yield self.encodeStrict("<!--%s-->" % token["data"]) elif type == "Entity": name = token["name"] @@ -289,7 +296,7 @@ class HTMLSerializer(object): if self.resolve_entities and key not in xmlEntities: data = entities[key] else: - data = u"&%s;" % name + data = "&%s;" % name yield self.encodeStrict(data) else: @@ -297,9 +304,9 @@ class HTMLSerializer(object): def render(self, treewalker, encoding=None): if encoding: - return "".join(list(self.serialize(treewalker, encoding))) + return b"".join(list(self.serialize(treewalker, encoding))) else: - return u"".join(list(self.serialize(treewalker))) + return "".join(list(self.serialize(treewalker))) def serializeError(self, data="XXX ERROR MESSAGE NEEDED"): # XXX The idea is to make data mandatory. @@ -307,6 +314,7 @@ class HTMLSerializer(object): if self.strict: raise SerializeError + def SerializeError(Exception): """Error in serialized tree""" pass diff --git a/libs/html5lib/serializer/xhtmlserializer.py b/libs/html5lib/serializer/xhtmlserializer.py deleted file mode 100644 index 7fdce47b..00000000 --- a/libs/html5lib/serializer/xhtmlserializer.py +++ /dev/null @@ -1,9 +0,0 @@ -from htmlserializer import HTMLSerializer - -class XHTMLSerializer(HTMLSerializer): - quote_attr_values = True - minimize_boolean_attributes = False - use_trailing_solidus = True - escape_lt_in_attrs = True - omit_optional_tags = False - escape_rcdata = True diff --git a/libs/html5lib/tokenizer.py b/libs/html5lib/tokenizer.py index 7e9eca88..79774578 100644 --- a/libs/html5lib/tokenizer.py +++ b/libs/html5lib/tokenizer.py @@ -1,27 +1,25 @@ +from __future__ import absolute_import, division, unicode_literals + try: - frozenset + chr = unichr # flake8: noqa except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset -try: - from collections import deque -except ImportError: - from utils import deque - -from constants import spaceCharacters -from constants import entitiesWindows1252, entities -from constants import asciiLowercase, asciiLetters, asciiUpper2Lower -from constants import digits, hexDigits, EOF -from constants import tokenTypes, tagTokenTypes -from constants import replacementCharacters + pass -from inputstream import HTMLInputStream +from collections import deque + +from .constants import spaceCharacters +from .constants import entities +from .constants import asciiLetters, asciiUpper2Lower +from .constants import digits, hexDigits, EOF +from .constants import tokenTypes, tagTokenTypes +from .constants import replacementCharacters + +from .inputstream import HTMLInputStream + +from .trie import Trie + +entitiesTrie = Trie(entities) -# Group entities by their first character, for faster lookups -entitiesByFirstChar = {} -for e in entities: - entitiesByFirstChar.setdefault(e[0], []).append(e) class HTMLTokenizer(object): """ This class takes care of tokenizing HTML. @@ -42,10 +40,10 @@ class HTMLTokenizer(object): self.stream = HTMLInputStream(stream, encoding, parseMeta, useChardet) self.parser = parser - #Perform case conversions? + # Perform case conversions? self.lowercaseElementName = lowercaseElementName self.lowercaseAttrName = lowercaseAttrName - + # Setup the initial tokenizer state self.escapeFlag = False self.lastFourChars = [] @@ -100,78 +98,79 @@ class HTMLTokenizer(object): if charAsInt in replacementCharacters: char = replacementCharacters[charAsInt] self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", - "datavars": {"charAsInt": charAsInt}}) - elif ((0xD800 <= charAsInt <= 0xDFFF) or + "illegal-codepoint-for-numeric-entity", + "datavars": {"charAsInt": charAsInt}}) + elif ((0xD800 <= charAsInt <= 0xDFFF) or (charAsInt > 0x10FFFF)): - char = u"\uFFFD" + char = "\uFFFD" self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", - "datavars": {"charAsInt": charAsInt}}) + "illegal-codepoint-for-numeric-entity", + "datavars": {"charAsInt": charAsInt}}) else: - #Should speed up this check somehow (e.g. move the set to a constant) - if ((0x0001 <= charAsInt <= 0x0008) or - (0x000E <= charAsInt <= 0x001F) or - (0x007F <= charAsInt <= 0x009F) or - (0xFDD0 <= charAsInt <= 0xFDEF) or - charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE, + # Should speed up this check somehow (e.g. move the set to a constant) + if ((0x0001 <= charAsInt <= 0x0008) or + (0x000E <= charAsInt <= 0x001F) or + (0x007F <= charAsInt <= 0x009F) or + (0xFDD0 <= charAsInt <= 0xFDEF) or + charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, - 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, + 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, - 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE, - 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, - 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, + 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE, + 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, + 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF])): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", + "illegal-codepoint-for-numeric-entity", "datavars": {"charAsInt": charAsInt}}) try: # Try/except needed as UCS-2 Python builds' unichar only works # within the BMP. - char = unichr(charAsInt) + char = chr(charAsInt) except ValueError: - char = eval("u'\\U%08x'" % charAsInt) + v = charAsInt - 0x10000 + char = chr(0xD800 | (v >> 10)) + chr(0xDC00 | (v & 0x3FF)) # Discard the ; if present. Otherwise, put it back on the queue and # invoke parseError on parser. - if c != u";": + if c != ";": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "numeric-entity-without-semicolon"}) + "numeric-entity-without-semicolon"}) self.stream.unget(c) return char def consumeEntity(self, allowedChar=None, fromAttribute=False): # Initialise to the default output for when no entity is matched - output = u"&" + output = "&" charStack = [self.stream.char()] - if (charStack[0] in spaceCharacters or charStack[0] in (EOF, u"<", u"&") - or (allowedChar is not None and allowedChar == charStack[0])): + if (charStack[0] in spaceCharacters or charStack[0] in (EOF, "<", "&") + or (allowedChar is not None and allowedChar == charStack[0])): self.stream.unget(charStack[0]) - elif charStack[0] == u"#": + elif charStack[0] == "#": # Read the next character to see if it's hex or decimal hex = False charStack.append(self.stream.char()) - if charStack[-1] in (u"x", u"X"): + if charStack[-1] in ("x", "X"): hex = True charStack.append(self.stream.char()) # charStack[-1] should be the first digit if (hex and charStack[-1] in hexDigits) \ - or (not hex and charStack[-1] in digits): + or (not hex and charStack[-1] in digits): # At least one digit found, so consume the whole number self.stream.unget(charStack[-1]) output = self.consumeNumberEntity(hex) else: # No digits found self.tokenQueue.append({"type": tokenTypes["ParseError"], - "data": "expected-numeric-entity"}) + "data": "expected-numeric-entity"}) self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) else: # At this point in the process might have named entity. Entities @@ -179,46 +178,40 @@ class HTMLTokenizer(object): # # Consume characters and compare to these to a substring of the # entity names in the list until the substring no longer matches. - filteredEntityList = entitiesByFirstChar.get(charStack[0], []) - - def entitiesStartingWith(name): - return [e for e in filteredEntityList if e.startswith(name)] - - while (charStack[-1] is not EOF and - entitiesStartingWith("".join(charStack))): + while (charStack[-1] is not EOF): + if not entitiesTrie.has_keys_with_prefix("".join(charStack)): + break charStack.append(self.stream.char()) # At this point we have a string that starts with some characters # that may match an entity - entityName = None - # Try to find the longest entity the string will match to take care # of ¬i for instance. - for entityLength in xrange(len(charStack)-1, 1, -1): - possibleEntityName = "".join(charStack[:entityLength]) - if possibleEntityName in entities: - entityName = possibleEntityName - break + try: + entityName = entitiesTrie.longest_prefix("".join(charStack[:-1])) + entityLength = len(entityName) + except KeyError: + entityName = None if entityName is not None: if entityName[-1] != ";": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "named-entity-without-semicolon"}) + "named-entity-without-semicolon"}) if (entityName[-1] != ";" and fromAttribute and (charStack[entityLength] in asciiLetters or charStack[entityLength] in digits or - charStack[entityLength] == "=")): + charStack[entityLength] == "=")): self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) else: output = entities[entityName] self.stream.unget(charStack.pop()) - output += u"".join(charStack[entityLength:]) + output += "".join(charStack[entityLength:]) else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-named-entity"}) + "expected-named-entity"}) self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) if fromAttribute: self.currentToken["data"][-1][1] += output @@ -246,28 +239,26 @@ class HTMLTokenizer(object): token["name"] = token["name"].translate(asciiUpper2Lower) if token["type"] == tokenTypes["EndTag"]: if token["data"]: - self.tokenQueue.append({"type":tokenTypes["ParseError"], - "data":"attributes-in-end-tag"}) + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "attributes-in-end-tag"}) if token["selfClosing"]: - self.tokenQueue.append({"type":tokenTypes["ParseError"], - "data":"self-closing-flag-on-end-tag"}) + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "self-closing-flag-on-end-tag"}) self.tokenQueue.append(token) self.state = self.dataState - # Below are the various tokenizer states worked out. - def dataState(self): data = self.stream.char() if data == "&": self.state = self.entityDataState elif data == "<": self.state = self.tagOpenState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], - "data":"invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\u0000"}) + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "invalid-codepoint"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\u0000"}) elif data is EOF: # Tokenization ends. return False @@ -276,21 +267,21 @@ class HTMLTokenizer(object): # state". At that point spaceCharacters are important so they are # emitted separately. self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data": - data + self.stream.charsUntil(spaceCharacters, True)}) + data + self.stream.charsUntil(spaceCharacters, True)}) # No need to update lastFourChars here, since the first space will # have already been appended to lastFourChars and will have broken # any <!-- or --> sequences else: - chars = self.stream.charsUntil((u"&", u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("&", "<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True def entityDataState(self): self.consumeEntity() self.state = self.dataState return True - + def rcdataState(self): data = self.stream.char() if data == "&": @@ -300,113 +291,113 @@ class HTMLTokenizer(object): elif data == EOF: # Tokenization ends. return False - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data in spaceCharacters: # Directly after emitting a token you switch back to the "data # state". At that point spaceCharacters are important so they are # emitted separately. self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data": - data + self.stream.charsUntil(spaceCharacters, True)}) + data + self.stream.charsUntil(spaceCharacters, True)}) # No need to update lastFourChars here, since the first space will # have already been appended to lastFourChars and will have broken # any <!-- or --> sequences else: - chars = self.stream.charsUntil((u"&", u"<")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("&", "<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True def characterReferenceInRcdata(self): self.consumeEntity() self.state = self.rcdataState return True - + def rawtextState(self): data = self.stream.char() if data == "<": self.state = self.rawtextLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: # Tokenization ends. return False else: - chars = self.stream.charsUntil((u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def scriptDataState(self): data = self.stream.char() if data == "<": self.state = self.scriptDataLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: # Tokenization ends. return False else: - chars = self.stream.charsUntil((u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def plaintextState(self): data = self.stream.char() if data == EOF: # Tokenization ends. return False - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + self.stream.charsUntil(u"\u0000")}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + self.stream.charsUntil("\u0000")}) return True def tagOpenState(self): data = self.stream.char() - if data == u"!": + if data == "!": self.state = self.markupDeclarationOpenState - elif data == u"/": + elif data == "/": self.state = self.closeTagOpenState elif data in asciiLetters: - self.currentToken = {"type": tokenTypes["StartTag"], + self.currentToken = {"type": tokenTypes["StartTag"], "name": data, "data": [], "selfClosing": False, "selfClosingAcknowledged": False} self.state = self.tagNameState - elif data == u">": + elif data == ">": # XXX In theory it could be something besides a tag name. But # do we really care? self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name-but-got-right-bracket"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<>"}) + "expected-tag-name-but-got-right-bracket"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<>"}) self.state = self.dataState - elif data == u"?": + elif data == "?": # XXX In theory it could be something besides a tag name. But # do we really care? self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name-but-got-question-mark"}) + "expected-tag-name-but-got-question-mark"}) self.stream.unget(data) self.state = self.bogusCommentState else: # XXX self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + "expected-tag-name"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.dataState return True @@ -415,22 +406,22 @@ class HTMLTokenizer(object): data = self.stream.char() if data in asciiLetters: self.currentToken = {"type": tokenTypes["EndTag"], "name": data, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.tagNameState - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-right-bracket"}) + "expected-closing-tag-but-got-right-bracket"}) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-eof"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + "expected-closing-tag-but-got-eof"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.state = self.dataState else: # XXX data can be _'_... self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-char", - "datavars": {"data": data}}) + "expected-closing-tag-but-got-char", + "datavars": {"data": data}}) self.stream.unget(data) self.state = self.bogusCommentState return True @@ -439,229 +430,229 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-tag-name"}) + "eof-in-tag-name"}) self.state = self.dataState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] += u"\uFFFD" + self.currentToken["name"] += "\uFFFD" else: self.currentToken["name"] += data # (Don't use charsUntil here, because tag names are # very short and it's faster to not do anything fancy) return True - + def rcdataLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.rcdataEndTagOpenState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.rcdataState return True - + def rcdataEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.rcdataEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.rcdataState return True - + def rcdataEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.rcdataState return True - + def rawtextLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.rawtextEndTagOpenState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.rawtextState return True - + def rawtextEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.rawtextEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.rawtextState return True - + def rawtextEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.rawtextState return True - + def scriptDataLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.scriptDataEndTagOpenState elif data == "!": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<!"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<!"}) self.state = self.scriptDataEscapeStartState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.scriptDataEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapeStartState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapeStartDashState else: self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapeStartDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashDashState else: self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapedState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashState elif data == "<": self.state = self.scriptDataEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: self.state = self.dataState else: - chars = self.stream.charsUntil((u"<", u"-", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "-", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def scriptDataEscapedDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashDashState elif data == "<": self.state = self.scriptDataEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataEscapedState elif data == EOF: self.state = self.dataState @@ -669,21 +660,21 @@ class HTMLTokenizer(object): self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedDashDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) elif data == "<": self.state = self.scriptDataEscapedLessThanSignState elif data == ">": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u">"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"}) self.state = self.scriptDataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataEscapedState elif data == EOF: self.state = self.dataState @@ -691,61 +682,61 @@ class HTMLTokenizer(object): self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.scriptDataEscapedEndTagOpenState elif data in asciiLetters: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<" + data}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<" + data}) self.temporaryBuffer = data self.state = self.scriptDataDoubleEscapeStartState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer = data self.state = self.scriptDataEscapedEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataDoubleEscapeStartState(self): data = self.stream.char() if data in (spaceCharacters | frozenset(("/", ">"))): @@ -761,87 +752,87 @@ class HTMLTokenizer(object): self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataDoubleEscapedState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataDoubleEscapedDashState elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) return True - + def scriptDataDoubleEscapedDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataDoubleEscapedDashDashState elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataDoubleEscapedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataDoubleEscapedState return True - - def scriptDataDoubleEscapedDashState(self): + + def scriptDataDoubleEscapedDashDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState elif data == ">": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u">"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"}) self.state = self.scriptDataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataDoubleEscapedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataDoubleEscapedState return True - + def scriptDataDoubleEscapedLessThanSignState(self): data = self.stream.char() if data == "/": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"/"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "/"}) self.temporaryBuffer = "" self.state = self.scriptDataDoubleEscapeEndState else: self.stream.unget(data) self.state = self.scriptDataDoubleEscapedState return True - + def scriptDataDoubleEscapeEndState(self): data = self.stream.char() if data in (spaceCharacters | frozenset(("/", ">"))): @@ -865,23 +856,23 @@ class HTMLTokenizer(object): elif data in asciiLetters: self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data in (u"'", u'"', u"=", u"<"): + elif data in ("'", '"', "=", "<"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-in-attribute-name"}) + "invalid-character-in-attribute-name"}) self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"].append([u"\uFFFD", ""]) + self.currentToken["data"].append(["\uFFFD", ""]) self.state = self.attributeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-name-but-got-eof"}) + "expected-attribute-name-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"].append([data, ""]) @@ -892,34 +883,34 @@ class HTMLTokenizer(object): data = self.stream.char() leavingThisState = True emitToken = False - if data == u"=": + if data == "=": self.state = self.beforeAttributeValueState elif data in asciiLetters: self.currentToken["data"][-1][0] += data +\ - self.stream.charsUntil(asciiLetters, True) + self.stream.charsUntil(asciiLetters, True) leavingThisState = False - elif data == u">": + elif data == ">": # XXX If we emit here the attributes are converted to a dict # without being checked and when the code below runs we error # because data is a dict not a list emitToken = True elif data in spaceCharacters: self.state = self.afterAttributeNameState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][0] += u"\uFFFD" + self.currentToken["data"][-1][0] += "\uFFFD" leavingThisState = False - elif data in (u"'", u'"', u"<"): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data in ("'", '"', "<"): + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-in-attribute-name"}) + "invalid-character-in-attribute-name"}) self.currentToken["data"][-1][0] += data leavingThisState = False elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "eof-in-attribute-name"}) self.state = self.dataState else: @@ -936,7 +927,7 @@ class HTMLTokenizer(object): for name, value in self.currentToken["data"][:-1]: if self.currentToken["data"][-1][0] == name: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "duplicate-attribute"}) + "duplicate-attribute"}) break # XXX Fix for above XXX if emitToken: @@ -947,28 +938,28 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.stream.charsUntil(spaceCharacters, True) - elif data == u"=": + elif data == "=": self.state = self.beforeAttributeValueState - elif data == u">": + elif data == ">": self.emitCurrentToken() elif data in asciiLetters: self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"].append([u"\uFFFD", ""]) + self.currentToken["data"].append(["\uFFFD", ""]) self.state = self.attributeNameState - elif data in (u"'", u'"', u"<"): + elif data in ("'", '"', "<"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-after-attribute-name"}) + "invalid-character-after-attribute-name"}) self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-end-of-tag-but-got-eof"}) + "expected-end-of-tag-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"].append([data, ""]) @@ -979,30 +970,30 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.stream.charsUntil(spaceCharacters, True) - elif data == u"\"": + elif data == "\"": self.state = self.attributeValueDoubleQuotedState - elif data == u"&": + elif data == "&": self.state = self.attributeValueUnQuotedState - self.stream.unget(data); - elif data == u"'": + self.stream.unget(data) + elif data == "'": self.state = self.attributeValueSingleQuotedState - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-value-but-got-right-bracket"}) + "expected-attribute-value-but-got-right-bracket"}) self.emitCurrentToken() - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" self.state = self.attributeValueUnQuotedState - elif data in (u"=", u"<", u"`"): + elif data in ("=", "<", "`"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "equals-in-unquoted-attribute-value"}) + "equals-in-unquoted-attribute-value"}) self.currentToken["data"][-1][1] += data self.state = self.attributeValueUnQuotedState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-value-but-got-eof"}) + "expected-attribute-value-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data @@ -1013,81 +1004,81 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterAttributeValueState - elif data == u"&": - self.processEntityInAttribute(u'"') - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "&": + self.processEntityInAttribute('"') + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-double-quote"}) + "eof-in-attribute-value-double-quote"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data +\ - self.stream.charsUntil(("\"", u"&")) + self.stream.charsUntil(("\"", "&", "\u0000")) return True def attributeValueSingleQuotedState(self): data = self.stream.char() if data == "'": self.state = self.afterAttributeValueState - elif data == u"&": - self.processEntityInAttribute(u"'") - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "&": + self.processEntityInAttribute("'") + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-single-quote"}) + "eof-in-attribute-value-single-quote"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data +\ - self.stream.charsUntil(("'", u"&")) + self.stream.charsUntil(("'", "&", "\u0000")) return True def attributeValueUnQuotedState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u"&": + elif data == "&": self.processEntityInAttribute(">") - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data in (u'"', u"'", u"=", u"<", u"`"): + elif data in ('"', "'", "=", "<", "`"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-in-unquoted-attribute-value"}) + "unexpected-character-in-unquoted-attribute-value"}) self.currentToken["data"][-1][1] += data - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-no-quotes"}) + "eof-in-attribute-value-no-quotes"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data + self.stream.charsUntil( - frozenset((u"&", u">", u'"', u"'", u"=", u"<", u"`")) | spaceCharacters) + frozenset(("&", ">", '"', "'", "=", "<", "`", "\u0000")) | spaceCharacters) return True def afterAttributeValueState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-EOF-after-attribute-value"}) + "unexpected-EOF-after-attribute-value"}) self.stream.unget(data) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-after-attribute-value"}) + "unexpected-character-after-attribute-value"}) self.stream.unget(data) self.state = self.beforeAttributeNameState return True @@ -1098,14 +1089,14 @@ class HTMLTokenizer(object): self.currentToken["selfClosing"] = True self.emitCurrentToken() elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-EOF-after-solidus-in-tag"}) + "unexpected-EOF-after-solidus-in-tag"}) self.stream.unget(data) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-after-soldius-in-tag"}) + "unexpected-character-after-solidus-in-tag"}) self.stream.unget(data) self.state = self.beforeAttributeNameState return True @@ -1114,10 +1105,10 @@ class HTMLTokenizer(object): # Make a new comment token and give it as value all the characters # until the first > or EOF (charsUntil checks for EOF automatically) # and emit it. - data = self.stream.charsUntil(u">") - data = data.replace(u"\u0000", u"\uFFFD") + data = self.stream.charsUntil(">") + data = data.replace("\u0000", "\uFFFD") self.tokenQueue.append( - {"type": tokenTypes["Comment"], "data": data}) + {"type": tokenTypes["Comment"], "data": data}) # Eat the character directly after the bogus comment which is either a # ">" or an EOF. @@ -1127,28 +1118,28 @@ class HTMLTokenizer(object): def markupDeclarationOpenState(self): charStack = [self.stream.char()] - if charStack[-1] == u"-": + if charStack[-1] == "-": charStack.append(self.stream.char()) - if charStack[-1] == u"-": - self.currentToken = {"type": tokenTypes["Comment"], "data": u""} + if charStack[-1] == "-": + self.currentToken = {"type": tokenTypes["Comment"], "data": ""} self.state = self.commentStartState return True - elif charStack[-1] in (u'd', u'D'): + elif charStack[-1] in ('d', 'D'): matched = True - for expected in ((u'o', u'O'), (u'c', u'C'), (u't', u'T'), - (u'y', u'Y'), (u'p', u'P'), (u'e', u'E')): + for expected in (('o', 'O'), ('c', 'C'), ('t', 'T'), + ('y', 'Y'), ('p', 'P'), ('e', 'E')): charStack.append(self.stream.char()) if charStack[-1] not in expected: matched = False break if matched: self.currentToken = {"type": tokenTypes["Doctype"], - "name": u"", - "publicId": None, "systemId": None, + "name": "", + "publicId": None, "systemId": None, "correct": True} self.state = self.doctypeState return True - elif (charStack[-1] == "[" and + elif (charStack[-1] == "[" and self.parser is not None and self.parser.tree.openElements and self.parser.tree.openElements[-1].namespace != self.parser.tree.defaultNamespace): @@ -1163,7 +1154,7 @@ class HTMLTokenizer(object): return True self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-dashes-or-doctype"}) + "expected-dashes-or-doctype"}) while charStack: self.stream.unget(charStack.pop()) @@ -1174,41 +1165,41 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "-": self.state = self.commentStartDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"\uFFFD" + self.currentToken["data"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "incorrect-comment"}) + "incorrect-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment"}) + "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.currentToken["data"] += data self.state = self.commentState return True - + def commentStartDashState(self): data = self.stream.char() if data == "-": self.state = self.commentEndState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"-\uFFFD" + self.currentToken["data"] += "-\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "incorrect-comment"}) + "incorrect-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment"}) + "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: @@ -1216,95 +1207,94 @@ class HTMLTokenizer(object): self.state = self.commentState return True - def commentState(self): data = self.stream.char() - if data == u"-": + if data == "-": self.state = self.commentEndDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"\uFFFD" + self.currentToken["data"] += "\uFFFD" elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.currentToken["data"] += data + \ - self.stream.charsUntil((u"-", u"\u0000")) + self.stream.charsUntil(("-", "\u0000")) return True def commentEndDashState(self): data = self.stream.char() - if data == u"-": + if data == "-": self.state = self.commentEndState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"-\uFFFD" + self.currentToken["data"] += "-\uFFFD" self.state = self.commentState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-end-dash"}) + "eof-in-comment-end-dash"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - self.currentToken["data"] += u"-" + data + self.currentToken["data"] += "-" + data self.state = self.commentState return True def commentEndState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"--\uFFFD" + self.currentToken["data"] += "--\uFFFD" self.state = self.commentState elif data == "!": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-bang-after-double-dash-in-comment"}) + "unexpected-bang-after-double-dash-in-comment"}) self.state = self.commentEndBangState - elif data == u"-": + elif data == "-": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-dash-after-double-dash-in-comment"}) + "unexpected-dash-after-double-dash-in-comment"}) self.currentToken["data"] += data elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-double-dash"}) + "eof-in-comment-double-dash"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: # XXX self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-comment"}) - self.currentToken["data"] += u"--" + data + "unexpected-char-in-comment"}) + self.currentToken["data"] += "--" + data self.state = self.commentState return True def commentEndBangState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"-": + elif data == "-": self.currentToken["data"] += "--!" self.state = self.commentEndDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"--!\uFFFD" + self.currentToken["data"] += "--!\uFFFD" self.state = self.commentState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-end-bang-state"}) + "eof-in-comment-end-bang-state"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - self.currentToken["data"] += u"--!" + data + self.currentToken["data"] += "--!" + data self.state = self.commentState return True @@ -1314,13 +1304,13 @@ class HTMLTokenizer(object): self.state = self.beforeDoctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-eof"}) + "expected-doctype-name-but-got-eof"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "need-space-after-doctype"}) + "need-space-after-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypeNameState return True @@ -1329,20 +1319,20 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: pass - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-right-bracket"}) + "expected-doctype-name-but-got-right-bracket"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] = u"\uFFFD" + self.currentToken["name"] = "\uFFFD" self.state = self.doctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-eof"}) + "expected-doctype-name-but-got-eof"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1356,18 +1346,18 @@ class HTMLTokenizer(object): if data in spaceCharacters: self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.state = self.afterDoctypeNameState - elif data == u">": + elif data == ">": self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] += u"\uFFFD" + self.currentToken["name"] += "\uFFFD" self.state = self.doctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype-name"}) + "eof-in-doctype-name"}) self.currentToken["correct"] = False self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.tokenQueue.append(self.currentToken) @@ -1380,21 +1370,21 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: pass - elif data == u">": + elif data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.currentToken["correct"] = False self.stream.unget(data) self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - if data in (u"p", u"P"): + if data in ("p", "P"): matched = True - for expected in ((u"u", u"U"), (u"b", u"B"), (u"l", u"L"), - (u"i", u"I"), (u"c", u"C")): + for expected in (("u", "U"), ("b", "B"), ("l", "L"), + ("i", "I"), ("c", "C")): data = self.stream.char() if data not in expected: matched = False @@ -1402,10 +1392,10 @@ class HTMLTokenizer(object): if matched: self.state = self.afterDoctypePublicKeywordState return True - elif data in (u"s", u"S"): + elif data in ("s", "S"): matched = True - for expected in ((u"y", u"Y"), (u"s", u"S"), (u"t", u"T"), - (u"e", u"E"), (u"m", u"M")): + for expected in (("y", "Y"), ("s", "S"), ("t", "T"), + ("e", "E"), ("m", "M")): data = self.stream.char() if data not in expected: matched = False @@ -1420,25 +1410,25 @@ class HTMLTokenizer(object): # and needs to be ungetted self.stream.unget(data) self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-space-or-right-bracket-in-doctype", "datavars": - {"data": data}}) + "expected-space-or-right-bracket-in-doctype", "datavars": + {"data": data}}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def afterDoctypePublicKeywordState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeDoctypePublicIdentifierState elif data in ("'", '"'): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypePublicIdentifierState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1452,26 +1442,26 @@ class HTMLTokenizer(object): if data in spaceCharacters: pass elif data == "\"": - self.currentToken["publicId"] = u"" + self.currentToken["publicId"] = "" self.state = self.doctypePublicIdentifierDoubleQuotedState elif data == "'": - self.currentToken["publicId"] = u"" + self.currentToken["publicId"] = "" self.state = self.doctypePublicIdentifierSingleQuotedState elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True @@ -1480,19 +1470,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterDoctypePublicIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["publicId"] += u"\uFFFD" + self.currentToken["publicId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1504,19 +1494,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "'": self.state = self.afterDoctypePublicIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["publicId"] += u"\uFFFD" + self.currentToken["publicId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1533,27 +1523,27 @@ class HTMLTokenizer(object): self.state = self.dataState elif data == '"': self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) - self.currentToken["systemId"] = u"" + "unexpected-char-in-doctype"}) + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) - self.currentToken["systemId"] = u"" + "unexpected-char-in-doctype"}) + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def betweenDoctypePublicAndSystemIdentifiersState(self): data = self.stream.char() if data in spaceCharacters: @@ -1562,36 +1552,36 @@ class HTMLTokenizer(object): self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data == '"': - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def afterDoctypeSystemKeywordState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeDoctypeSystemIdentifierState elif data in ("'", '"'): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypeSystemIdentifierState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1599,32 +1589,32 @@ class HTMLTokenizer(object): self.stream.unget(data) self.state = self.beforeDoctypeSystemIdentifierState return True - + def beforeDoctypeSystemIdentifierState(self): data = self.stream.char() if data in spaceCharacters: pass elif data == "\"": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True @@ -1633,19 +1623,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterDoctypeSystemIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["systemId"] += u"\uFFFD" + self.currentToken["systemId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1657,19 +1647,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "'": self.state = self.afterDoctypeSystemIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["systemId"] += u"\uFFFD" + self.currentToken["systemId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1686,19 +1676,19 @@ class HTMLTokenizer(object): self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.state = self.bogusDoctypeState return True def bogusDoctypeState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: @@ -1713,32 +1703,29 @@ class HTMLTokenizer(object): def cdataSectionState(self): data = [] while True: - data.append(self.stream.charsUntil(u"]")) - charStack = [] - - for expected in ["]", "]", ">"]: - charStack.append(self.stream.char()) - matched = True - if charStack[-1] == EOF: - data.extend(charStack[:-1]) - break - elif charStack[-1] != expected: - matched = False - data.extend(charStack) - break - - if matched: + data.append(self.stream.charsUntil("]")) + data.append(self.stream.charsUntil(">")) + char = self.stream.char() + if char == EOF: break + else: + assert char == ">" + if data[-1][-2:] == "]]": + data[-1] = data[-1][:-2] + break + else: + data.append(char) + data = "".join(data) - #Deal with null here rather than in the parser - nullCount = data.count(u"\u0000") + # Deal with null here rather than in the parser + nullCount = data.count("\u0000") if nullCount > 0: - for i in xrange(nullCount): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + for i in range(nullCount): + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - data = data.replace(u"\u0000", u"\uFFFD") + data = data.replace("\u0000", "\uFFFD") if data: - self.tokenQueue.append({"type": tokenTypes["Characters"], + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.dataState return True diff --git a/libs/html5lib/treeadapters/__init__.py b/libs/html5lib/treeadapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/html5lib/treeadapters/sax.py b/libs/html5lib/treeadapters/sax.py new file mode 100644 index 00000000..ad47df95 --- /dev/null +++ b/libs/html5lib/treeadapters/sax.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import, division, unicode_literals + +from xml.sax.xmlreader import AttributesNSImpl + +from ..constants import adjustForeignAttributes, unadjustForeignAttributes + +prefix_mapping = {} +for prefix, localName, namespace in adjustForeignAttributes.values(): + if prefix is not None: + prefix_mapping[prefix] = namespace + + +def to_sax(walker, handler): + """Call SAX-like content handler based on treewalker walker""" + handler.startDocument() + for prefix, namespace in prefix_mapping.items(): + handler.startPrefixMapping(prefix, namespace) + + for token in walker: + type = token["type"] + if type == "Doctype": + continue + elif type in ("StartTag", "EmptyTag"): + attrs = AttributesNSImpl(token["data"], + unadjustForeignAttributes) + handler.startElementNS((token["namespace"], token["name"]), + token["name"], + attrs) + if type == "EmptyTag": + handler.endElementNS((token["namespace"], token["name"]), + token["name"]) + elif type == "EndTag": + handler.endElementNS((token["namespace"], token["name"]), + token["name"]) + elif type in ("Characters", "SpaceCharacters"): + handler.characters(token["data"]) + elif type == "Comment": + pass + else: + assert False, "Unknown token type" + + for prefix, namespace in prefix_mapping.items(): + handler.endPrefixMapping(prefix) + handler.endDocument() diff --git a/libs/html5lib/treebuilders/__init__.py b/libs/html5lib/treebuilders/__init__.py index 14f66d40..6a6b2a4c 100755 --- a/libs/html5lib/treebuilders/__init__.py +++ b/libs/html5lib/treebuilders/__init__.py @@ -7,7 +7,7 @@ implement several things: 1) A set of classes for various types of elements: Document, Doctype, Comment, Element. These must implement the interface of _base.treebuilders.Node (although comment nodes have a different -signature for their constructor, see treebuilders.simpletree.Comment) +signature for their constructor, see treebuilders.etree.Comment) Textual content may also be implemented as another node type, or not, as your tree implementation requires. @@ -24,73 +24,53 @@ getDocument - Returns the root node of the complete document tree testSerializer method on your treebuilder which accepts a node and returns a string containing Node and its children serialized according to the format used in the unittests - -The supplied simpletree module provides a python-only implementation -of a full treebuilder and is a useful reference for the semantics of -the various methods. """ +from __future__ import absolute_import, division, unicode_literals + +from ..utils import default_etree + treeBuilderCache = {} -import sys def getTreeBuilder(treeType, implementation=None, **kwargs): """Get a TreeBuilder class for various types of tree with built-in support - + treeType - the name of the tree type required (case-insensitive). Supported - values are "simpletree", "dom", "etree" and "beautifulsoup" - - "simpletree" - a built-in DOM-ish tree type with support for some - more pythonic idioms. - "dom" - A generic builder for DOM implementations, defaulting to - a xml.dom.minidom based implementation for the sake of - backwards compatibility (as releases up until 0.10 had a - builder called "dom" that was a minidom implemenation). - "etree" - A generic builder for tree implementations exposing an - elementtree-like interface (known to work with - ElementTree, cElementTree and lxml.etree). - "beautifulsoup" - Beautiful soup (if installed) - + values are: + + "dom" - A generic builder for DOM implementations, defaulting to + a xml.dom.minidom based implementation. + "etree" - A generic builder for tree implementations exposing an + ElementTree-like interface, defaulting to + xml.etree.cElementTree if available and + xml.etree.ElementTree if not. + "lxml" - A etree-based builder for lxml.etree, handling + limitations of lxml's implementation. + implementation - (Currently applies to the "etree" and "dom" tree types). A module implementing the tree type e.g. - xml.etree.ElementTree or lxml.etree.""" - + xml.etree.ElementTree or xml.etree.cElementTree.""" + treeType = treeType.lower() if treeType not in treeBuilderCache: if treeType == "dom": - import dom - # XXX: Keep backwards compatibility by using minidom if no implementation is given - if implementation == None: + from . import dom + # Come up with a sane default (pref. from the stdlib) + if implementation is None: from xml.dom import minidom implementation = minidom - # XXX: NEVER cache here, caching is done in the dom submodule + # NEVER cache here, caching is done in the dom submodule return dom.getDomModule(implementation, **kwargs).TreeBuilder - elif treeType == "simpletree": - import simpletree - treeBuilderCache[treeType] = simpletree.TreeBuilder - elif treeType == "beautifulsoup": - import soup - treeBuilderCache[treeType] = soup.TreeBuilder elif treeType == "lxml": - import etree_lxml + from . import etree_lxml treeBuilderCache[treeType] = etree_lxml.TreeBuilder elif treeType == "etree": - # Come up with a sane default - if implementation == None: - try: - import xml.etree.cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except ImportError: - try: - import cElementTree as ET - except ImportError: - import elementtree.ElementTree as ET - implementation = ET - import etree + from . import etree + if implementation is None: + implementation = default_etree # NEVER cache here, caching is done in the etree submodule return etree.getETreeModule(implementation, **kwargs).TreeBuilder else: - raise ValueError("""Unrecognised treebuilder "%s" """%treeType) + raise ValueError("""Unrecognised treebuilder "%s" """ % treeType) return treeBuilderCache.get(treeType) diff --git a/libs/html5lib/treebuilders/_base.py b/libs/html5lib/treebuilders/_base.py index f3782d28..8b97cc11 100755 --- a/libs/html5lib/treebuilders/_base.py +++ b/libs/html5lib/treebuilders/_base.py @@ -1,25 +1,34 @@ -from html5lib.constants import scopingElements, tableInsertModeElements, namespaces -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from ..constants import scopingElements, tableInsertModeElements, namespaces # The scope markers are inserted when entering object elements, # marquees, table cells, and table captions, and are used to prevent formatting # from "leaking" into tables, object elements, and marquees. Marker = None +listElementsMap = { + None: (frozenset(scopingElements), False), + "button": (frozenset(scopingElements | set([(namespaces["html"], "button")])), False), + "list": (frozenset(scopingElements | set([(namespaces["html"], "ol"), + (namespaces["html"], "ul")])), False), + "table": (frozenset([(namespaces["html"], "html"), + (namespaces["html"], "table")]), False), + "select": (frozenset([(namespaces["html"], "optgroup"), + (namespaces["html"], "option")]), True) +} + + class Node(object): def __init__(self, name): """Node representing an item in the tree. name - The tag name associated with the node parent - The parent of the current node (or None for the document node) - value - The value of the current node (applies to text nodes and + value - The value of the current node (applies to text nodes and comments attributes - a dict holding name, value pairs for attributes of the node - childNodes - a list of child nodes of the current node. This must + childNodes - a list of child nodes of the current node. This must include all elements but not necessarily other node types _flags - A list of miscellaneous flags that can be set on the node """ @@ -30,14 +39,14 @@ class Node(object): self.childNodes = [] self._flags = [] - def __unicode__(self): - attributesStr = " ".join(["%s=\"%s\""%(name, value) - for name, value in - self.attributes.iteritems()]) + def __str__(self): + attributesStr = " ".join(["%s=\"%s\"" % (name, value) + for name, value in + self.attributes.items()]) if attributesStr: - return "<%s %s>"%(self.name,attributesStr) + return "<%s %s>" % (self.name, attributesStr) else: - return "<%s>"%(self.name) + return "<%s>" % (self.name) def __repr__(self): return "<%s>" % (self.name) @@ -48,14 +57,14 @@ class Node(object): raise NotImplementedError def insertText(self, data, insertBefore=None): - """Insert data as text in the current node, positioned before the + """Insert data as text in the current node, positioned before the start of node insertBefore or to the end of the node's text. """ raise NotImplementedError def insertBefore(self, node, refNode): - """Insert node as a child of the current node, before refNode in the - list of child nodes. Raises ValueError if refNode is not a child of + """Insert node as a child of the current node, before refNode in the + list of child nodes. Raises ValueError if refNode is not a child of the current node""" raise NotImplementedError @@ -65,11 +74,11 @@ class Node(object): raise NotImplementedError def reparentChildren(self, newParent): - """Move all the children of the current node to newParent. - This is needed so that trees that don't store text as nodes move the + """Move all the children of the current node to newParent. + This is needed so that trees that don't store text as nodes move the text in the correct way """ - #XXX - should this method be made more general? + # XXX - should this method be made more general? for child in self.childNodes: newParent.appendChild(child) self.childNodes = [] @@ -80,12 +89,12 @@ class Node(object): """ raise NotImplementedError - def hasContent(self): """Return true if the node has children or text, false otherwise """ raise NotImplementedError + class ActiveFormattingElements(list): def append(self, node): equalCount = 0 @@ -103,12 +112,13 @@ class ActiveFormattingElements(list): def nodesEqual(self, node1, node2): if not node1.nameTuple == node2.nameTuple: return False - + if not node1.attributes == node2.attributes: return False - + return True + class TreeBuilder(object): """Base treebuilder implementation documentClass - the class to use for the bottommost node of a document @@ -117,19 +127,19 @@ class TreeBuilder(object): doctypeClass - the class to use for doctypes """ - #Document class + # Document class documentClass = None - #The class to use for creating a node + # The class to use for creating a node elementClass = None - #The class to use for creating comments + # The class to use for creating comments commentClass = None - #The class to use for creating doctypes + # The class to use for creating doctypes doctypeClass = None - - #Fragment class + + # Fragment class fragmentClass = None def __init__(self, namespaceHTMLElements): @@ -138,12 +148,12 @@ class TreeBuilder(object): else: self.defaultNamespace = None self.reset() - + def reset(self): self.openElements = [] self.activeFormattingElements = ActiveFormattingElements() - #XXX - rename these to headElement, formElement + # XXX - rename these to headElement, formElement self.headPointer = None self.formPointer = None @@ -153,30 +163,20 @@ class TreeBuilder(object): def elementInScope(self, target, variant=None): - #If we pass a node in we match that. if we pass a string - #match any node with that name + # If we pass a node in we match that. if we pass a string + # match any node with that name exactNode = hasattr(target, "nameTuple") - listElementsMap = { - None:(scopingElements, False), - "button":(scopingElements | set([(namespaces["html"], "button")]), False), - "list":(scopingElements | set([(namespaces["html"], "ol"), - (namespaces["html"], "ul")]), False), - "table":(set([(namespaces["html"], "html"), - (namespaces["html"], "table")]), False), - "select":(set([(namespaces["html"], "optgroup"), - (namespaces["html"], "option")]), True) - } listElements, invert = listElementsMap[variant] for node in reversed(self.openElements): if (node.name == target and not exactNode or - node == target and exactNode): + node == target and exactNode): return True - elif (invert ^ (node.nameTuple in listElements)): + elif (invert ^ (node.nameTuple in listElements)): return False - assert False # We should never reach this point + assert False # We should never reach this point def reconstructActiveFormattingElements(self): # Within this algorithm the order of steps described in the @@ -196,7 +196,7 @@ class TreeBuilder(object): # Step 6 while entry != Marker and entry not in self.openElements: if i == 0: - #This will be reset to 0 below + # This will be reset to 0 below i = -1 break i -= 1 @@ -209,13 +209,13 @@ class TreeBuilder(object): # Step 8 entry = self.activeFormattingElements[i] - clone = entry.cloneNode() #Mainly to get a new copy of the attributes + clone = entry.cloneNode() # Mainly to get a new copy of the attributes # Step 9 - element = self.insertElement({"type":"StartTag", - "name":clone.name, - "namespace":clone.namespace, - "data":clone.attributes}) + element = self.insertElement({"type": "StartTag", + "name": clone.name, + "namespace": clone.namespace, + "data": clone.attributes}) # Step 10 self.activeFormattingElements[i] = element @@ -260,7 +260,7 @@ class TreeBuilder(object): if parent is None: parent = self.openElements[-1] parent.appendChild(self.commentClass(token["data"])) - + def createElement(self, token): """Create an element but don't insert it anywhere""" name = token["name"] @@ -282,10 +282,10 @@ class TreeBuilder(object): self.insertElement = self.insertElementNormal insertFromTable = property(_getInsertFromTable, _setInsertFromTable) - + def insertElementNormal(self, token): name = token["name"] - assert type(name) == unicode, "Element %s not unicode"%name + assert isinstance(name, text_type), "Element %s not unicode" % name namespace = token.get("namespace", self.defaultNamespace) element = self.elementClass(name, namespace) element.attributes = token["data"] @@ -294,13 +294,13 @@ class TreeBuilder(object): return element def insertElementTable(self, token): - """Create an element and insert it into the tree""" + """Create an element and insert it into the tree""" element = self.createElement(token) if self.openElements[-1].name not in tableInsertModeElements: return self.insertElementNormal(token) else: - #We should be in the InTable mode. This means we want to do - #special magic element rearranging + # We should be in the InTable mode. This means we want to do + # special magic element rearranging parent, insertBefore = self.getTableMisnestedNodePosition() if insertBefore is None: parent.appendChild(element) @@ -315,7 +315,7 @@ class TreeBuilder(object): parent = self.openElements[-1] if (not self.insertFromTable or (self.insertFromTable and - self.openElements[-1].name + self.openElements[-1].name not in tableInsertModeElements)): parent.insertText(data) else: @@ -323,14 +323,14 @@ class TreeBuilder(object): # special magic element rearranging parent, insertBefore = self.getTableMisnestedNodePosition() parent.insertText(data, insertBefore) - + def getTableMisnestedNodePosition(self): """Get the foster parent element, and sibling to insert before (or None) when inserting a misnested table node""" # The foster parent element is the one which comes before the most # recently opened table element # XXX - this is really inelegant - lastTable=None + lastTable = None fosterParent = None insertBefore = None for elm in self.openElements[::-1]: @@ -354,7 +354,7 @@ class TreeBuilder(object): name = self.openElements[-1].name # XXX td, th and tr are not actually needed if (name in frozenset(("dd", "dt", "li", "option", "optgroup", "p", "rp", "rt")) - and name != exclude): + and name != exclude): self.openElements.pop() # XXX This is not entirely what the specification says. We should # investigate it more closely. @@ -363,10 +363,10 @@ class TreeBuilder(object): def getDocument(self): "Return the final tree" return self.document - + def getFragment(self): "Return the final fragment" - #assert self.innerHTML + # assert self.innerHTML fragment = self.fragmentClass() self.openElements[0].reparentChildren(fragment) return fragment diff --git a/libs/html5lib/treebuilders/dom.py b/libs/html5lib/treebuilders/dom.py index 9578da2b..61e5ed79 100644 --- a/libs/html5lib/treebuilders/dom.py +++ b/libs/html5lib/treebuilders/dom.py @@ -1,45 +1,38 @@ +from __future__ import absolute_import, division, unicode_literals -from xml.dom import minidom, Node, XML_NAMESPACE, XMLNS_NAMESPACE -try: - from types import ModuleType -except: - from new import module as ModuleType -import re + +from xml.dom import minidom, Node import weakref -import _base -from html5lib import constants, ihatexml -from html5lib.constants import namespaces +from . import _base +from .. import constants +from ..constants import namespaces +from ..utils import moduleFactoryFactory -moduleCache = {} - -def getDomModule(DomImplementation): - name = "_" + DomImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType(name) - objs = getDomBuilder(DomImplementation) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getDomBuilder(DomImplementation): Dom = DomImplementation + class AttrList(object): def __init__(self, element): self.element = element + def __iter__(self): - return self.element.attributes.items().__iter__() + return list(self.element.attributes.items()).__iter__() + def __setitem__(self, name, value): self.element.setAttribute(name, value) + def __len__(self): - return len(self.element.attributes.items()) + return len(list(self.element.attributes.items())) + def items(self): return [(item[0], item[1]) for item in - self.element.attributes.items()] + list(self.element.attributes.items())] + def keys(self): - return self.element.attributes.keys() + return list(self.element.attributes.keys()) + def __getitem__(self, name): return self.element.getAttribute(name) @@ -48,68 +41,68 @@ def getDomBuilder(DomImplementation): raise NotImplementedError else: return self.element.hasAttribute(name) - + class NodeBuilder(_base.Node): def __init__(self, element): _base.Node.__init__(self, element.nodeName) self.element = element - namespace = property(lambda self:hasattr(self.element, "namespaceURI") + namespace = property(lambda self: hasattr(self.element, "namespaceURI") and self.element.namespaceURI or None) def appendChild(self, node): node.parent = self self.element.appendChild(node.element) - + def insertText(self, data, insertBefore=None): text = self.element.ownerDocument.createTextNode(data) if insertBefore: self.element.insertBefore(text, insertBefore.element) else: self.element.appendChild(text) - + def insertBefore(self, node, refNode): self.element.insertBefore(node.element, refNode.element) node.parent = self - + def removeChild(self, node): if node.element.parentNode == self.element: self.element.removeChild(node.element) node.parent = None - + def reparentChildren(self, newParent): while self.element.hasChildNodes(): child = self.element.firstChild self.element.removeChild(child) newParent.element.appendChild(child) self.childNodes = [] - + def getAttributes(self): return AttrList(self.element) - + def setAttributes(self, attributes): if attributes: - for name, value in attributes.items(): + for name, value in list(attributes.items()): if isinstance(name, tuple): if name[0] is not None: qualifiedName = (name[0] + ":" + name[1]) else: qualifiedName = name[1] - self.element.setAttributeNS(name[2], qualifiedName, + self.element.setAttributeNS(name[2], qualifiedName, value) else: self.element.setAttribute( name, value) attributes = property(getAttributes, setAttributes) - + def cloneNode(self): return NodeBuilder(self.element.cloneNode(False)) - + def hasContent(self): return self.element.hasChildNodes() def getNameTuple(self): - if self.namespace == None: + if self.namespace is None: return namespaces["html"], self.name else: return self.namespace, self.name @@ -118,9 +111,9 @@ def getDomBuilder(DomImplementation): class TreeBuilder(_base.TreeBuilder): def documentClass(self): - self.dom = Dom.getDOMImplementation().createDocument(None,None,None) + self.dom = Dom.getDOMImplementation().createDocument(None, None, None) return weakref.proxy(self) - + def insertDoctype(self, token): name = token["name"] publicId = token["publicId"] @@ -131,7 +124,7 @@ def getDomBuilder(DomImplementation): self.document.appendChild(NodeBuilder(doctype)) if Dom == minidom: doctype.ownerDocument = self.dom - + def elementClass(self, name, namespace=None): if namespace is None and self.defaultNamespace is None: node = self.dom.createElement(name) @@ -139,70 +132,72 @@ def getDomBuilder(DomImplementation): node = self.dom.createElementNS(namespace, name) return NodeBuilder(node) - + def commentClass(self, data): return NodeBuilder(self.dom.createComment(data)) - + def fragmentClass(self): return NodeBuilder(self.dom.createDocumentFragment()) - + def appendChild(self, node): self.dom.appendChild(node.element) - + def testSerializer(self, element): return testSerializer(element) - + def getDocument(self): return self.dom - + def getFragment(self): return _base.TreeBuilder.getFragment(self).element - + def insertText(self, data, parent=None): - data=data - if parent <> self: + data = data + if parent != self: _base.TreeBuilder.insertText(self, data, parent) else: # HACK: allow text nodes as children of the document node if hasattr(self.dom, '_child_node_types'): if not Node.TEXT_NODE in self.dom._child_node_types: - self.dom._child_node_types=list(self.dom._child_node_types) + self.dom._child_node_types = list(self.dom._child_node_types) self.dom._child_node_types.append(Node.TEXT_NODE) self.dom.appendChild(self.dom.createTextNode(data)) - + + implementation = DomImplementation name = None - + def testSerializer(element): element.normalize() rv = [] + def serializeElement(element, indent=0): if element.nodeType == Node.DOCUMENT_TYPE_NODE: if element.name: if element.publicId or element.systemId: publicId = element.publicId or "" systemId = element.systemId or "" - rv.append( """|%s<!DOCTYPE %s "%s" "%s">"""%( - ' '*indent, element.name, publicId, systemId)) + rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" % + (' ' * indent, element.name, publicId, systemId)) else: - rv.append("|%s<!DOCTYPE %s>"%(' '*indent, element.name)) + rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, element.name)) else: - rv.append("|%s<!DOCTYPE >"%(' '*indent,)) + rv.append("|%s<!DOCTYPE >" % (' ' * indent,)) elif element.nodeType == Node.DOCUMENT_NODE: rv.append("#document") elif element.nodeType == Node.DOCUMENT_FRAGMENT_NODE: rv.append("#document-fragment") elif element.nodeType == Node.COMMENT_NODE: - rv.append("|%s<!-- %s -->"%(' '*indent, element.nodeValue)) + rv.append("|%s<!-- %s -->" % (' ' * indent, element.nodeValue)) elif element.nodeType == Node.TEXT_NODE: - rv.append("|%s\"%s\"" %(' '*indent, element.nodeValue)) + rv.append("|%s\"%s\"" % (' ' * indent, element.nodeValue)) else: if (hasattr(element, "namespaceURI") and - element.namespaceURI != None): - name = "%s %s"%(constants.prefixes[element.namespaceURI], - element.nodeName) + element.namespaceURI is not None): + name = "%s %s" % (constants.prefixes[element.namespaceURI], + element.nodeName) else: name = element.nodeName - rv.append("|%s<%s>"%(' '*indent, name)) + rv.append("|%s<%s>" % (' ' * indent, name)) if element.hasAttributes(): attributes = [] for i in range(len(element.attributes)): @@ -211,81 +206,22 @@ def getDomBuilder(DomImplementation): value = attr.value ns = attr.namespaceURI if ns: - name = "%s %s"%(constants.prefixes[ns], attr.localName) + name = "%s %s" % (constants.prefixes[ns], attr.localName) else: name = attr.nodeName attributes.append((name, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) indent += 2 for child in element.childNodes: serializeElement(child, indent) serializeElement(element, 0) - + return "\n".join(rv) - - def dom2sax(node, handler, nsmap={'xml':XML_NAMESPACE}): - if node.nodeType == Node.ELEMENT_NODE: - if not nsmap: - handler.startElement(node.nodeName, node.attributes) - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endElement(node.nodeName) - else: - attributes = dict(node.attributes.itemsNS()) - - # gather namespace declarations - prefixes = [] - for attrname in node.attributes.keys(): - attr = node.getAttributeNode(attrname) - if (attr.namespaceURI == XMLNS_NAMESPACE or - (attr.namespaceURI == None and attr.nodeName.startswith('xmlns'))): - prefix = (attr.nodeName != 'xmlns' and attr.nodeName or None) - handler.startPrefixMapping(prefix, attr.nodeValue) - prefixes.append(prefix) - nsmap = nsmap.copy() - nsmap[prefix] = attr.nodeValue - del attributes[(attr.namespaceURI, attr.nodeName)] - - # apply namespace declarations - for attrname in node.attributes.keys(): - attr = node.getAttributeNode(attrname) - if attr.namespaceURI == None and ':' in attr.nodeName: - prefix = attr.nodeName.split(':')[0] - if nsmap.has_key(prefix): - del attributes[(attr.namespaceURI, attr.nodeName)] - attributes[(nsmap[prefix],attr.nodeName)]=attr.nodeValue - - # SAX events - ns = node.namespaceURI or nsmap.get(None,None) - handler.startElementNS((ns,node.nodeName), node.nodeName, attributes) - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endElementNS((ns, node.nodeName), node.nodeName) - for prefix in prefixes: handler.endPrefixMapping(prefix) - - elif node.nodeType in [Node.TEXT_NODE, Node.CDATA_SECTION_NODE]: - handler.characters(node.nodeValue) - - elif node.nodeType == Node.DOCUMENT_NODE: - handler.startDocument() - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endDocument() - - elif node.nodeType == Node.DOCUMENT_FRAGMENT_NODE: - for child in node.childNodes: dom2sax(child, handler, nsmap) - - else: - # ATTRIBUTE_NODE - # ENTITY_NODE - # PROCESSING_INSTRUCTION_NODE - # COMMENT_NODE - # DOCUMENT_TYPE_NODE - # NOTATION_NODE - pass - + return locals() -# Keep backwards compatibility with things that directly load -# classes/functions from this module -for key, value in getDomModule(minidom).__dict__.items(): - globals()[key] = value + +# The actual means to get a module! +getDomModule = moduleFactoryFactory(getDomBuilder) diff --git a/libs/html5lib/treebuilders/etree.py b/libs/html5lib/treebuilders/etree.py index 95be4755..2c8ed19f 100755 --- a/libs/html5lib/treebuilders/etree.py +++ b/libs/html5lib/treebuilders/etree.py @@ -1,32 +1,21 @@ -try: - from types import ModuleType -except: - from new import module as ModuleType -import re -import types +from __future__ import absolute_import, division, unicode_literals +from six import text_type -import _base -from html5lib import ihatexml -from html5lib import constants -from html5lib.constants import namespaces +import re + +from . import _base +from .. import ihatexml +from .. import constants +from ..constants import namespaces +from ..utils import moduleFactoryFactory tag_regexp = re.compile("{([^}]*)}(.*)") -moduleCache = {} - -def getETreeModule(ElementTreeImplementation, fullTree=False): - name = "_" + ElementTreeImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder") - objs = getETreeBuilder(ElementTreeImplementation, fullTree) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getETreeBuilder(ElementTreeImplementation, fullTree=False): ElementTree = ElementTreeImplementation + ElementTreeCommentType = ElementTree.Comment("asd").tag + class Element(_base.Node): def __init__(self, name, namespace=None): self._name = name @@ -45,16 +34,16 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): if namespace is None: etree_tag = name else: - etree_tag = "{%s}%s"%(namespace, name) + etree_tag = "{%s}%s" % (namespace, name) return etree_tag - + def _setName(self, name): self._name = name self._element.tag = self._getETreeTag(self._name, self._namespace) - + def _getName(self): return self._name - + name = property(_getName, _setName) def _setNamespace(self, namespace): @@ -65,81 +54,82 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): return self._namespace namespace = property(_getNamespace, _setNamespace) - + def _getAttributes(self): return self._element.attrib - + def _setAttributes(self, attributes): - #Delete existing attributes first - #XXX - there may be a better way to do this... - for key in self._element.attrib.keys(): + # Delete existing attributes first + # XXX - there may be a better way to do this... + for key in list(self._element.attrib.keys()): del self._element.attrib[key] - for key, value in attributes.iteritems(): + for key, value in attributes.items(): if isinstance(key, tuple): - name = "{%s}%s"%(key[2], key[1]) + name = "{%s}%s" % (key[2], key[1]) else: name = key self._element.set(name, value) - + attributes = property(_getAttributes, _setAttributes) - + def _getChildNodes(self): - return self._childNodes + return self._childNodes + def _setChildNodes(self, value): del self._element[:] self._childNodes = [] for element in value: self.insertChild(element) - + childNodes = property(_getChildNodes, _setChildNodes) - + def hasContent(self): """Return true if the node has children or text""" return bool(self._element.text or len(self._element)) - + def appendChild(self, node): self._childNodes.append(node) self._element.append(node._element) node.parent = self - + def insertBefore(self, node, refNode): index = list(self._element).index(refNode._element) self._element.insert(index, node._element) node.parent = self - + def removeChild(self, node): self._element.remove(node._element) - node.parent=None - + node.parent = None + def insertText(self, data, insertBefore=None): if not(len(self._element)): if not self._element.text: self._element.text = "" self._element.text += data elif insertBefore is None: - #Insert the text as the tail of the last child element + # Insert the text as the tail of the last child element if not self._element[-1].tail: self._element[-1].tail = "" self._element[-1].tail += data else: - #Insert the text before the specified node + # Insert the text before the specified node children = list(self._element) index = children.index(insertBefore._element) if index > 0: - if not self._element[index-1].tail: - self._element[index-1].tail = "" - self._element[index-1].tail += data + if not self._element[index - 1].tail: + self._element[index - 1].tail = "" + self._element[index - 1].tail += data else: if not self._element.text: self._element.text = "" self._element.text += data - + def cloneNode(self): element = type(self)(self.name, self.namespace) - for name, value in self.attributes.iteritems(): + for name, value in self.attributes.items(): element.attributes[name] = value return element - + def reparentChildren(self, newParent): if newParent.childNodes: newParent.childNodes[-1]._element.tail += self._element.text @@ -150,60 +140,60 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): newParent._element.text += self._element.text self._element.text = "" _base.Node.reparentChildren(self, newParent) - + class Comment(Element): def __init__(self, data): - #Use the superclass constructor to set all properties on the - #wrapper element + # Use the superclass constructor to set all properties on the + # wrapper element self._element = ElementTree.Comment(data) self.parent = None self._childNodes = [] self._flags = [] - + def _getData(self): return self._element.text - + def _setData(self, value): self._element.text = value - + data = property(_getData, _setData) - + class DocumentType(Element): def __init__(self, name, publicId, systemId): - Element.__init__(self, "<!DOCTYPE>") + Element.__init__(self, "<!DOCTYPE>") self._element.text = name self.publicId = publicId self.systemId = systemId def _getPublicId(self): - return self._element.get(u"publicId", "") + return self._element.get("publicId", "") def _setPublicId(self, value): if value is not None: - self._element.set(u"publicId", value) + self._element.set("publicId", value) publicId = property(_getPublicId, _setPublicId) - + def _getSystemId(self): - return self._element.get(u"systemId", "") + return self._element.get("systemId", "") def _setSystemId(self, value): if value is not None: - self._element.set(u"systemId", value) + self._element.set("systemId", value) systemId = property(_getSystemId, _setSystemId) - + class Document(Element): def __init__(self): - Element.__init__(self, "<DOCUMENT_ROOT>") - + Element.__init__(self, "DOCUMENT_ROOT") + class DocumentFragment(Element): def __init__(self): - Element.__init__(self, "<DOCUMENT_FRAGMENT>") - + Element.__init__(self, "DOCUMENT_FRAGMENT") + def testSerializer(element): rv = [] - finalText = None + def serializeElement(element, indent=0): if not(hasattr(element, "tag")): element = element.getroot() @@ -211,20 +201,23 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): if element.get("publicId") or element.get("systemId"): publicId = element.get("publicId") or "" systemId = element.get("systemId") or "" - rv.append( """<!DOCTYPE %s "%s" "%s">"""%( - element.text, publicId, systemId)) - else: - rv.append("<!DOCTYPE %s>"%(element.text,)) - elif element.tag == "<DOCUMENT_ROOT>": + rv.append("""<!DOCTYPE %s "%s" "%s">""" % + (element.text, publicId, systemId)) + else: + rv.append("<!DOCTYPE %s>" % (element.text,)) + elif element.tag == "DOCUMENT_ROOT": rv.append("#document") - if element.text: - rv.append("|%s\"%s\""%(' '*(indent+2), element.text)) - if element.tail: - finalText = element.tail - elif element.tag == ElementTree.Comment: - rv.append("|%s<!-- %s -->"%(' '*indent, element.text)) + if element.text is not None: + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) + if element.tail is not None: + raise TypeError("Document node cannot have tail") + if hasattr(element, "attrib") and len(element.attrib): + raise TypeError("Document node cannot have attributes") + elif element.tag == ElementTreeCommentType: + rv.append("|%s<!-- %s -->" % (' ' * indent, element.text)) else: - assert type(element.tag) in types.StringTypes, "Expected unicode, got %s"%type(element.tag) + assert isinstance(element.tag, text_type), \ + "Expected unicode, got %s, %s" % (type(element.tag), element.tag) nsmatch = tag_regexp.match(element.tag) if nsmatch is None: @@ -232,113 +225,113 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): else: ns, name = nsmatch.groups() prefix = constants.prefixes[ns] - name = "%s %s"%(prefix, name) - rv.append("|%s<%s>"%(' '*indent, name)) + name = "%s %s" % (prefix, name) + rv.append("|%s<%s>" % (' ' * indent, name)) if hasattr(element, "attrib"): attributes = [] - for name, value in element.attrib.iteritems(): + for name, value in element.attrib.items(): nsmatch = tag_regexp.match(name) if nsmatch is not None: ns, name = nsmatch.groups() prefix = constants.prefixes[ns] - attr_string = "%s %s"%(prefix, name) + attr_string = "%s %s" % (prefix, name) else: attr_string = name attributes.append((attr_string, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) if element.text: - rv.append("|%s\"%s\"" %(' '*(indent+2), element.text)) + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) indent += 2 for child in element: serializeElement(child, indent) if element.tail: - rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail)) + rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail)) serializeElement(element, 0) - - if finalText is not None: - rv.append("|%s\"%s\""%(' '*2, finalText)) - + return "\n".join(rv) - + def tostring(element): """Serialize an element and its child nodes to a string""" rv = [] - finalText = None filter = ihatexml.InfosetFilter() + def serializeElement(element): - if type(element) == type(ElementTree.ElementTree): + if isinstance(element, ElementTree.ElementTree): element = element.getroot() - + if element.tag == "<!DOCTYPE>": if element.get("publicId") or element.get("systemId"): publicId = element.get("publicId") or "" systemId = element.get("systemId") or "" - rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%( - element.text, publicId, systemId)) - else: - rv.append("<!DOCTYPE %s>"%(element.text,)) - elif element.tag == "<DOCUMENT_ROOT>": - if element.text: - rv.append(element.text) - if element.tail: - finalText = element.tail - - for child in element: - serializeElement(child) - - elif type(element.tag) == type(ElementTree.Comment): - rv.append("<!--%s-->"%(element.text,)) - else: - #This is assumed to be an ordinary element - if not element.attrib: - rv.append("<%s>"%(filter.fromXmlName(element.tag),)) + rv.append("""<!DOCTYPE %s PUBLIC "%s" "%s">""" % + (element.text, publicId, systemId)) else: - attr = " ".join(["%s=\"%s\""%( - filter.fromXmlName(name), value) - for name, value in element.attrib.iteritems()]) - rv.append("<%s %s>"%(element.tag, attr)) - if element.text: + rv.append("<!DOCTYPE %s>" % (element.text,)) + elif element.tag == "DOCUMENT_ROOT": + if element.text is not None: rv.append(element.text) - + if element.tail is not None: + raise TypeError("Document node cannot have tail") + if hasattr(element, "attrib") and len(element.attrib): + raise TypeError("Document node cannot have attributes") + for child in element: serializeElement(child) - - rv.append("</%s>"%(element.tag,)) - + + elif element.tag == ElementTreeCommentType: + rv.append("<!--%s-->" % (element.text,)) + else: + # This is assumed to be an ordinary element + if not element.attrib: + rv.append("<%s>" % (filter.fromXmlName(element.tag),)) + else: + attr = " ".join(["%s=\"%s\"" % ( + filter.fromXmlName(name), value) + for name, value in element.attrib.items()]) + rv.append("<%s %s>" % (element.tag, attr)) + if element.text: + rv.append(element.text) + + for child in element: + serializeElement(child) + + rv.append("</%s>" % (element.tag,)) + if element.tail: rv.append(element.tail) - + serializeElement(element) - - if finalText is not None: - rv.append("%s\""%(' '*2, finalText)) - + return "".join(rv) - + class TreeBuilder(_base.TreeBuilder): documentClass = Document doctypeClass = DocumentType elementClass = Element commentClass = Comment fragmentClass = DocumentFragment - + implementation = ElementTreeImplementation + def testSerializer(self, element): return testSerializer(element) - + def getDocument(self): if fullTree: return self.document._element else: if self.defaultNamespace is not None: return self.document._element.find( - "{%s}html"%self.defaultNamespace) + "{%s}html" % self.defaultNamespace) else: return self.document._element.find("html") - + def getFragment(self): return _base.TreeBuilder.getFragment(self)._element - + return locals() + + +getETreeModule = moduleFactoryFactory(getETreeBuilder) diff --git a/libs/html5lib/treebuilders/etree_lxml.py b/libs/html5lib/treebuilders/etree_lxml.py index eee1e3b2..35d08efa 100644 --- a/libs/html5lib/treebuilders/etree_lxml.py +++ b/libs/html5lib/treebuilders/etree_lxml.py @@ -1,20 +1,3 @@ -import warnings -import re - -import _base -from html5lib.constants import DataLossWarning -import html5lib.constants as constants -import etree as etree_builders -from html5lib import ihatexml - -try: - import lxml.etree as etree -except ImportError: - pass - -fullTree = True -tag_regexp = re.compile("{([^}]*)}(.*)") - """Module for supporting the lxml.etree library. The idea here is to use as much of the native library as possible, without using fragile hacks like custom element names that break between releases. The downside of this is that we cannot represent @@ -26,12 +9,34 @@ Docypes with no name When any of these things occur, we emit a DataLossWarning """ +from __future__ import absolute_import, division, unicode_literals + +import warnings +import re +import sys + +from . import _base +from ..constants import DataLossWarning +from .. import constants +from . import etree as etree_builders +from .. import ihatexml + +import lxml.etree as etree + + +fullTree = True +tag_regexp = re.compile("{([^}]*)}(.*)") + +comment_type = etree.Comment("asd").tag + + class DocumentType(object): def __init__(self, name, publicId, systemId): - self.name = name + self.name = name self.publicId = publicId self.systemId = systemId + class Document(object): def __init__(self): self._elementTree = None @@ -42,118 +47,126 @@ class Document(object): def _getChildNodes(self): return self._childNodes - + childNodes = property(_getChildNodes) + def testSerializer(element): rv = [] finalText = None - filter = ihatexml.InfosetFilter() + infosetFilter = ihatexml.InfosetFilter() + def serializeElement(element, indent=0): if not hasattr(element, "tag"): - if hasattr(element, "getroot"): - #Full tree case + if hasattr(element, "getroot"): + # Full tree case rv.append("#document") if element.docinfo.internalDTD: - if not (element.docinfo.public_id or + if not (element.docinfo.public_id or element.docinfo.system_url): - dtd_str = "<!DOCTYPE %s>"%element.docinfo.root_name + dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name else: - dtd_str = """<!DOCTYPE %s "%s" "%s">"""%( - element.docinfo.root_name, + dtd_str = """<!DOCTYPE %s "%s" "%s">""" % ( + element.docinfo.root_name, element.docinfo.public_id, element.docinfo.system_url) - rv.append("|%s%s"%(' '*(indent+2), dtd_str)) + rv.append("|%s%s" % (' ' * (indent + 2), dtd_str)) next_element = element.getroot() while next_element.getprevious() is not None: next_element = next_element.getprevious() while next_element is not None: - serializeElement(next_element, indent+2) + serializeElement(next_element, indent + 2) next_element = next_element.getnext() - elif isinstance(element, basestring): - #Text in a fragment - rv.append("|%s\"%s\""%(' '*indent, element)) + elif isinstance(element, str) or isinstance(element, bytes): + # Text in a fragment + assert isinstance(element, str) or sys.version_info.major == 2 + rv.append("|%s\"%s\"" % (' ' * indent, element)) else: - #Fragment case + # Fragment case rv.append("#document-fragment") for next_element in element: - serializeElement(next_element, indent+2) - elif type(element.tag) == type(etree.Comment): - rv.append("|%s<!-- %s -->"%(' '*indent, element.text)) + serializeElement(next_element, indent + 2) + elif element.tag == comment_type: + rv.append("|%s<!-- %s -->" % (' ' * indent, element.text)) + if hasattr(element, "tail") and element.tail: + rv.append("|%s\"%s\"" % (' ' * indent, element.tail)) else: + assert isinstance(element, etree._Element) nsmatch = etree_builders.tag_regexp.match(element.tag) if nsmatch is not None: ns = nsmatch.group(1) tag = nsmatch.group(2) prefix = constants.prefixes[ns] - rv.append("|%s<%s %s>"%(' '*indent, prefix, - filter.fromXmlName(tag))) + rv.append("|%s<%s %s>" % (' ' * indent, prefix, + infosetFilter.fromXmlName(tag))) else: - rv.append("|%s<%s>"%(' '*indent, - filter.fromXmlName(element.tag))) + rv.append("|%s<%s>" % (' ' * indent, + infosetFilter.fromXmlName(element.tag))) if hasattr(element, "attrib"): attributes = [] - for name, value in element.attrib.iteritems(): + for name, value in element.attrib.items(): nsmatch = tag_regexp.match(name) if nsmatch is not None: ns, name = nsmatch.groups() - name = filter.fromXmlName(name) + name = infosetFilter.fromXmlName(name) prefix = constants.prefixes[ns] - attr_string = "%s %s"%(prefix, name) + attr_string = "%s %s" % (prefix, name) else: - attr_string = filter.fromXmlName(name) + attr_string = infosetFilter.fromXmlName(name) attributes.append((attr_string, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) if element.text: - rv.append("|%s\"%s\"" %(' '*(indent+2), element.text)) + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) indent += 2 - for child in element.getchildren(): + for child in element: serializeElement(child, indent) - if hasattr(element, "tail") and element.tail: - rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail)) + if hasattr(element, "tail") and element.tail: + rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail)) serializeElement(element, 0) if finalText is not None: - rv.append("|%s\"%s\""%(' '*2, finalText)) + rv.append("|%s\"%s\"" % (' ' * 2, finalText)) return "\n".join(rv) + def tostring(element): """Serialize an element and its child nodes to a string""" rv = [] finalText = None + def serializeElement(element): if not hasattr(element, "tag"): if element.docinfo.internalDTD: if element.docinfo.doctype: dtd_str = element.docinfo.doctype else: - dtd_str = "<!DOCTYPE %s>"%element.docinfo.root_name + dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name rv.append(dtd_str) serializeElement(element.getroot()) - - elif type(element.tag) == type(etree.Comment): - rv.append("<!--%s-->"%(element.text,)) - + + elif element.tag == comment_type: + rv.append("<!--%s-->" % (element.text,)) + else: - #This is assumed to be an ordinary element + # This is assumed to be an ordinary element if not element.attrib: - rv.append("<%s>"%(element.tag,)) + rv.append("<%s>" % (element.tag,)) else: - attr = " ".join(["%s=\"%s\""%(name, value) - for name, value in element.attrib.iteritems()]) - rv.append("<%s %s>"%(element.tag, attr)) + attr = " ".join(["%s=\"%s\"" % (name, value) + for name, value in element.attrib.items()]) + rv.append("<%s %s>" % (element.tag, attr)) if element.text: rv.append(element.text) - for child in element.getchildren(): + for child in element: serializeElement(child) - rv.append("</%s>"%(element.tag,)) + rv.append("</%s>" % (element.tag,)) if hasattr(element, "tail") and element.tail: rv.append(element.tail) @@ -161,56 +174,57 @@ def tostring(element): serializeElement(element) if finalText is not None: - rv.append("%s\""%(' '*2, finalText)) + rv.append("%s\"" % (' ' * 2, finalText)) return "".join(rv) - + class TreeBuilder(_base.TreeBuilder): documentClass = Document doctypeClass = DocumentType elementClass = None commentClass = None - fragmentClass = Document + fragmentClass = Document + implementation = etree - def __init__(self, namespaceHTMLElements, fullTree = False): + def __init__(self, namespaceHTMLElements, fullTree=False): builder = etree_builders.getETreeModule(etree, fullTree=fullTree) - filter = self.filter = ihatexml.InfosetFilter() + infosetFilter = self.infosetFilter = ihatexml.InfosetFilter() self.namespaceHTMLElements = namespaceHTMLElements class Attributes(dict): def __init__(self, element, value={}): self._element = element dict.__init__(self, value) - for key, value in self.iteritems(): + for key, value in self.items(): if isinstance(key, tuple): - name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1])) + name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1])) else: - name = filter.coerceAttribute(key) + name = infosetFilter.coerceAttribute(key) self._element._element.attrib[name] = value def __setitem__(self, key, value): dict.__setitem__(self, key, value) if isinstance(key, tuple): - name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1])) + name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1])) else: - name = filter.coerceAttribute(key) + name = infosetFilter.coerceAttribute(key) self._element._element.attrib[name] = value class Element(builder.Element): def __init__(self, name, namespace): - name = filter.coerceElement(name) + name = infosetFilter.coerceElement(name) builder.Element.__init__(self, name, namespace=namespace) self._attributes = Attributes(self) def _setName(self, name): - self._name = filter.coerceElement(name) + self._name = infosetFilter.coerceElement(name) self._element.tag = self._getETreeTag( self._name, self._namespace) - + def _getName(self): - return filter.fromXmlName(self._name) - + return infosetFilter.fromXmlName(self._name) + name = property(_getName, _setName) def _getAttributes(self): @@ -218,24 +232,23 @@ class TreeBuilder(_base.TreeBuilder): def _setAttributes(self, attributes): self._attributes = Attributes(self, attributes) - + attributes = property(_getAttributes, _setAttributes) def insertText(self, data, insertBefore=None): - data = filter.coerceCharacters(data) + data = infosetFilter.coerceCharacters(data) builder.Element.insertText(self, data, insertBefore) def appendChild(self, child): builder.Element.appendChild(self, child) - class Comment(builder.Comment): def __init__(self, data): - data = filter.coerceComment(data) + data = infosetFilter.coerceComment(data) builder.Comment.__init__(self, data) def _setData(self, data): - data = filter.coerceComment(data) + data = infosetFilter.coerceComment(data) self._element.text = data def _getData(self): @@ -245,9 +258,9 @@ class TreeBuilder(_base.TreeBuilder): self.elementClass = Element self.commentClass = builder.Comment - #self.fragmentClass = builder.DocumentFragment + # self.fragmentClass = builder.DocumentFragment _base.TreeBuilder.__init__(self, namespaceHTMLElements) - + def reset(self): _base.TreeBuilder.reset(self) self.insertComment = self.insertCommentInitial @@ -262,13 +275,13 @@ class TreeBuilder(_base.TreeBuilder): return self.document._elementTree else: return self.document._elementTree.getroot() - + def getFragment(self): fragment = [] element = self.openElements[0]._element if element.text: fragment.append(element.text) - fragment.extend(element.getchildren()) + fragment.extend(list(element)) if element.tail: fragment.append(element.tail) return fragment @@ -278,59 +291,79 @@ class TreeBuilder(_base.TreeBuilder): publicId = token["publicId"] systemId = token["systemId"] - if not name or ihatexml.nonXmlNameBMPRegexp.search(name) or name[0] == '"': - warnings.warn("lxml cannot represent null or non-xml doctype", DataLossWarning) + if not name: + warnings.warn("lxml cannot represent empty doctype", DataLossWarning) + self.doctype = None + else: + coercedName = self.infosetFilter.coerceElement(name) + if coercedName != name: + warnings.warn("lxml cannot represent non-xml doctype", DataLossWarning) + + doctype = self.doctypeClass(coercedName, publicId, systemId) + self.doctype = doctype - doctype = self.doctypeClass(name, publicId, systemId) - self.doctype = doctype - def insertCommentInitial(self, data, parent=None): self.initial_comments.append(data) - + + def insertCommentMain(self, data, parent=None): + if (parent == self.document and + self.document._elementTree.getroot()[-1].tag == comment_type): + warnings.warn("lxml cannot represent adjacent comments beyond the root elements", DataLossWarning) + super(TreeBuilder, self).insertComment(data, parent) + def insertRoot(self, token): """Create the document root""" - #Because of the way libxml2 works, it doesn't seem to be possible to - #alter information like the doctype after the tree has been parsed. - #Therefore we need to use the built-in parser to create our iniial - #tree, after which we can add elements like normal + # Because of the way libxml2 works, it doesn't seem to be possible to + # alter information like the doctype after the tree has been parsed. + # Therefore we need to use the built-in parser to create our iniial + # tree, after which we can add elements like normal docStr = "" - if self.doctype and self.doctype.name and not self.doctype.name.startswith('"'): - docStr += "<!DOCTYPE %s"%self.doctype.name - if (self.doctype.publicId is not None or - self.doctype.systemId is not None): - docStr += ' PUBLIC "%s" "%s"'%(self.doctype.publicId or "", - self.doctype.systemId or "") + if self.doctype: + assert self.doctype.name + docStr += "<!DOCTYPE %s" % self.doctype.name + if (self.doctype.publicId is not None or + self.doctype.systemId is not None): + docStr += (' PUBLIC "%s" ' % + (self.infosetFilter.coercePubid(self.doctype.publicId or ""))) + if self.doctype.systemId: + sysid = self.doctype.systemId + if sysid.find("'") >= 0 and sysid.find('"') >= 0: + warnings.warn("DOCTYPE system cannot contain single and double quotes", DataLossWarning) + sysid = sysid.replace("'", 'U00027') + if sysid.find("'") >= 0: + docStr += '"%s"' % sysid + else: + docStr += "'%s'" % sysid + else: + docStr += "''" docStr += ">" + if self.doctype.name != token["name"]: + warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning) docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>" - - try: - root = etree.fromstring(docStr) - except etree.XMLSyntaxError: - print docStr - raise - - #Append the initial comments: + root = etree.fromstring(docStr) + + # Append the initial comments: for comment_token in self.initial_comments: root.addprevious(etree.Comment(comment_token["data"])) - - #Create the root document and add the ElementTree to it + + # Create the root document and add the ElementTree to it self.document = self.documentClass() self.document._elementTree = root.getroottree() - + # Give the root element the right name name = token["name"] namespace = token.get("namespace", self.defaultNamespace) if namespace is None: etree_tag = name else: - etree_tag = "{%s}%s"%(namespace, name) + etree_tag = "{%s}%s" % (namespace, name) root.tag = etree_tag - - #Add the root element to the internal child/open data structures + + # Add the root element to the internal child/open data structures root_element = self.elementClass(name, namespace) root_element._element = root self.document._childNodes.append(root_element) self.openElements.append(root_element) - - #Reset to the default insert comment function - self.insertComment = super(TreeBuilder, self).insertComment + + # Reset to the default insert comment function + self.insertComment = self.insertCommentMain diff --git a/libs/html5lib/treebuilders/simpletree.py b/libs/html5lib/treebuilders/simpletree.py deleted file mode 100755 index 67fe7583..00000000 --- a/libs/html5lib/treebuilders/simpletree.py +++ /dev/null @@ -1,256 +0,0 @@ -import _base -from html5lib.constants import voidElements, namespaces, prefixes -from xml.sax.saxutils import escape - -# Really crappy basic implementation of a DOM-core like thing -class Node(_base.Node): - type = -1 - def __init__(self, name): - self.name = name - self.parent = None - self.value = None - self.childNodes = [] - self._flags = [] - - def __iter__(self): - for node in self.childNodes: - yield node - for item in node: - yield item - - def __unicode__(self): - return self.name - - def toxml(self): - raise NotImplementedError - - def printTree(self, indent=0): - tree = '\n|%s%s' % (' '* indent, unicode(self)) - for child in self.childNodes: - tree += child.printTree(indent + 2) - return tree - - def appendChild(self, node): - assert isinstance(node, Node) - if (isinstance(node, TextNode) and self.childNodes and - isinstance(self.childNodes[-1], TextNode)): - self.childNodes[-1].value += node.value - else: - self.childNodes.append(node) - node.parent = self - - def insertText(self, data, insertBefore=None): - assert isinstance(data, unicode), "data %s is of type %s expected unicode"%(repr(data), type(data)) - if insertBefore is None: - self.appendChild(TextNode(data)) - else: - self.insertBefore(TextNode(data), insertBefore) - - def insertBefore(self, node, refNode): - index = self.childNodes.index(refNode) - if (isinstance(node, TextNode) and index > 0 and - isinstance(self.childNodes[index - 1], TextNode)): - self.childNodes[index - 1].value += node.value - else: - self.childNodes.insert(index, node) - node.parent = self - - def removeChild(self, node): - try: - self.childNodes.remove(node) - except: - # XXX - raise - node.parent = None - - def cloneNode(self): - raise NotImplementedError - - def hasContent(self): - """Return true if the node has children or text""" - return bool(self.childNodes) - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class Document(Node): - type = 1 - def __init__(self): - Node.__init__(self, None) - - def __str__(self): - return "#document" - - def __unicode__(self): - return str(self) - - def appendChild(self, child): - Node.appendChild(self, child) - - def toxml(self, encoding="utf=8"): - result = "" - for child in self.childNodes: - result += child.toxml() - return result.encode(encoding) - - def hilite(self, encoding="utf-8"): - result = "<pre>" - for child in self.childNodes: - result += child.hilite() - return result.encode(encoding) + "</pre>" - - def printTree(self): - tree = unicode(self) - for child in self.childNodes: - tree += child.printTree(2) - return tree - - def cloneNode(self): - return Document() - -class DocumentFragment(Document): - type = 2 - def __str__(self): - return "#document-fragment" - - def __unicode__(self): - return str(self) - - def cloneNode(self): - return DocumentFragment() - -class DocumentType(Node): - type = 3 - def __init__(self, name, publicId, systemId): - Node.__init__(self, name) - self.publicId = publicId - self.systemId = systemId - - def __unicode__(self): - if self.publicId or self.systemId: - publicId = self.publicId or "" - systemId = self.systemId or "" - return """<!DOCTYPE %s "%s" "%s">"""%( - self.name, publicId, systemId) - - else: - return u"<!DOCTYPE %s>" % self.name - - - toxml = __unicode__ - - def hilite(self): - return '<code class="markup doctype"><!DOCTYPE %s></code>' % self.name - - def cloneNode(self): - return DocumentType(self.name, self.publicId, self.systemId) - -class TextNode(Node): - type = 4 - def __init__(self, value): - Node.__init__(self, None) - self.value = value - - def __unicode__(self): - return u"\"%s\"" % self.value - - def toxml(self): - return escape(self.value) - - hilite = toxml - - def cloneNode(self): - return TextNode(self.value) - -class Element(Node): - type = 5 - def __init__(self, name, namespace=None): - Node.__init__(self, name) - self.namespace = namespace - self.attributes = {} - - def __unicode__(self): - if self.namespace == None: - return u"<%s>" % self.name - else: - return u"<%s %s>"%(prefixes[self.namespace], self.name) - - def toxml(self): - result = '<' + self.name - if self.attributes: - for name,value in self.attributes.iteritems(): - result += u' %s="%s"' % (name, escape(value,{'"':'"'})) - if self.childNodes: - result += '>' - for child in self.childNodes: - result += child.toxml() - result += u'</%s>' % self.name - else: - result += u'/>' - return result - - def hilite(self): - result = '<<code class="markup element-name">%s</code>' % self.name - if self.attributes: - for name, value in self.attributes.iteritems(): - result += ' <code class="markup attribute-name">%s</code>=<code class="markup attribute-value">"%s"</code>' % (name, escape(value, {'"':'"'})) - if self.childNodes: - result += ">" - for child in self.childNodes: - result += child.hilite() - elif self.name in voidElements: - return result + ">" - return result + '</<code class="markup element-name">%s</code>>' % self.name - - def printTree(self, indent): - tree = '\n|%s%s' % (' '*indent, unicode(self)) - indent += 2 - if self.attributes: - for name, value in sorted(self.attributes.iteritems()): - if isinstance(name, tuple): - name = "%s %s"%(name[0], name[1]) - tree += '\n|%s%s="%s"' % (' ' * indent, name, value) - for child in self.childNodes: - tree += child.printTree(indent) - return tree - - def cloneNode(self): - newNode = Element(self.name) - if hasattr(self, 'namespace'): - newNode.namespace = self.namespace - for attr, value in self.attributes.iteritems(): - newNode.attributes[attr] = value - return newNode - -class CommentNode(Node): - type = 6 - def __init__(self, data): - Node.__init__(self, None) - self.data = data - - def __unicode__(self): - return "<!-- %s -->" % self.data - - def toxml(self): - return "<!--%s-->" % self.data - - def hilite(self): - return '<code class="markup comment"><!--%s--></code>' % escape(self.data) - - def cloneNode(self): - return CommentNode(self.data) - -class TreeBuilder(_base.TreeBuilder): - documentClass = Document - doctypeClass = DocumentType - elementClass = Element - commentClass = CommentNode - fragmentClass = DocumentFragment - - def testSerializer(self, node): - return node.printTree() diff --git a/libs/html5lib/treebuilders/soup.py b/libs/html5lib/treebuilders/soup.py deleted file mode 100644 index 9bc5ff0e..00000000 --- a/libs/html5lib/treebuilders/soup.py +++ /dev/null @@ -1,236 +0,0 @@ -import warnings - -warnings.warn("BeautifulSoup 3.x (as of 3.1) is not fully compatible with html5lib and support will be removed in the future", DeprecationWarning) - -from BeautifulSoup import BeautifulSoup, Tag, NavigableString, Comment, Declaration - -import _base -from html5lib.constants import namespaces, DataLossWarning - -class AttrList(object): - def __init__(self, element): - self.element = element - self.attrs = dict(self.element.attrs) - def __iter__(self): - return self.attrs.items().__iter__() - def __setitem__(self, name, value): - "set attr", name, value - self.element[name] = value - def items(self): - return self.attrs.items() - def keys(self): - return self.attrs.keys() - def __getitem__(self, name): - return self.attrs[name] - def __contains__(self, name): - return name in self.attrs.keys() - def __eq__(self, other): - if len(self.keys()) != len(other.keys()): - return False - for item in self.keys(): - if item not in other: - return False - if self[item] != other[item]: - return False - return True - -class Element(_base.Node): - def __init__(self, element, soup, namespace): - _base.Node.__init__(self, element.name) - self.element = element - self.soup = soup - self.namespace = namespace - - def _nodeIndex(self, node, refNode): - # Finds a node by identity rather than equality - for index in range(len(self.element.contents)): - if id(self.element.contents[index]) == id(refNode.element): - return index - return None - - def appendChild(self, node): - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[-1].__class__ == NavigableString): - # Concatenate new text onto old text node - # (TODO: This has O(n^2) performance, for input like "a</a>a</a>a</a>...") - newStr = NavigableString(self.element.contents[-1]+node.element) - - # Remove the old text node - # (Can't simply use .extract() by itself, because it fails if - # an equal text node exists within the parent node) - oldElement = self.element.contents[-1] - del self.element.contents[-1] - oldElement.parent = None - oldElement.extract() - - self.element.insert(len(self.element.contents), newStr) - else: - self.element.insert(len(self.element.contents), node.element) - node.parent = self - - def getAttributes(self): - return AttrList(self.element) - - def setAttributes(self, attributes): - if attributes: - for name, value in attributes.items(): - self.element[name] = value - - attributes = property(getAttributes, setAttributes) - - def insertText(self, data, insertBefore=None): - text = TextNode(NavigableString(data), self.soup) - if insertBefore: - self.insertBefore(text, insertBefore) - else: - self.appendChild(text) - - def insertBefore(self, node, refNode): - index = self._nodeIndex(node, refNode) - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[index-1].__class__ == NavigableString): - # (See comments in appendChild) - newStr = NavigableString(self.element.contents[index-1]+node.element) - oldNode = self.element.contents[index-1] - del self.element.contents[index-1] - oldNode.parent = None - oldNode.extract() - - self.element.insert(index-1, newStr) - else: - self.element.insert(index, node.element) - node.parent = self - - def removeChild(self, node): - index = self._nodeIndex(node.parent, node) - del node.parent.element.contents[index] - node.element.parent = None - node.element.extract() - node.parent = None - - def reparentChildren(self, newParent): - while self.element.contents: - child = self.element.contents[0] - child.extract() - if isinstance(child, Tag): - newParent.appendChild(Element(child, self.soup, namespaces["html"])) - else: - newParent.appendChild(TextNode(child, self.soup)) - - def cloneNode(self): - node = Element(Tag(self.soup, self.element.name), self.soup, self.namespace) - for key,value in self.attributes: - node.attributes[key] = value - return node - - def hasContent(self): - return self.element.contents - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class TextNode(Element): - def __init__(self, element, soup): - _base.Node.__init__(self, None) - self.element = element - self.soup = soup - - def cloneNode(self): - raise NotImplementedError - -class TreeBuilder(_base.TreeBuilder): - def __init__(self, namespaceHTMLElements): - if namespaceHTMLElements: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - _base.TreeBuilder.__init__(self, namespaceHTMLElements) - - def documentClass(self): - self.soup = BeautifulSoup("") - return Element(self.soup, self.soup, None) - - def insertDoctype(self, token): - name = token["name"] - publicId = token["publicId"] - systemId = token["systemId"] - - if publicId: - self.soup.insert(0, Declaration("DOCTYPE %s PUBLIC \"%s\" \"%s\""%(name, publicId, systemId or ""))) - elif systemId: - self.soup.insert(0, Declaration("DOCTYPE %s SYSTEM \"%s\""% - (name, systemId))) - else: - self.soup.insert(0, Declaration("DOCTYPE %s"%name)) - - def elementClass(self, name, namespace): - if namespace is not None: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - return Element(Tag(self.soup, name), self.soup, namespace) - - def commentClass(self, data): - return TextNode(Comment(data), self.soup) - - def fragmentClass(self): - self.soup = BeautifulSoup("") - self.soup.name = "[document_fragment]" - return Element(self.soup, self.soup, None) - - def appendChild(self, node): - self.soup.insert(len(self.soup.contents), node.element) - - def testSerializer(self, element): - return testSerializer(element) - - def getDocument(self): - return self.soup - - def getFragment(self): - return _base.TreeBuilder.getFragment(self).element - -def testSerializer(element): - import re - rv = [] - def serializeElement(element, indent=0): - if isinstance(element, Declaration): - doctype_regexp = r'DOCTYPE\s+(?P<name>[^\s]*)( PUBLIC "(?P<publicId>.*)" "(?P<systemId1>.*)"| SYSTEM "(?P<systemId2>.*)")?' - m = re.compile(doctype_regexp).match(element.string) - assert m is not None, "DOCTYPE did not match expected format" - name = m.group('name') - publicId = m.group('publicId') - if publicId is not None: - systemId = m.group('systemId1') or "" - else: - systemId = m.group('systemId2') - - if publicId is not None or systemId is not None: - rv.append("""|%s<!DOCTYPE %s "%s" "%s">"""% - (' '*indent, name, publicId or "", systemId or "")) - else: - rv.append("|%s<!DOCTYPE %s>"%(' '*indent, name)) - - elif isinstance(element, BeautifulSoup): - if element.name == "[document_fragment]": - rv.append("#document-fragment") - else: - rv.append("#document") - - elif isinstance(element, Comment): - rv.append("|%s<!-- %s -->"%(' '*indent, element.string)) - elif isinstance(element, unicode): - rv.append("|%s\"%s\"" %(' '*indent, element)) - else: - rv.append("|%s<%s>"%(' '*indent, element.name)) - if element.attrs: - for name, value in sorted(element.attrs): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) - indent += 2 - if hasattr(element, "contents"): - for child in element.contents: - serializeElement(child, indent) - serializeElement(element, 0) - - return "\n".join(rv) diff --git a/libs/html5lib/treewalkers/__init__.py b/libs/html5lib/treewalkers/__init__.py index 3a606a8b..18124e75 100644 --- a/libs/html5lib/treewalkers/__init__.py +++ b/libs/html5lib/treewalkers/__init__.py @@ -8,23 +8,27 @@ implements a 'serialize' method taking a tree as sole argument and returning an iterator generating tokens. """ +from __future__ import absolute_import, division, unicode_literals + +import sys + +from ..utils import default_etree + treeWalkerCache = {} + def getTreeWalker(treeType, implementation=None, **kwargs): """Get a TreeWalker class for various types of tree with built-in support treeType - the name of the tree type required (case-insensitive). Supported - values are "simpletree", "dom", "etree" and "beautifulsoup" + values are: - "simpletree" - a built-in DOM-ish tree type with support for some - more pythonic idioms. "dom" - The xml.dom.minidom DOM implementation "pulldom" - The xml.dom.pulldom event stream "etree" - A generic walker for tree implementations exposing an elementtree-like interface (known to work with ElementTree, cElementTree and lxml.etree). "lxml" - Optimized walker for lxml.etree - "beautifulsoup" - Beautiful soup (if installed) "genshi" - a Genshi stream implementation - (Currently applies to the "etree" tree type only). A module @@ -33,20 +37,21 @@ def getTreeWalker(treeType, implementation=None, **kwargs): treeType = treeType.lower() if treeType not in treeWalkerCache: - if treeType in ("dom", "pulldom", "simpletree"): - mod = __import__(treeType, globals()) + if treeType in ("dom", "pulldom"): + name = "%s.%s" % (__name__, treeType) + __import__(name) + mod = sys.modules[name] treeWalkerCache[treeType] = mod.TreeWalker elif treeType == "genshi": - import genshistream + from . import genshistream treeWalkerCache[treeType] = genshistream.TreeWalker - elif treeType == "beautifulsoup": - import soup - treeWalkerCache[treeType] = soup.TreeWalker elif treeType == "lxml": - import lxmletree + from . import lxmletree treeWalkerCache[treeType] = lxmletree.TreeWalker elif treeType == "etree": - import etree + from . import etree + if implementation is None: + implementation = default_etree # XXX: NEVER cache here, caching is done in the etree submodule return etree.getETreeModule(implementation, **kwargs).TreeWalker return treeWalkerCache.get(treeType) diff --git a/libs/html5lib/treewalkers/_base.py b/libs/html5lib/treewalkers/_base.py index 5929ba05..34252e50 100644 --- a/libs/html5lib/treewalkers/_base.py +++ b/libs/html5lib/treewalkers/_base.py @@ -1,94 +1,9 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type, string_types + import gettext _ = gettext.gettext -from html5lib.constants import voidElements, spaceCharacters -spaceCharacters = u"".join(spaceCharacters) - -class TreeWalker(object): - def __init__(self, tree): - self.tree = tree - - def __iter__(self): - raise NotImplementedError - - def error(self, msg): - return {"type": "SerializeError", "data": msg} - - def normalizeAttrs(self, attrs): - newattrs = {} - if attrs: - #TODO: treewalkers should always have attrs - for (namespace,name),value in attrs.iteritems(): - namespace = unicode(namespace) if namespace else None - name = unicode(name) - value = unicode(value) - newattrs[(namespace,name)] = value - return newattrs - - def emptyTag(self, namespace, name, attrs, hasChildren=False): - yield {"type": "EmptyTag", "name": unicode(name), - "namespace":unicode(namespace), - "data": self.normalizeAttrs(attrs)} - if hasChildren: - yield self.error(_("Void element has children")) - - def startTag(self, namespace, name, attrs): - return {"type": "StartTag", - "name": unicode(name), - "namespace":unicode(namespace), - "data": self.normalizeAttrs(attrs)} - - def endTag(self, namespace, name): - return {"type": "EndTag", - "name": unicode(name), - "namespace":unicode(namespace), - "data": {}} - - def text(self, data): - data = unicode(data) - middle = data.lstrip(spaceCharacters) - left = data[:len(data)-len(middle)] - if left: - yield {"type": "SpaceCharacters", "data": left} - data = middle - middle = data.rstrip(spaceCharacters) - right = data[len(middle):] - if middle: - yield {"type": "Characters", "data": middle} - if right: - yield {"type": "SpaceCharacters", "data": right} - - def comment(self, data): - return {"type": "Comment", "data": unicode(data)} - - def doctype(self, name, publicId=None, systemId=None, correct=True): - return {"type": "Doctype", - "name": name is not None and unicode(name) or u"", - "publicId": publicId, - "systemId": systemId, - "correct": correct} - - def entity(self, name): - return {"type": "Entity", "name": unicode(name)} - - def unknown(self, nodeType): - return self.error(_("Unknown node type: ") + nodeType) - -class RecursiveTreeWalker(TreeWalker): - def walkChildren(self, node): - raise NodeImplementedError - - def element(self, node, namespace, name, attrs, hasChildren): - if name in voidElements: - for token in self.emptyTag(namespace, name, attrs, hasChildren): - yield token - else: - yield self.startTag(name, attrs) - if hasChildren: - for token in self.walkChildren(node): - yield token - yield self.endTag(name) - from xml.dom import Node DOCUMENT = Node.DOCUMENT_NODE @@ -99,16 +14,127 @@ COMMENT = Node.COMMENT_NODE ENTITY = Node.ENTITY_NODE UNKNOWN = "<#UNKNOWN#>" +from ..constants import voidElements, spaceCharacters +spaceCharacters = "".join(spaceCharacters) + + +def to_text(s, blank_if_none=True): + """Wrapper around six.text_type to convert None to empty string""" + if s is None: + if blank_if_none: + return "" + else: + return None + elif isinstance(s, text_type): + return s + else: + return text_type(s) + + +def is_text_or_none(string): + """Wrapper around isinstance(string_types) or is None""" + return string is None or isinstance(string, string_types) + + +class TreeWalker(object): + def __init__(self, tree): + self.tree = tree + + def __iter__(self): + raise NotImplementedError + + def error(self, msg): + return {"type": "SerializeError", "data": msg} + + def emptyTag(self, namespace, name, attrs, hasChildren=False): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(name) + assert all((namespace is None or isinstance(namespace, string_types)) and + isinstance(name, string_types) and + isinstance(value, string_types) + for (namespace, name), value in attrs.items()) + + yield {"type": "EmptyTag", "name": to_text(name, False), + "namespace": to_text(namespace), + "data": attrs} + if hasChildren: + yield self.error(_("Void element has children")) + + def startTag(self, namespace, name, attrs): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(name) + assert all((namespace is None or isinstance(namespace, string_types)) and + isinstance(name, string_types) and + isinstance(value, string_types) + for (namespace, name), value in attrs.items()) + + return {"type": "StartTag", + "name": text_type(name), + "namespace": to_text(namespace), + "data": dict(((to_text(namespace, False), to_text(name)), + to_text(value, False)) + for (namespace, name), value in attrs.items())} + + def endTag(self, namespace, name): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(namespace) + + return {"type": "EndTag", + "name": to_text(name, False), + "namespace": to_text(namespace), + "data": {}} + + def text(self, data): + assert isinstance(data, string_types), type(data) + + data = to_text(data) + middle = data.lstrip(spaceCharacters) + left = data[:len(data) - len(middle)] + if left: + yield {"type": "SpaceCharacters", "data": left} + data = middle + middle = data.rstrip(spaceCharacters) + right = data[len(middle):] + if middle: + yield {"type": "Characters", "data": middle} + if right: + yield {"type": "SpaceCharacters", "data": right} + + def comment(self, data): + assert isinstance(data, string_types), type(data) + + return {"type": "Comment", "data": text_type(data)} + + def doctype(self, name, publicId=None, systemId=None, correct=True): + assert is_text_or_none(name), type(name) + assert is_text_or_none(publicId), type(publicId) + assert is_text_or_none(systemId), type(systemId) + + return {"type": "Doctype", + "name": to_text(name), + "publicId": to_text(publicId), + "systemId": to_text(systemId), + "correct": to_text(correct)} + + def entity(self, name): + assert isinstance(name, string_types), type(name) + + return {"type": "Entity", "name": text_type(name)} + + def unknown(self, nodeType): + return self.error(_("Unknown node type: ") + nodeType) + + class NonRecursiveTreeWalker(TreeWalker): def getNodeDetails(self, node): raise NotImplementedError - + def getFirstChild(self, node): raise NotImplementedError - + def getNextSibling(self, node): raise NotImplementedError - + def getParentNode(self, node): raise NotImplementedError @@ -118,7 +144,6 @@ class NonRecursiveTreeWalker(TreeWalker): details = self.getNodeDetails(currentNode) type, details = details[0], details[1:] hasChildren = False - endTag = None if type == DOCTYPE: yield self.doctype(*details) @@ -130,12 +155,11 @@ class NonRecursiveTreeWalker(TreeWalker): elif type == ELEMENT: namespace, name, attributes, hasChildren = details if name in voidElements: - for token in self.emptyTag(namespace, name, attributes, + for token in self.emptyTag(namespace, name, attributes, hasChildren): yield token hasChildren = False else: - endTag = name yield self.startTag(namespace, name, attributes) elif type == COMMENT: @@ -149,12 +173,12 @@ class NonRecursiveTreeWalker(TreeWalker): else: yield self.unknown(details[0]) - + if hasChildren: firstChild = self.getFirstChild(currentNode) else: firstChild = None - + if firstChild is not None: currentNode = firstChild else: diff --git a/libs/html5lib/treewalkers/dom.py b/libs/html5lib/treewalkers/dom.py index 383b46cb..a01287a9 100644 --- a/libs/html5lib/treewalkers/dom.py +++ b/libs/html5lib/treewalkers/dom.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + from xml.dom import Node import gettext _ = gettext.gettext -import _base -from html5lib.constants import voidElements +from . import _base + class TreeWalker(_base.NonRecursiveTreeWalker): def getNodeDetails(self, node): @@ -16,10 +18,13 @@ class TreeWalker(_base.NonRecursiveTreeWalker): elif node.nodeType == Node.ELEMENT_NODE: attrs = {} - for attr in node.attributes.keys(): + for attr in list(node.attributes.keys()): attr = node.getAttributeNode(attr) - attrs[(attr.namespaceURI,attr.localName)] = attr.value - return (_base.ELEMENT, node.namespaceURI, node.nodeName, + if attr.namespaceURI: + attrs[(attr.namespaceURI, attr.localName)] = attr.value + else: + attrs[(None, attr.name)] = attr.value + return (_base.ELEMENT, node.namespaceURI, node.nodeName, attrs, node.hasChildNodes()) elif node.nodeType == Node.COMMENT_NODE: diff --git a/libs/html5lib/treewalkers/etree.py b/libs/html5lib/treewalkers/etree.py index 13b03194..fd8a9cc9 100644 --- a/libs/html5lib/treewalkers/etree.py +++ b/libs/html5lib/treewalkers/etree.py @@ -1,33 +1,28 @@ +from __future__ import absolute_import, division, unicode_literals + +try: + from collections import OrderedDict +except ImportError: + try: + from ordereddict import OrderedDict + except ImportError: + OrderedDict = dict import gettext _ = gettext.gettext -try: - from types import ModuleType -except: - from new import module as ModuleType -import copy import re -import _base -from html5lib.constants import voidElements +from six import text_type + +from . import _base +from ..utils import moduleFactoryFactory tag_regexp = re.compile("{([^}]*)}(.*)") -moduleCache = {} - -def getETreeModule(ElementTreeImplementation): - name = "_" + ElementTreeImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder") - objs = getETreeBuilder(ElementTreeImplementation) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getETreeBuilder(ElementTreeImplementation): ElementTree = ElementTreeImplementation + ElementTreeCommentType = ElementTree.Comment("asd").tag class TreeWalker(_base.NonRecursiveTreeWalker): """Given the particular ElementTree representation, this implementation, @@ -35,16 +30,16 @@ def getETreeBuilder(ElementTreeImplementation): content: 1. The current element - + 2. The index of the element relative to its parent - + 3. A stack of ancestor elements - + 4. A flag "text", "tail" or None to indicate if the current node is a text node; either the text or tail of the current element (1) """ def getNodeDetails(self, node): - if isinstance(node, tuple): # It might be the root Element + if isinstance(node, tuple): # It might be the root Element elt, key, parents, flag = node if flag in ("text", "tail"): return _base.TEXT, getattr(elt, flag) @@ -54,41 +49,41 @@ def getETreeBuilder(ElementTreeImplementation): if not(hasattr(node, "tag")): node = node.getroot() - if node.tag in ("<DOCUMENT_ROOT>", "<DOCUMENT_FRAGMENT>"): + if node.tag in ("DOCUMENT_ROOT", "DOCUMENT_FRAGMENT"): return (_base.DOCUMENT,) elif node.tag == "<!DOCTYPE>": - return (_base.DOCTYPE, node.text, + return (_base.DOCTYPE, node.text, node.get("publicId"), node.get("systemId")) - elif node.tag == ElementTree.Comment: + elif node.tag == ElementTreeCommentType: return _base.COMMENT, node.text else: - assert type(node.tag) in (str, unicode), type(node.tag) - #This is assumed to be an ordinary element + assert type(node.tag) == text_type, type(node.tag) + # This is assumed to be an ordinary element match = tag_regexp.match(node.tag) if match: namespace, tag = match.groups() else: namespace = None tag = node.tag - attrs = {} - for name, value in node.attrib.items(): + attrs = OrderedDict() + for name, value in list(node.attrib.items()): match = tag_regexp.match(name) if match: - attrs[(match.group(1),match.group(2))] = value + attrs[(match.group(1), match.group(2))] = value else: - attrs[(None,name)] = value - return (_base.ELEMENT, namespace, tag, + attrs[(None, name)] = value + return (_base.ELEMENT, namespace, tag, attrs, len(node) or node.text) - + def getFirstChild(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: element, key, parents, flag = node, None, [], None - + if flag in ("text", "tail"): return None else: @@ -99,13 +94,13 @@ def getETreeBuilder(ElementTreeImplementation): return element[0], 0, parents, None else: return None - + def getNextSibling(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: return None - + if flag == "text": if len(element): parents.append(element) @@ -116,16 +111,16 @@ def getETreeBuilder(ElementTreeImplementation): if element.tail and flag != "tail": return element, key, parents, "tail" elif key < len(parents[-1]) - 1: - return parents[-1][key+1], key+1, parents, None + return parents[-1][key + 1], key + 1, parents, None else: return None - + def getParentNode(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: return None - + if flag == "text": if not parents: return element @@ -139,3 +134,5 @@ def getETreeBuilder(ElementTreeImplementation): return parent, list(parents[-1]).index(parent), parents, None return locals() + +getETreeModule = moduleFactoryFactory(getETreeBuilder) diff --git a/libs/html5lib/treewalkers/genshistream.py b/libs/html5lib/treewalkers/genshistream.py index ef71a83e..f559c45d 100644 --- a/libs/html5lib/treewalkers/genshistream.py +++ b/libs/html5lib/treewalkers/genshistream.py @@ -1,50 +1,49 @@ +from __future__ import absolute_import, division, unicode_literals + +from genshi.core import QName from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT -from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT -from genshi.output import NamespaceFlattener +from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT -import _base +from . import _base + +from ..constants import voidElements, namespaces -from html5lib.constants import voidElements class TreeWalker(_base.TreeWalker): def __iter__(self): - depth = 0 - ignore_until = None + # Buffer the events so we can pass in the following one previous = None for event in self.tree: if previous is not None: - if previous[0] == START: - depth += 1 - if ignore_until <= depth: - ignore_until = None - if ignore_until is None: - for token in self.tokens(previous, event): - yield token - if token["type"] == "EmptyTag": - ignore_until = depth - if previous[0] == END: - depth -= 1 - previous = event - if previous is not None: - if ignore_until is None or ignore_until <= depth: - for token in self.tokens(previous, None): + for token in self.tokens(previous, event): yield token - elif ignore_until is not None: - raise ValueError("Illformed DOM event stream: void element without END_ELEMENT") + previous = event + + # Don't forget the final event! + if previous is not None: + for token in self.tokens(previous, None): + yield token def tokens(self, event, next): kind, data, pos = event if kind == START: - tag, attrib = data + tag, attribs = data name = tag.localname namespace = tag.namespace - if tag in voidElements: - for token in self.emptyTag(namespace, name, list(attrib), - not next or next[0] != END + converted_attribs = {} + for k, v in attribs: + if isinstance(k, QName): + converted_attribs[(k.namespace, k.localname)] = v + else: + converted_attribs[(None, k)] = v + + if namespace == namespaces["html"] and name in voidElements: + for token in self.emptyTag(namespace, name, converted_attribs, + not next or next[0] != END or next[1] != tag): yield token else: - yield self.startTag(namespace, name, list(attrib)) + yield self.startTag(namespace, name, converted_attribs) elif kind == END: name = data.localname @@ -62,8 +61,8 @@ class TreeWalker(_base.TreeWalker): elif kind == DOCTYPE: yield self.doctype(*data) - elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \ - START_CDATA, END_CDATA, PI): + elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, + START_CDATA, END_CDATA, PI): pass else: diff --git a/libs/html5lib/treewalkers/lxmletree.py b/libs/html5lib/treewalkers/lxmletree.py index 46f4908c..375cc2e8 100644 --- a/libs/html5lib/treewalkers/lxmletree.py +++ b/libs/html5lib/treewalkers/lxmletree.py @@ -1,186 +1,208 @@ -from lxml import etree -from html5lib.treebuilders.etree import tag_regexp - -from gettext import gettext -_ = gettext - -import _base - -from html5lib.constants import voidElements -from html5lib import ihatexml - -class Root(object): - def __init__(self, et): - self.elementtree = et - self.children = [] - if et.docinfo.internalDTD: - self.children.append(Doctype(self, et.docinfo.root_name, - et.docinfo.public_id, - et.docinfo.system_url)) - root = et.getroot() - node = root - - while node.getprevious() is not None: - node = node.getprevious() - while node is not None: - self.children.append(node) - node = node.getnext() - - self.text = None - self.tail = None - - def __getitem__(self, key): - return self.children[key] - - def getnext(self): - return None - - def __len__(self): - return 1 - -class Doctype(object): - def __init__(self, root_node, name, public_id, system_id): - self.root_node = root_node - self.name = name - self.public_id = public_id - self.system_id = system_id - - self.text = None - self.tail = None - - def getnext(self): - return self.root_node.children[1] - -class FragmentRoot(Root): - def __init__(self, children): - self.children = [FragmentWrapper(self, child) for child in children] - self.text = self.tail = None - - def getnext(self): - return None - -class FragmentWrapper(object): - def __init__(self, fragment_root, obj): - self.root_node = fragment_root - self.obj = obj - if hasattr(self.obj, 'text'): - self.text = self.obj.text - else: - self.text = None - if hasattr(self.obj, 'tail'): - self.tail = self.obj.tail - else: - self.tail = None - self.isstring = isinstance(obj, basestring) - - def __getattr__(self, name): - return getattr(self.obj, name) - - def getnext(self): - siblings = self.root_node.children - idx = siblings.index(self) - if idx < len(siblings) - 1: - return siblings[idx + 1] - else: - return None - - def __getitem__(self, key): - return self.obj[key] - - def __nonzero__(self): - return bool(self.obj) - - def getparent(self): - return None - - def __str__(self): - return str(self.obj) - - def __unicode__(self): - return unicode(self.obj) - - def __len__(self): - return len(self.obj) - - -class TreeWalker(_base.NonRecursiveTreeWalker): - def __init__(self, tree): - if hasattr(tree, "getroot"): - tree = Root(tree) - elif isinstance(tree, list): - tree = FragmentRoot(tree) - _base.NonRecursiveTreeWalker.__init__(self, tree) - self.filter = ihatexml.InfosetFilter() - def getNodeDetails(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - return _base.TEXT, getattr(node, key) - - elif isinstance(node, Root): - return (_base.DOCUMENT,) - - elif isinstance(node, Doctype): - return _base.DOCTYPE, node.name, node.public_id, node.system_id - - elif isinstance(node, FragmentWrapper) and node.isstring: - return _base.TEXT, node - - elif node.tag == etree.Comment: - return _base.COMMENT, node.text - - elif node.tag == etree.Entity: - return _base.ENTITY, node.text[1:-1] # strip &; - - else: - #This is assumed to be an ordinary element - match = tag_regexp.match(node.tag) - if match: - namespace, tag = match.groups() - else: - namespace = None - tag = node.tag - attrs = {} - for name, value in node.attrib.items(): - match = tag_regexp.match(name) - if match: - attrs[(match.group(1),match.group(2))] = value - else: - attrs[(None,name)] = value - return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), - attrs, len(node) > 0 or node.text) - - def getFirstChild(self, node): - assert not isinstance(node, tuple), _("Text nodes have no children") - - assert len(node) or node.text, "Node has no children" - if node.text: - return (node, "text") - else: - return node[0] - - def getNextSibling(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - if key == "text": - # XXX: we cannot use a "bool(node) and node[0] or None" construct here - # because node[0] might evaluate to False if it has no child element - if len(node): - return node[0] - else: - return None - else: # tail - return node.getnext() - - return node.tail and (node, "tail") or node.getnext() - - def getParentNode(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - if key == "text": - return node - # else: fallback to "normal" processing - - return node.getparent() +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from lxml import etree +from ..treebuilders.etree import tag_regexp + +from gettext import gettext +_ = gettext + +from . import _base + +from .. import ihatexml + + +def ensure_str(s): + if s is None: + return None + elif isinstance(s, text_type): + return s + else: + return s.decode("utf-8", "strict") + + +class Root(object): + def __init__(self, et): + self.elementtree = et + self.children = [] + if et.docinfo.internalDTD: + self.children.append(Doctype(self, + ensure_str(et.docinfo.root_name), + ensure_str(et.docinfo.public_id), + ensure_str(et.docinfo.system_url))) + root = et.getroot() + node = root + + while node.getprevious() is not None: + node = node.getprevious() + while node is not None: + self.children.append(node) + node = node.getnext() + + self.text = None + self.tail = None + + def __getitem__(self, key): + return self.children[key] + + def getnext(self): + return None + + def __len__(self): + return 1 + + +class Doctype(object): + def __init__(self, root_node, name, public_id, system_id): + self.root_node = root_node + self.name = name + self.public_id = public_id + self.system_id = system_id + + self.text = None + self.tail = None + + def getnext(self): + return self.root_node.children[1] + + +class FragmentRoot(Root): + def __init__(self, children): + self.children = [FragmentWrapper(self, child) for child in children] + self.text = self.tail = None + + def getnext(self): + return None + + +class FragmentWrapper(object): + def __init__(self, fragment_root, obj): + self.root_node = fragment_root + self.obj = obj + if hasattr(self.obj, 'text'): + self.text = ensure_str(self.obj.text) + else: + self.text = None + if hasattr(self.obj, 'tail'): + self.tail = ensure_str(self.obj.tail) + else: + self.tail = None + self.isstring = isinstance(obj, str) or isinstance(obj, bytes) + # Support for bytes here is Py2 + if self.isstring: + self.obj = ensure_str(self.obj) + + def __getattr__(self, name): + return getattr(self.obj, name) + + def getnext(self): + siblings = self.root_node.children + idx = siblings.index(self) + if idx < len(siblings) - 1: + return siblings[idx + 1] + else: + return None + + def __getitem__(self, key): + return self.obj[key] + + def __bool__(self): + return bool(self.obj) + + def getparent(self): + return None + + def __str__(self): + return str(self.obj) + + def __unicode__(self): + return str(self.obj) + + def __len__(self): + return len(self.obj) + + +class TreeWalker(_base.NonRecursiveTreeWalker): + def __init__(self, tree): + if hasattr(tree, "getroot"): + tree = Root(tree) + elif isinstance(tree, list): + tree = FragmentRoot(tree) + _base.NonRecursiveTreeWalker.__init__(self, tree) + self.filter = ihatexml.InfosetFilter() + + def getNodeDetails(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + return _base.TEXT, ensure_str(getattr(node, key)) + + elif isinstance(node, Root): + return (_base.DOCUMENT,) + + elif isinstance(node, Doctype): + return _base.DOCTYPE, node.name, node.public_id, node.system_id + + elif isinstance(node, FragmentWrapper) and node.isstring: + return _base.TEXT, node.obj + + elif node.tag == etree.Comment: + return _base.COMMENT, ensure_str(node.text) + + elif node.tag == etree.Entity: + return _base.ENTITY, ensure_str(node.text)[1:-1] # strip &; + + else: + # This is assumed to be an ordinary element + match = tag_regexp.match(ensure_str(node.tag)) + if match: + namespace, tag = match.groups() + else: + namespace = None + tag = ensure_str(node.tag) + attrs = {} + for name, value in list(node.attrib.items()): + name = ensure_str(name) + value = ensure_str(value) + match = tag_regexp.match(name) + if match: + attrs[(match.group(1), match.group(2))] = value + else: + attrs[(None, name)] = value + return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), + attrs, len(node) > 0 or node.text) + + def getFirstChild(self, node): + assert not isinstance(node, tuple), _("Text nodes have no children") + + assert len(node) or node.text, "Node has no children" + if node.text: + return (node, "text") + else: + return node[0] + + def getNextSibling(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + if key == "text": + # XXX: we cannot use a "bool(node) and node[0] or None" construct here + # because node[0] might evaluate to False if it has no child element + if len(node): + return node[0] + else: + return None + else: # tail + return node.getnext() + + return (node, "tail") if node.tail else node.getnext() + + def getParentNode(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + if key == "text": + return node + # else: fallback to "normal" processing + + return node.getparent() diff --git a/libs/html5lib/treewalkers/pulldom.py b/libs/html5lib/treewalkers/pulldom.py index 1f8b95b8..0b0f515f 100644 --- a/libs/html5lib/treewalkers/pulldom.py +++ b/libs/html5lib/treewalkers/pulldom.py @@ -1,9 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \ COMMENT, IGNORABLE_WHITESPACE, CHARACTERS -import _base +from . import _base + +from ..constants import voidElements -from html5lib.constants import voidElements class TreeWalker(_base.TreeWalker): def __iter__(self): @@ -11,7 +14,7 @@ class TreeWalker(_base.TreeWalker): previous = None for event in self.tree: if previous is not None and \ - (ignore_until is None or previous[1] is ignore_until): + (ignore_until is None or previous[1] is ignore_until): if previous[1] is ignore_until: ignore_until = None for token in self.tokens(previous, event): @@ -31,9 +34,9 @@ class TreeWalker(_base.TreeWalker): name = node.nodeName namespace = node.namespaceURI attrs = {} - for attr in node.attributes.keys(): + for attr in list(node.attributes.keys()): attr = node.getAttributeNode(attr) - attrs[(attr.namespaceURI,attr.localName)] = attr.value + attrs[(attr.namespaceURI, attr.localName)] = attr.value if name in voidElements: for token in self.emptyTag(namespace, name, diff --git a/libs/html5lib/treewalkers/simpletree.py b/libs/html5lib/treewalkers/simpletree.py deleted file mode 100644 index 9e6bd4c5..00000000 --- a/libs/html5lib/treewalkers/simpletree.py +++ /dev/null @@ -1,78 +0,0 @@ -import gettext -_ = gettext.gettext - -import _base - -class TreeWalker(_base.NonRecursiveTreeWalker): - """Given that simpletree has no performant way of getting a node's - next sibling, this implementation returns "nodes" as tuples with the - following content: - - 1. The parent Node (Element, Document or DocumentFragment) - - 2. The child index of the current node in its parent's children list - - 3. A list used as a stack of all ancestors. It is a pair tuple whose - first item is a parent Node and second item is a child index. - """ - - def getNodeDetails(self, node): - if isinstance(node, tuple): # It might be the root Node - parent, idx, parents = node - node = parent.childNodes[idx] - - # testing node.type allows us not to import treebuilders.simpletree - if node.type in (1, 2): # Document or DocumentFragment - return (_base.DOCUMENT,) - - elif node.type == 3: # DocumentType - return _base.DOCTYPE, node.name, node.publicId, node.systemId - - elif node.type == 4: # TextNode - return _base.TEXT, node.value - - elif node.type == 5: # Element - attrs = {} - for name, value in node.attributes.items(): - if isinstance(name, tuple): - attrs[(name[2],name[1])] = value - else: - attrs[(None,name)] = value - return (_base.ELEMENT, node.namespace, node.name, - attrs, node.hasContent()) - - elif node.type == 6: # CommentNode - return _base.COMMENT, node.data - - else: - return _node.UNKNOWN, node.type - - def getFirstChild(self, node): - if isinstance(node, tuple): # It might be the root Node - parent, idx, parents = node - parents.append((parent, idx)) - node = parent.childNodes[idx] - else: - parents = [] - - assert node.hasContent(), "Node has no children" - return (node, 0, parents) - - def getNextSibling(self, node): - assert isinstance(node, tuple), "Node is not a tuple: " + str(node) - parent, idx, parents = node - idx += 1 - if len(parent.childNodes) > idx: - return (parent, idx, parents) - else: - return None - - def getParentNode(self, node): - assert isinstance(node, tuple) - parent, idx, parents = node - if parents: - parent, idx = parents.pop() - return parent, idx, parents - else: - # HACK: We could return ``parent`` but None will stop the algorithm the same way - return None diff --git a/libs/html5lib/treewalkers/soup.py b/libs/html5lib/treewalkers/soup.py deleted file mode 100644 index fca65ecb..00000000 --- a/libs/html5lib/treewalkers/soup.py +++ /dev/null @@ -1,60 +0,0 @@ -import re -import gettext -_ = gettext.gettext - -from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag -from html5lib.constants import namespaces -import _base - -class TreeWalker(_base.NonRecursiveTreeWalker): - doctype_regexp = re.compile( - r'DOCTYPE\s+(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?') - def getNodeDetails(self, node): - if isinstance(node, BeautifulSoup): # Document or DocumentFragment - return (_base.DOCUMENT,) - - elif isinstance(node, Declaration): # DocumentType - string = unicode(node.string) - #Slice needed to remove markup added during unicode conversion, - #but only in some versions of BeautifulSoup/Python - if string.startswith('<!') and string.endswith('>'): - string = string[2:-1] - m = self.doctype_regexp.match(string) - #This regexp approach seems wrong and fragile - #but beautiful soup stores the doctype as a single thing and we want the seperate bits - #It should work as long as the tree is created by html5lib itself but may be wrong if it's - #been modified at all - #We could just feed to it a html5lib tokenizer, I guess... - assert m is not None, "DOCTYPE did not match expected format" - - name = m.group('name') - publicId = m.group('publicId') - if publicId is not None: - systemId = m.group('systemId1') - else: - systemId = m.group('systemId2') - return _base.DOCTYPE, name, publicId or "", systemId or "" - - elif isinstance(node, Comment): - string = unicode(node.string) - if string.startswith('<!--') and string.endswith('-->'): - string = string[4:-3] - return _base.COMMENT, string - - elif isinstance(node, unicode): # TextNode - return _base.TEXT, node - - elif isinstance(node, Tag): # Element - return (_base.ELEMENT, namespaces["html"], node.name, - dict(node.attrs).items(), node.contents) - else: - return _base.UNKNOWN, node.__class__.__name__ - - def getFirstChild(self, node): - return node.contents[0] - - def getNextSibling(self, node): - return node.nextSibling - - def getParentNode(self, node): - return node.parent diff --git a/libs/html5lib/trie/__init__.py b/libs/html5lib/trie/__init__.py new file mode 100644 index 00000000..a8cca8a9 --- /dev/null +++ b/libs/html5lib/trie/__init__.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + +from .py import Trie as PyTrie + +Trie = PyTrie + +try: + from .datrie import Trie as DATrie +except ImportError: + pass +else: + Trie = DATrie diff --git a/libs/html5lib/trie/_base.py b/libs/html5lib/trie/_base.py new file mode 100644 index 00000000..724486b1 --- /dev/null +++ b/libs/html5lib/trie/_base.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import, division, unicode_literals + +from collections import Mapping + + +class Trie(Mapping): + """Abstract base class for tries""" + + def keys(self, prefix=None): + keys = super().keys() + + if prefix is None: + return set(keys) + + # Python 2.6: no set comprehensions + return set([x for x in keys if x.startswith(prefix)]) + + def has_keys_with_prefix(self, prefix): + for key in self.keys(): + if key.startswith(prefix): + return True + + return False + + def longest_prefix(self, prefix): + if prefix in self: + return prefix + + for i in range(1, len(prefix) + 1): + if prefix[:-i] in self: + return prefix[:-i] + + raise KeyError(prefix) + + def longest_prefix_item(self, prefix): + lprefix = self.longest_prefix(prefix) + return (lprefix, self[lprefix]) diff --git a/libs/html5lib/trie/datrie.py b/libs/html5lib/trie/datrie.py new file mode 100644 index 00000000..51f3d046 --- /dev/null +++ b/libs/html5lib/trie/datrie.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import, division, unicode_literals + +from datrie import Trie as DATrie +from six import text_type + +from ._base import Trie as ABCTrie + + +class Trie(ABCTrie): + def __init__(self, data): + chars = set() + for key in data.keys(): + if not isinstance(key, text_type): + raise TypeError("All keys must be strings") + for char in key: + chars.add(char) + + self._data = DATrie("".join(chars)) + for key, value in data.items(): + self._data[key] = value + + def __contains__(self, key): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + raise NotImplementedError() + + def __getitem__(self, key): + return self._data[key] + + def keys(self, prefix=None): + return self._data.keys(prefix) + + def has_keys_with_prefix(self, prefix): + return self._data.has_keys_with_prefix(prefix) + + def longest_prefix(self, prefix): + return self._data.longest_prefix(prefix) + + def longest_prefix_item(self, prefix): + return self._data.longest_prefix_item(prefix) diff --git a/libs/html5lib/trie/py.py b/libs/html5lib/trie/py.py new file mode 100644 index 00000000..c2ba3da7 --- /dev/null +++ b/libs/html5lib/trie/py.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from bisect import bisect_left + +from ._base import Trie as ABCTrie + + +class Trie(ABCTrie): + def __init__(self, data): + if not all(isinstance(x, text_type) for x in data.keys()): + raise TypeError("All keys must be strings") + + self._data = data + self._keys = sorted(data.keys()) + self._cachestr = "" + self._cachepoints = (0, len(data)) + + def __contains__(self, key): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + return iter(self._data) + + def __getitem__(self, key): + return self._data[key] + + def keys(self, prefix=None): + if prefix is None or prefix == "" or not self._keys: + return set(self._keys) + + if prefix.startswith(self._cachestr): + lo, hi = self._cachepoints + start = i = bisect_left(self._keys, prefix, lo, hi) + else: + start = i = bisect_left(self._keys, prefix) + + keys = set() + if start == len(self._keys): + return keys + + while self._keys[i].startswith(prefix): + keys.add(self._keys[i]) + i += 1 + + self._cachestr = prefix + self._cachepoints = (start, i) + + return keys + + def has_keys_with_prefix(self, prefix): + if prefix in self._data: + return True + + if prefix.startswith(self._cachestr): + lo, hi = self._cachepoints + i = bisect_left(self._keys, prefix, lo, hi) + else: + i = bisect_left(self._keys, prefix) + + if i == len(self._keys): + return False + + return self._keys[i].startswith(prefix) diff --git a/libs/html5lib/utils.py b/libs/html5lib/utils.py index d53f6788..2f41f4df 100644 --- a/libs/html5lib/utils.py +++ b/libs/html5lib/utils.py @@ -1,9 +1,16 @@ +from __future__ import absolute_import, division, unicode_literals + +from types import ModuleType + try: - frozenset -except NameError: - #Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset + import xml.etree.cElementTree as default_etree +except ImportError: + import xml.etree.ElementTree as default_etree + + +__all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair", + "surrogatePairToCodepoint", "moduleFactoryFactory"] + class MethodDispatcher(dict): """Dict with 2 special properties: @@ -23,7 +30,7 @@ class MethodDispatcher(dict): # twice as fast. Please do careful performance testing before changing # anything here. _dictEntries = [] - for name,value in items: + for name, value in items: if type(name) in (list, tuple, frozenset, set): for item in name: _dictEntries.append((item, value)) @@ -35,141 +42,41 @@ class MethodDispatcher(dict): def __getitem__(self, key): return dict.get(self, key, self.default) -#Pure python implementation of deque taken from the ASPN Python Cookbook -#Original code by Raymond Hettinger -class deque(object): +# Some utility functions to dal with weirdness around UCS2 vs UCS4 +# python builds - def __init__(self, iterable=(), maxsize=-1): - if not hasattr(self, 'data'): - self.left = self.right = 0 - self.data = {} - self.maxsize = maxsize - self.extend(iterable) - - def append(self, x): - self.data[self.right] = x - self.right += 1 - if self.maxsize != -1 and len(self) > self.maxsize: - self.popleft() - - def appendleft(self, x): - self.left -= 1 - self.data[self.left] = x - if self.maxsize != -1 and len(self) > self.maxsize: - self.pop() - - def pop(self): - if self.left == self.right: - raise IndexError('cannot pop from empty deque') - self.right -= 1 - elem = self.data[self.right] - del self.data[self.right] - return elem - - def popleft(self): - if self.left == self.right: - raise IndexError('cannot pop from empty deque') - elem = self.data[self.left] - del self.data[self.left] - self.left += 1 - return elem - - def clear(self): - self.data.clear() - self.left = self.right = 0 - - def extend(self, iterable): - for elem in iterable: - self.append(elem) - - def extendleft(self, iterable): - for elem in iterable: - self.appendleft(elem) - - def rotate(self, n=1): - if self: - n %= len(self) - for i in xrange(n): - self.appendleft(self.pop()) - - def __getitem__(self, i): - if i < 0: - i += len(self) - try: - return self.data[i + self.left] - except KeyError: - raise IndexError - - def __setitem__(self, i, value): - if i < 0: - i += len(self) - try: - self.data[i + self.left] = value - except KeyError: - raise IndexError - - def __delitem__(self, i): - size = len(self) - if not (-size <= i < size): - raise IndexError - data = self.data - if i < 0: - i += size - for j in xrange(self.left+i, self.right-1): - data[j] = data[j+1] - self.pop() - - def __len__(self): - return self.right - self.left - - def __cmp__(self, other): - if type(self) != type(other): - return cmp(type(self), type(other)) - return cmp(list(self), list(other)) - - def __repr__(self, _track=[]): - if id(self) in _track: - return '...' - _track.append(id(self)) - r = 'deque(%r)' % (list(self),) - _track.remove(id(self)) - return r - - def __getstate__(self): - return (tuple(self),) - - def __setstate__(self, s): - self.__init__(s[0]) - - def __hash__(self): - raise TypeError - - def __copy__(self): - return self.__class__(self) - - def __deepcopy__(self, memo={}): - from copy import deepcopy - result = self.__class__() - memo[id(self)] = result - result.__init__(deepcopy(tuple(self), memo)) - return result - -#Some utility functions to dal with weirdness around UCS2 vs UCS4 -#python builds - -def encodingType(): - if len() == 2: - return "UCS2" - else: - return "UCS4" - -def isSurrogatePair(data): +def isSurrogatePair(data): return (len(data) == 2 and ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF) + def surrogatePairToCodepoint(data): - char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + + char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + (ord(data[1]) - 0xDC00)) return char_val + +# Module Factory Factory (no, this isn't Java, I know) +# Here to stop this being duplicated all over the place. + + +def moduleFactoryFactory(factory): + moduleCache = {} + + def moduleFactory(baseModule, *args, **kwargs): + if isinstance(ModuleType.__name__, type("")): + name = "_%s_factory" % baseModule.__name__ + else: + name = b"_%s_factory" % baseModule.__name__ + + if name in moduleCache: + return moduleCache[name] + else: + mod = ModuleType(name) + objs = factory(baseModule, *args, **kwargs) + mod.__dict__.update(objs) + moduleCache[name] = mod + return mod + + return moduleFactory diff --git a/libs/httplib2/__init__.py b/libs/httplib2/__init__.py index 01151f7f..9780d4e5 100644 --- a/libs/httplib2/__init__.py +++ b/libs/httplib2/__init__.py @@ -3,7 +3,7 @@ from __future__ import generators httplib2 A caching http interface that supports ETags and gzip -to conserve bandwidth. +to conserve bandwidth. Requires Python 2.3 or later @@ -15,17 +15,17 @@ Changelog: __author__ = "Joe Gregorio (joe@bitworking.org)" __copyright__ = "Copyright 2006, Joe Gregorio" __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", - "James Antill", - "Xavier Verges Farrero", - "Jonathan Feinberg", - "Blair Zajac", - "Sam Ruby", - "Louis Nyffenegger"] + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] __license__ = "MIT" -__version__ = "$Rev$" +__version__ = "0.8" -import re -import sys +import re +import sys import email import email.Utils import email.Message @@ -35,6 +35,7 @@ import gzip import zlib import httplib import urlparse +import urllib import base64 import os import copy @@ -42,10 +43,10 @@ import calendar import time import random import errno -# remove depracated warning in python2.6 try: from hashlib import sha1 as _sha, md5 as _md5 except ImportError: + # prior to Python 2.5, these were separate modules import sha import md5 _sha = sha.new @@ -54,21 +55,38 @@ import hmac from gettext import gettext as _ import socket -# Try using local version, followed by system, and none if neither are found try: - import lib.socks as socks + from httplib2 import socks except ImportError: try: - import socks as socks - except ImportError: + import socks + except (ImportError, AttributeError): socks = None # Build the appropriate socket wrapper for ssl try: import ssl # python 2.6 - _ssl_wrap_socket = ssl.wrap_socket -except ImportError: - def _ssl_wrap_socket(sock, key_file, cert_file): + ssl_SSLError = ssl.SSLError + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if disable_validation: + cert_reqs = ssl.CERT_NONE + else: + cert_reqs = ssl.CERT_REQUIRED + # We should be specifying SSL version 3 or TLS v1, but the ssl module + # doesn't expose the necessary knobs. So we need to go with the default + # of SSLv23. + return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, + cert_reqs=cert_reqs, ca_certs=ca_certs) +except (AttributeError, ImportError): + ssl_SSLError = None + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if not disable_validation: + raise CertificateValidationUnsupported( + "SSL certificate validation is not supported without " + "the ssl module installed. To avoid this error, install " + "the ssl module, or explicity disable validation.") ssl_sock = socket.ssl(sock, key_file, cert_file) return httplib.FakeSocket(sock, ssl_sock) @@ -84,15 +102,19 @@ def has_timeout(timeout): # python 2.6 return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) return (timeout is not None) -__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', - 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', - 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', - 'debuglevel'] +__all__ = [ + 'Http', 'Response', 'ProxyInfo', 'HttpLib2Error', 'RedirectMissingLocation', + 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', + 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel', 'ProxiesUnavailableError'] # The httplib debug level, set to a non-zero value to get debug output debuglevel = 0 +# A request will be tried 'RETRIES' times if it fails at the socket/connection level. +RETRIES = 2 # Python 2.3 support if sys.version_info < (2,4): @@ -113,8 +135,8 @@ if not hasattr(httplib.HTTPResponse, 'getheaders'): # All exceptions raised here derive from HttpLib2Error class HttpLib2Error(Exception): pass -# Some exceptions can be caught and optionally -# be turned back into responses. +# Some exceptions can be caught and optionally +# be turned back into responses. class HttpLib2ErrorWithResponse(HttpLib2Error): def __init__(self, desc, response, content): self.response = response @@ -127,8 +149,18 @@ class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class MalformedHeader(HttpLib2Error): pass class RelativeURIError(HttpLib2Error): pass class ServerNotFoundError(HttpLib2Error): pass +class ProxiesUnavailableError(HttpLib2Error): pass +class CertificateValidationUnsupported(HttpLib2Error): pass +class SSLHandshakeError(HttpLib2Error): pass +class NotSupportedOnThisPlatform(HttpLib2Error): pass +class CertificateHostnameMismatch(SSLHandshakeError): + def __init__(self, desc, host, cert): + HttpLib2Error.__init__(self, desc) + self.host = host + self.cert = cert # Open Items: # ----------- @@ -152,6 +184,16 @@ class ServerNotFoundError(HttpLib2Error): pass # requesting that URI again. DEFAULT_MAX_REDIRECTS = 5 +try: + # Users can optionally provide a module that tells us where the CA_CERTS + # are located. + import ca_certs_locater + CA_CERTS = ca_certs_locater.get() +except ImportError: + # Default CA certificates file bundled with httplib2. + CA_CERTS = os.path.join( + os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt") + # Which headers are hop-by-hop headers by default HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] @@ -176,7 +218,7 @@ def urlnorm(uri): raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) authority = authority.lower() scheme = scheme.lower() - if not path: + if not path: path = "/" # Could do syntax based normalization of the URI before # computing the digest. See Section 6.2.2 of Std 66. @@ -228,7 +270,7 @@ def _parse_cache_control(headers): parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] retval = dict(parts_with_args + parts_wo_args) - return retval + return retval # Whether to use a strict mode to parse WWW-Authenticate headers # Might lead to bad results in case of ill-formed header value, @@ -249,25 +291,30 @@ def _parse_www_authenticate(headers, headername='www-authenticate'): per auth_scheme.""" retval = {} if headers.has_key(headername): - authenticate = headers[headername].strip() - www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED - while authenticate: - # Break off the scheme at the beginning of the line - if headername == 'authentication-info': - (auth_scheme, the_rest) = ('digest', authenticate) - else: - (auth_scheme, the_rest) = authenticate.split(" ", 1) - # Now loop over all the key value pairs that come after the scheme, - # being careful not to roll into the next scheme - match = www_auth.search(the_rest) - auth_params = {} - while match: - if match and len(match.groups()) == 3: - (key, value, the_rest) = match.groups() - auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) + try: + + authenticate = headers[headername].strip() + www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED + while authenticate: + # Break off the scheme at the beginning of the line + if headername == 'authentication-info': + (auth_scheme, the_rest) = ('digest', authenticate) + else: + (auth_scheme, the_rest) = authenticate.split(" ", 1) + # Now loop over all the key value pairs that come after the scheme, + # being careful not to roll into the next scheme match = www_auth.search(the_rest) - retval[auth_scheme.lower()] = auth_params - authenticate = the_rest.strip() + auth_params = {} + while match: + if match and len(match.groups()) == 3: + (key, value, the_rest) = match.groups() + auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) + match = www_auth.search(the_rest) + retval[auth_scheme.lower()] = auth_params + authenticate = the_rest.strip() + + except ValueError: + raise MalformedHeader("WWW-Authenticate") return retval @@ -279,17 +326,17 @@ def _entry_disposition(response_headers, request_headers): 1. Cache-Control: max-stale 2. Age: headers are not used in the calculations. - Not that this algorithm is simpler than you might think + Not that this algorithm is simpler than you might think because we are operating as a private (non-shared) cache. This lets us ignore 's-maxage'. We can also ignore 'proxy-invalidate' since we aren't a proxy. - We will never return a stale document as - fresh as a design decision, and thus the non-implementation - of 'max-stale'. This also lets us safely ignore 'must-revalidate' + We will never return a stale document as + fresh as a design decision, and thus the non-implementation + of 'max-stale'. This also lets us safely ignore 'must-revalidate' since we operate as if every server has sent 'must-revalidate'. Since we are private we get to ignore both 'public' and 'private' parameters. We also ignore 'no-transform' since - we don't do any transformations. + we don't do any transformations. The 'no-store' parameter is handled at a higher level. So the only Cache-Control parameters we look at are: @@ -298,7 +345,7 @@ def _entry_disposition(response_headers, request_headers): max-age min-fresh """ - + retval = "STALE" cc = _parse_cache_control(request_headers) cc_response = _parse_cache_control(response_headers) @@ -340,10 +387,10 @@ def _entry_disposition(response_headers, request_headers): min_fresh = int(cc['min-fresh']) except ValueError: min_fresh = 0 - current_age += min_fresh + current_age += min_fresh if freshness_lifetime > current_age: retval = "FRESH" - return retval + return retval def _decompressContent(response, new_content): content = new_content @@ -391,7 +438,7 @@ def _updateCache(request_headers, response_headers, content, cache, cachekey): if status == 304: status = 200 - status_header = 'status: %d\r\n' % response_headers.status + status_header = 'status: %d\r\n' % status header_str = info.as_string() @@ -408,10 +455,10 @@ def _wsse_username_token(cnonce, iso_now, password): return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip() -# For credentials we need two things, first +# For credentials we need two things, first # a pool of credential to try (not necesarily tied to BAsic, Digest, etc.) # Then we also need a list of URIs that have already demanded authentication -# That list is tricky since sub-URIs can take the same auth, or the +# That list is tricky since sub-URIs can take the same auth, or the # auth scheme may change as you descend the tree. # So we also need each Auth instance to be able to tell us # how close to the 'top' it is. @@ -435,7 +482,7 @@ class Authentication(object): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate - Authorization header. Over-rise this in sub-classes.""" + Authorization header. Over-ride this in sub-classes.""" pass def response(self, response, content): @@ -443,7 +490,7 @@ class Authentication(object): or such returned from the last authorized response. Over-rise this in sub-classes if necessary. - Return TRUE is the request is to be retried, for + Return TRUE is the request is to be retried, for example Digest may return stale=true. """ return False @@ -461,7 +508,7 @@ class BasicAuthentication(Authentication): class DigestAuthentication(Authentication): - """Only do qop='auth' and MD5, since that + """Only do qop='auth' and MD5, since that is all Apache currently implements""" def __init__(self, credentials, host, request_uri, headers, response, content, http): Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) @@ -474,7 +521,7 @@ class DigestAuthentication(Authentication): self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper() if self.challenge['algorithm'] != 'MD5': raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) - self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) + self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) self.challenge['nc'] = 1 def request(self, method, request_uri, headers, content, cnonce = None): @@ -482,23 +529,24 @@ class DigestAuthentication(Authentication): H = lambda x: _md5(x).hexdigest() KD = lambda s, d: H("%s:%s" % (s, d)) A2 = "".join([method, ":", request_uri]) - self.challenge['cnonce'] = cnonce or _cnonce() - request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % (self.challenge['nonce'], - '%08x' % self.challenge['nc'], - self.challenge['cnonce'], - self.challenge['qop'], H(A2) - )) - headers['Authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( - self.credentials[0], + self.challenge['cnonce'] = cnonce or _cnonce() + request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % ( + self.challenge['nonce'], + '%08x' % self.challenge['nc'], + self.challenge['cnonce'], + self.challenge['qop'], H(A2))) + headers['authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( + self.credentials[0], self.challenge['realm'], self.challenge['nonce'], - request_uri, + request_uri, self.challenge['algorithm'], request_digest, self.challenge['qop'], self.challenge['nc'], - self.challenge['cnonce'], - ) + self.challenge['cnonce']) + if self.challenge.get('opaque'): + headers['authorization'] += ', opaque="%s"' % self.challenge['opaque'] self.challenge['nc'] += 1 def response(self, response, content): @@ -506,14 +554,14 @@ class DigestAuthentication(Authentication): challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {}) if 'true' == challenge.get('stale'): self.challenge['nonce'] = challenge['nonce'] - self.challenge['nc'] = 1 + self.challenge['nc'] = 1 return True else: updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {}) if updated_challenge.has_key('nextnonce'): self.challenge['nonce'] = updated_challenge['nextnonce'] - self.challenge['nc'] = 1 + self.challenge['nc'] = 1 return False @@ -547,9 +595,8 @@ class HmacDigestAuthentication(Authentication): else: self.pwhashmod = _sha self.key = "".join([self.credentials[0], ":", - self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), - ":", self.challenge['realm'] - ]) + self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), + ":", self.challenge['realm']]) self.key = self.pwhashmod.new(self.key).hexdigest().lower() def request(self, method, request_uri, headers, content): @@ -561,16 +608,15 @@ class HmacDigestAuthentication(Authentication): cnonce = _cnonce() request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val) request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() - headers['Authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( - self.credentials[0], + headers['authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( + self.credentials[0], self.challenge['realm'], self.challenge['snonce'], cnonce, - request_uri, + request_uri, created, request_digest, - keylist, - ) + keylist) def response(self, response, content): challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {}) @@ -583,7 +629,7 @@ class WsseAuthentication(Authentication): """This is thinly tested and should not be relied upon. At this time there isn't any third party server to test against. Blogger and TypePad implemented this algorithm at one point - but Blogger has since switched to Basic over HTTPS and + but Blogger has since switched to Basic over HTTPS and TypePad has implemented it wrong, by never issuing a 401 challenge but instead requiring your client to telepathically know that their endpoint is expecting WSSE profile="UsernameToken".""" @@ -593,7 +639,7 @@ class WsseAuthentication(Authentication): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers['Authorization'] = 'WSSE profile="UsernameToken"' + headers['authorization'] = 'WSSE profile="UsernameToken"' iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) @@ -629,7 +675,7 @@ class GoogleLoginAuthentication(Authentication): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth AUTH_SCHEME_CLASSES = { @@ -644,13 +690,13 @@ AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] class FileCache(object): """Uses a local directory as a store for cached files. - Not really safe to use if multiple threads or processes are going to + Not really safe to use if multiple threads or processes are going to be running on the same cache. """ def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior self.cache = cache self.safe = safe - if not os.path.exists(cache): + if not os.path.exists(cache): os.makedirs(self.cache) def get(self, key): @@ -660,7 +706,7 @@ class FileCache(object): f = file(cacheFullPath, "rb") retval = f.read() f.close() - except IOError, e: + except IOError: pass return retval @@ -688,34 +734,127 @@ class Credentials(object): def iter(self, domain): for (cdomain, name, password) in self.credentials: if cdomain == "" or domain == cdomain: - yield (name, password) + yield (name, password) class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" pass +class AllHosts(object): + pass class ProxyInfo(object): - """Collect information required to use a proxy.""" - def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): - """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX - constants. For example: + """Collect information required to use a proxy.""" + bypass_hosts = () -p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) - """ - self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass + def __init__(self, proxy_type, proxy_host, proxy_port, + proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: - def astuple(self): - return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, - self.proxy_user, self.proxy_pass) + p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, + proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type = proxy_type + self.proxy_host = proxy_host + self.proxy_port = proxy_port + self.proxy_rdns = proxy_rdns + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass - def isgood(self): - return socks and (self.proxy_host != None) and (self.proxy_port != None) + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, + self.proxy_rdns, self.proxy_user, self.proxy_pass) + + def isgood(self): + return (self.proxy_host != None) and (self.proxy_port != None) + + def applies_to(self, hostname): + return not self.bypass_host(hostname) + + def bypass_host(self, hostname): + """Has this host been excluded from the proxy config""" + if self.bypass_hosts is AllHosts: + return True + + bypass = False + for domain in self.bypass_hosts: + if hostname.endswith(domain): + bypass = True + + return bypass + + +def proxy_info_from_environment(method='http'): + """ + Read proxy info from the environment variables. + """ + if method not in ['http', 'https']: + return + + env_var = method + '_proxy' + url = os.environ.get(env_var, os.environ.get(env_var.upper())) + if not url: + return + pi = proxy_info_from_url(url, method) + + no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) + bypass_hosts = [] + if no_proxy: + bypass_hosts = no_proxy.split(',') + # special case, no_proxy=* means all hosts bypassed + if no_proxy == '*': + bypass_hosts = AllHosts + + pi.bypass_hosts = bypass_hosts + return pi + +def proxy_info_from_url(url, method='http'): + """ + Construct a ProxyInfo from a URL (such as http_proxy env var) + """ + url = urlparse.urlparse(url) + username = None + password = None + port = None + if '@' in url[1]: + ident, host_port = url[1].split('@', 1) + if ':' in ident: + username, password = ident.split(':', 1) + else: + password = ident + else: + host_port = url[1] + if ':' in host_port: + host, port = host_port.split(':', 1) + else: + host = host_port + + if port: + port = int(port) + else: + port = dict(https=443, http=80)[method] + + proxy_type = 3 # socks.PROXY_TYPE_HTTP + return ProxyInfo( + proxy_type = proxy_type, + proxy_host = host, + proxy_port = port, + proxy_user = username or None, + proxy_pass = password or None, + ) class HTTPConnectionWithTimeout(httplib.HTTPConnection): - """HTTPConnection subclass that supports timeouts""" + """ + HTTPConnection subclass that supports timeouts + + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): httplib.HTTPConnection.__init__(self, host, port, strict) @@ -725,27 +864,46 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): def connect(self): """Connect to the host and port specified in __init__.""" # Mostly verbatim from httplib.py. + if self.proxy_info and socks is None: + raise ProxiesUnavailableError( + 'Proxy support missing but proxy use was requested!') msg = "getaddrinfo returns an empty list" - for res in socket.getaddrinfo(self.host, self.port, 0, - socket.SOCK_STREAM): + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: - if self.proxy_info and self.proxy_info.isgood(): + if use_proxy: self.sock = socks.socksocket(af, socktype, proto) - self.sock.setproxy(*self.proxy_info.astuple()) + self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) else: self.sock = socket.socket(af, socktype, proto) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Different from httplib: support timeouts. if has_timeout(self.timeout): self.sock.settimeout(self.timeout) # End of difference from httplib. if self.debuglevel > 0: - print "connect: (%s, %s)" % (self.host, self.port) + print "connect: (%s, %s) ************" % (self.host, self.port) + if use_proxy: + print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) - self.sock.connect(sa) + self.sock.connect((self.host, self.port) + sa[2:]) except socket.error, msg: if self.debuglevel > 0: - print 'connect fail:', (self.host, self.port) + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) if self.sock: self.sock.close() self.sock = None @@ -755,56 +913,265 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): raise socket.error, msg class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): - "This class allows communication via SSL." + """ + This class allows communication via SSL. + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=None, proxy_info=None): - httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, - cert_file=cert_file, strict=strict) + strict=None, timeout=None, proxy_info=None, + ca_certs=None, disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, + key_file=key_file, + cert_file=cert_file, strict=strict) self.timeout = timeout self.proxy_info = proxy_info + if ca_certs is None: + ca_certs = CA_CERTS + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + + # The following two methods were adapted from https_wrapper.py, released + # with the Google Appengine SDK at + # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py + # under the following license: + # + # Copyright 2007 Google Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + def _GetValidHostsForCert(self, cert): + """Returns a list of valid host globs for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + Returns: + list: A list of valid host globs. + """ + if 'subjectAltName' in cert: + return [x[1] for x in cert['subjectAltName'] + if x[0].lower() == 'dns'] + else: + return [x[0][1] for x in cert['subject'] + if x[0][0].lower() == 'commonname'] + + def _ValidateCertificateHostname(self, cert, hostname): + """Validates that a given hostname is valid for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + hostname: The hostname to test. + Returns: + bool: Whether or not the hostname is valid for this certificate. + """ + hosts = self._GetValidHostsForCert(cert) + for host in hosts: + host_re = host.replace('.', '\.').replace('*', '[^.]*') + if re.search('^%s$' % (host_re,), hostname, re.I): + return True + return False def connect(self): "Connect to a host on a given (SSL) port." + msg = "getaddrinfo returns an empty list" if self.proxy_info and self.proxy_info.isgood(): - sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) - sock.setproxy(*self.proxy_info.astuple()) + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() else: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - if has_timeout(self.timeout): - sock.settimeout(self.timeout) - sock.connect((self.host, self.port)) - self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) + for family, socktype, proto, canonname, sockaddr in address_info: + try: + if use_proxy: + sock = socks.socksocket(family, socktype, proto) + + sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) + else: + sock = socket.socket(family, socktype, proto) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket( + sock, self.key_file, self.cert_file, + self.disable_ssl_certificate_validation, self.ca_certs) + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if not self.disable_ssl_certificate_validation: + cert = self.sock.getpeercert() + hostname = self.host.split(':', 0)[0] + if not self._ValidateCertificateHostname(cert, hostname): + raise CertificateHostnameMismatch( + 'Server presented certificate that does not match ' + 'host %s: %s' % (hostname, cert), hostname, cert) + except ssl_SSLError, e: + if sock: + sock.close() + if self.sock: + self.sock.close() + self.sock = None + # Unfortunately the ssl module doesn't seem to provide any way + # to get at more detailed error information, in particular + # whether the error is due to certificate validation or + # something else (such as SSL protocol mismatch). + if e.errno == ssl.SSL_ERROR_SSL: + raise SSLHandshakeError(e) + else: + raise + except (socket.timeout, socket.gaierror): + raise + except socket.error, msg: + if self.debuglevel > 0: + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +SCHEME_TO_CONNECTION = { + 'http': HTTPConnectionWithTimeout, + 'https': HTTPSConnectionWithTimeout +} + +# Use a different connection object for Google App Engine +try: + try: + from google.appengine.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google.appengine.api.urlfetch import fetch + from google.appengine.api.urlfetch import InvalidURLError + except (ImportError, AttributeError): + from google3.apphosting.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google3.apphosting.api.urlfetch import fetch + from google3.apphosting.api.urlfetch import InvalidURLError + + def _new_fixed_fetch(validate_certificate): + def fixed_fetch(url, payload=None, method="GET", headers={}, + allow_truncated=False, follow_redirects=True, + deadline=5): + return fetch(url, payload=payload, method=method, headers=headers, + allow_truncated=allow_truncated, + follow_redirects=follow_redirects, deadline=deadline, + validate_certificate=validate_certificate) + return fixed_fetch + + class AppEngineHttpConnection(httplib.HTTPConnection): + """Use httplib on App Engine, but compensate for its weirdness. + + The parameters key_file, cert_file, proxy_info, ca_certs, and + disable_ssl_certificate_validation are all dropped on the ground. + """ + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + httplib.HTTPConnection.__init__(self, host, port=port, + strict=strict, timeout=timeout) + + class AppEngineHttpsConnection(httplib.HTTPSConnection): + """Same as AppEngineHttpConnection, but for HTTPS URIs.""" + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, + key_file=key_file, + cert_file=cert_file, strict=strict, + timeout=timeout) + self._fetch = _new_fixed_fetch( + not disable_ssl_certificate_validation) + + # Update the connection classes to use the Googel App Engine specific ones. + SCHEME_TO_CONNECTION = { + 'http': AppEngineHttpConnection, + 'https': AppEngineHttpsConnection + } +except (ImportError, AttributeError): + pass class Http(object): """An HTTP client that handles: -- all methods -- caching -- ETags -- compression, -- HTTPS -- Basic -- Digest -- WSSE -and more. + - all methods + - caching + - ETags + - compression, + - HTTPS + - Basic + - Digest + - WSSE + + and more. """ - def __init__(self, cache=None, timeout=None, proxy_info=None): - """The value of proxy_info is a ProxyInfo instance. + def __init__(self, cache=None, timeout=None, + proxy_info=proxy_info_from_environment, + ca_certs=None, disable_ssl_certificate_validation=False): + """If 'cache' is a string then it is used as a directory name for + a disk cache. Otherwise it must be an object that supports the + same interface as FileCache. -If 'cache' is a string then it is used as a directory name -for a disk cache. Otherwise it must be an object that supports -the same interface as FileCache.""" + All timeouts are in seconds. If None is passed for timeout + then Python's default timeout for sockets will be used. See + for example the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + + `proxy_info` may be: + - a callable that takes the http scheme ('http' or 'https') and + returns a ProxyInfo instance per request. By default, uses + proxy_nfo_from_environment. + - a ProxyInfo instance (static proxy config). + - None (proxy disabled). + + ca_certs is the path of a file containing root CA certificates for SSL + server certificate validation. By default, a CA cert file bundled with + httplib2 is used. + + If disable_ssl_certificate_validation is true, SSL cert validation will + not be performed. + """ self.proxy_info = proxy_info + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + # Map domain name to an httplib connection self.connections = {} # The location of the cache, for now a directory # where cached responses are held. - if cache and isinstance(cache, str): + if cache and isinstance(cache, basestring): self.cache = FileCache(cache) else: self.cache = cache @@ -820,10 +1187,10 @@ the same interface as FileCache.""" # If set to False then no redirects are followed, even safe ones. self.follow_redirects = True - + # Which HTTP methods do we apply optimistic concurrency to, i.e. # which methods get an "if-match:" etag header added to them. - self.optimistic_concurrency_methods = ["PUT"] + self.optimistic_concurrency_methods = ["PUT", "PATCH"] # If 'follow_redirects' is True, and this is set to True then # all redirecs are followed, including unsafe ones. @@ -831,10 +1198,27 @@ the same interface as FileCache.""" self.ignore_etag = False - self.force_exception_to_status_code = False + self.force_exception_to_status_code = False self.timeout = timeout + # Keep Authorization: headers on a redirect. + self.forward_authorization_headers = False + + def __getstate__(self): + state_dict = copy.copy(self.__dict__) + # In case request is augmented by some foreign object such as + # credentials which handle auth + if 'request' in state_dict: + del state_dict['request'] + if 'connections' in state_dict: + del state_dict['connections'] + return state_dict + + def __setstate__(self, state): + self.__dict__.update(state) + self.connections = {} + def _auth_from_challenge(self, host, request_uri, headers, response, content): """A generator that creates Authorization objects that can be applied to requests. @@ -862,37 +1246,56 @@ the same interface as FileCache.""" self.authorizations = [] def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(2): + for i in range(RETRIES): try: + if hasattr(conn, 'sock') and conn.sock is None: + conn.connect() conn.request(method, request_uri, body, headers) + except socket.timeout: + raise except socket.gaierror: conn.close() raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except ssl_SSLError: + conn.close() + raise except socket.error, e: - if not hasattr(e, 'errno'): # I don't know what this is so lets raise it if it happens + err = 0 + if hasattr(e, 'args'): + err = getattr(e, 'args')[0] + else: + err = e.errno + if err == errno.ECONNREFUSED: # Connection refused raise - elif e.errno == errno.ECONNREFUSED: # Connection refused - raise - # Just because the server closed the connection doesn't apparently mean - # that the server didn't send a response. - pass except httplib.HTTPException: # Just because the server closed the connection doesn't apparently mean # that the server didn't send a response. - pass + if hasattr(conn, 'sock') and conn.sock is None: + if i < RETRIES-1: + conn.close() + conn.connect() + continue + else: + conn.close() + raise + if i < RETRIES-1: + conn.close() + conn.connect() + continue try: response = conn.getresponse() except (socket.error, httplib.HTTPException): - if i == 0: + if i < RETRIES-1: conn.close() conn.connect() continue else: + conn.close() raise else: content = "" if method == "HEAD": - response.close() + conn.close() else: content = response.read() response = Response(response) @@ -908,12 +1311,12 @@ the same interface as FileCache.""" auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] auth = auths and sorted(auths)[0][1] or None - if auth: + if auth: auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) - if auth: + if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers ) @@ -921,7 +1324,7 @@ the same interface as FileCache.""" if response.status == 401: for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): - authorization.request(method, request_uri, headers, body) + authorization.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) if response.status != 401: self.authorizations.append(authorization) @@ -944,26 +1347,31 @@ the same interface as FileCache.""" if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri _updateCache(headers, response, content, self.cache, cachekey) if headers.has_key('if-none-match'): del headers['if-none-match'] if headers.has_key('if-modified-since'): del headers['if-modified-since'] + if 'authorization' in headers and not self.forward_authorization_headers: + del headers['authorization'] if response.has_key('location'): location = response['location'] old_response = copy.deepcopy(response) if not old_response.has_key('content-location'): - old_response['content-location'] = absolute_uri - redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method + old_response['content-location'] = absolute_uri + redirect_method = method + if response.status in [302, 303]: + redirect_method = "GET" + body = None (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) response.previous = old_response else: - raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) - elif response.status in [200, 203] and method == "GET": + raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content) + elif response.status in [200, 203] and method in ["GET", "HEAD"]: # Don't cache 206's since we aren't going to handle byte range requests if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri _updateCache(headers, response, content, self.cache, cachekey) return (response, content) @@ -978,24 +1386,25 @@ the same interface as FileCache.""" def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): """ Performs a single HTTP request. -The 'uri' is the URI of the HTTP resource and can begin -with either 'http' or 'https'. The value of 'uri' must be an absolute URI. -The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. -There is no restriction on the methods allowed. + The 'uri' is the URI of the HTTP resource and can begin with either + 'http' or 'https'. The value of 'uri' must be an absolute URI. -The 'body' is the entity body to be sent with the request. It is a string -object. + The 'method' is the HTTP method to perform, such as GET, POST, DELETE, + etc. There is no restriction on the methods allowed. -Any extra headers that are to be sent with the request should be provided in the -'headers' dictionary. + The 'body' is the entity body to be sent with the request. It is a + string object. -The maximum number of redirect to follow before raising an -exception is 'redirections. The default is 5. + Any extra headers that are to be sent with the request should be + provided in the 'headers' dictionary. -The return value is a tuple of (response, content), the first -being and instance of the 'Response' class, the second being -a string that contains the response entity body. + The maximum number of redirect to follow before raising an + exception is 'redirections. The default is 5. + + The return value is a tuple of (response, content), the first + being and instance of the 'Response' class, the second being + a string that contains the response entity body. """ try: if headers is None: @@ -1004,7 +1413,7 @@ a string that contains the response entity body. headers = self._normalize_headers(headers) if not headers.has_key('user-agent'): - headers['user-agent'] = "Python-httplib2/%s" % __version__ + headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) @@ -1014,21 +1423,38 @@ a string that contains the response entity body. scheme = 'https' authority = domain_port[0] + proxy_info = self._get_proxy_info(scheme, authority) + conn_key = scheme+":"+authority if conn_key in self.connections: conn = self.connections[conn_key] else: if not connection_type: - connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout + connection_type = SCHEME_TO_CONNECTION[scheme] certs = list(self.certificates.iter(authority)) - if scheme == 'https' and certs: - conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], - cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) + if scheme == 'https': + if certs: + conn = self.connections[conn_key] = connection_type( + authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) + else: + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) else: - conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info) conn.set_debuglevel(debuglevel) - if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: + if 'range' not in headers and 'accept-encoding' not in headers: headers['accept-encoding'] = 'gzip, deflate' info = email.Message.Message() @@ -1048,7 +1474,7 @@ a string that contains the response entity body. feedparser.feed(info) info = feedparser.close() feedparser._parse = None - except IndexError, ValueError: + except (IndexError, ValueError): self.cache.delete(cachekey) cachekey = None cached_value = None @@ -1071,13 +1497,15 @@ a string that contains the response entity body. for header in vary_headers: key = '-varied-%s' % header value = info[key] - if headers.get(header, '') != value: - cached_value = None - break + if headers.get(header, None) != value: + cached_value = None + break if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: if info.has_key('-x-permanent-redirect-url'): # Should cached permanent redirects be counted in our redirection count? For now, yes. + if redirections <= 0: + raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "") (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) response.previous = Response(info) response.previous.fromcache = True @@ -1085,13 +1513,13 @@ a string that contains the response entity body. # Determine our course of action: # Is the cached entry fresh or stale? # Has the client requested a non-cached response? - # - # There seems to be three possible answers: + # + # There seems to be three possible answers: # 1. [FRESH] Return the cache entry w/o doing a GET # 2. [STALE] Do the GET (but add in cache validators if available) # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request - entry_disposition = _entry_disposition(info, headers) - + entry_disposition = _entry_disposition(info, headers) + if entry_disposition == "FRESH": if not cached_value: info['status'] = '504' @@ -1113,7 +1541,7 @@ a string that contains the response entity body. if response.status == 304 and method == "GET": # Rewrite the cache entry with the new end-to-end headers - # Take all headers that are in response + # Take all headers that are in response # and overwrite their values in info. # unless they are hop-by-hop, or are listed in the connection header. @@ -1125,14 +1553,14 @@ a string that contains the response entity body. _updateCache(headers, merged_response, content, self.cache, cachekey) response = merged_response response.status = 200 - response.fromcache = True + response.fromcache = True elif response.status == 200: content = new_content else: self.cache.delete(cachekey) - content = new_content - else: + content = new_content + else: cc = _parse_cache_control(headers) if cc.has_key('only-if-cached'): info['status'] = '504' @@ -1146,34 +1574,47 @@ a string that contains the response entity body. response = e.response content = e.content response.status = 500 - response.reason = str(e) - elif isinstance(e, socket.timeout) or (isinstance(e, socket.error) and 'timed out' in str(e)): + response.reason = str(e) + elif isinstance(e, socket.timeout): content = "Request Timeout" - response = Response( { - "content-type": "text/plain", - "status": "408", - "content-length": len(content) - }) + response = Response({ + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) response.reason = "Request Timeout" else: - content = str(e) - response = Response( { - "content-type": "text/plain", - "status": "400", - "content-length": len(content) - }) - response.reason = "Bad Request" + content = str(e) + response = Response({ + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" else: raise - + return (response, content) - + def _get_proxy_info(self, scheme, authority): + """Return a ProxyInfo instance (or None) based on the scheme + and authority. + """ + hostname, port = urllib.splitport(authority) + proxy_info = self.proxy_info + if callable(proxy_info): + proxy_info = proxy_info(scheme) + + if (hasattr(proxy_info, 'applies_to') + and not proxy_info.applies_to(hostname)): + proxy_info = None + return proxy_info + class Response(dict): """An object more like email.Message than httplib.HTTPResponse.""" - + """Is this response from our local cache""" fromcache = False @@ -1189,27 +1630,28 @@ class Response(dict): previous = None def __init__(self, info): - # info is either an email.Message or + # info is either an email.Message or # an httplib.HTTPResponse object. if isinstance(info, httplib.HTTPResponse): - for key, value in info.getheaders(): - self[key.lower()] = value + for key, value in info.getheaders(): + self[key.lower()] = value self.status = info.status self['status'] = str(self.status) self.reason = info.reason self.version = info.version elif isinstance(info, email.Message.Message): - for key, value in info.items(): - self[key] = value + for key, value in info.items(): + self[key.lower()] = value self.status = int(self['status']) else: - for key, value in info.iteritems(): - self[key] = value + for key, value in info.iteritems(): + self[key.lower()] = value self.status = int(self.get('status', self.status)) + self.reason = self.get('reason', self.reason) def __getattr__(self, name): if name == 'dict': - return self - else: - raise AttributeError, name + return self + else: + raise AttributeError, name diff --git a/libs/httplib2/cacerts.txt b/libs/httplib2/cacerts.txt new file mode 100644 index 00000000..d8a0027c --- /dev/null +++ b/libs/httplib2/cacerts.txt @@ -0,0 +1,739 @@ +# Certifcate Authority certificates for validating SSL connections. +# +# This file contains PEM format certificates generated from +# http://mxr.mozilla.org/seamonkey/source/security/nss/lib/ckfw/builtins/certdata.txt +# +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Netscape security libraries. +# +# The Initial Developer of the Original Code is +# Netscape Communications Corporation. +# Portions created by the Initial Developer are Copyright (C) 1994-2000 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +Verisign/RSA Secure Server CA +============================= + +-----BEGIN CERTIFICATE----- +MIICNDCCAaECEAKtZn5ORf5eV288mBle3cAwDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxIDAeBgNVBAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYD +VQQLEyVTZWN1cmUgU2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk0 +MTEwOTAwMDAwMFoXDTEwMDEwNzIzNTk1OVowXzELMAkGA1UEBhMCVVMxIDAeBgNV +BAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYDVQQLEyVTZWN1cmUgU2Vy +dmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGbMA0GCSqGSIb3DQEBAQUAA4GJ +ADCBhQJ+AJLOesGugz5aqomDV6wlAXYMra6OLDfO6zV4ZFQD5YRAUcm/jwjiioII +0haGN1XpsSECrXZogZoFokvJSyVmIlZsiAeP94FZbYQHZXATcXY+m3dM41CJVphI +uR2nKRoTLkoRWZweFdVJVCxzOmmCsZc5nG1wZ0jl3S3WyB57AgMBAAEwDQYJKoZI +hvcNAQECBQADfgBl3X7hsuyw4jrg7HFGmhkRuNPHoLQDQCYCPgmc4RKz0Vr2N6W3 +YQO2WxZpO8ZECAyIUwxrl0nHPjXcbLm7qt9cuzovk2C2qUtN8iD3zV9/ZHuO3ABc +1/p3yjkWWW8O6tO1g39NTUJWdrTJXwT4OPjr0l91X817/OWOgHz8UA== +-----END CERTIFICATE----- + +Thawte Personal Basic CA +======================== + +-----BEGIN CERTIFICATE----- +MIIDITCCAoqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFBlcnNvbmFsIEJhc2lj +IENBMSgwJgYJKoZIhvcNAQkBFhlwZXJzb25hbC1iYXNpY0B0aGF3dGUuY29tMB4X +DTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgcsxCzAJBgNVBAYTAlpBMRUw +EwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEaMBgGA1UE +ChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRpb24gU2Vy +dmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQZXJzb25hbCBCYXNpYyBD +QTEoMCYGCSqGSIb3DQEJARYZcGVyc29uYWwtYmFzaWNAdGhhd3RlLmNvbTCBnzAN +BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvLyTU23AUE+CFeZIlDWmWr5vQvoPR+53 +dXLdjUmbllegeNTKP1GzaQuRdhciB5dqxFGTS+CN7zeVoQxN2jSQHReJl+A1OFdK +wPQIcOk8RHtQfmGakOMj04gRRif1CwcOu93RfyAKiLlWCy4cgNrx454p7xS9CkT7 +G1sY0b8jkyECAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQQF +AAOBgQAt4plrsD16iddZopQBHyvdEktTwq1/qqcAXJFAVyVKOKqEcLnZgA+le1z7 +c8a914phXAPjLSeoF+CEhULcXpvGt7Jtu3Sv5D/Lp7ew4F2+eIMllNLbgQ95B21P +9DkVWlIBe94y1k049hJcBlDfBVu9FEuh3ym6O0GN92NWod8isQ== +-----END CERTIFICATE----- + +Thawte Personal Premium CA +========================== + +-----BEGIN CERTIFICATE----- +MIIDKTCCApKgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBzzELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEjMCEGA1UEAxMaVGhhd3RlIFBlcnNvbmFsIFByZW1p +dW0gQ0ExKjAoBgkqhkiG9w0BCQEWG3BlcnNvbmFsLXByZW1pdW1AdGhhd3RlLmNv +bTAeFw05NjAxMDEwMDAwMDBaFw0yMDEyMzEyMzU5NTlaMIHPMQswCQYDVQQGEwJa +QTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xGjAY +BgNVBAoTEVRoYXd0ZSBDb25zdWx0aW5nMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9u +IFNlcnZpY2VzIERpdmlzaW9uMSMwIQYDVQQDExpUaGF3dGUgUGVyc29uYWwgUHJl +bWl1bSBDQTEqMCgGCSqGSIb3DQEJARYbcGVyc29uYWwtcHJlbWl1bUB0aGF3dGUu +Y29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJZtn4B0TPuYwu8KHvE0Vs +Bd/eJxZRNkERbGw77f4QfRKe5ZtCmv5gMcNmt3M6SK5O0DI3lIi1DbbZ8/JE2dWI +Et12TfIa/G8jHnrx2JhFTgcQ7xZC0EN1bUre4qrJMf8fAHB8Zs8QJQi6+u4A6UYD +ZicRFTuqW/KY3TZCstqIdQIDAQABoxMwETAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBBAUAA4GBAGk2ifc0KjNyL2071CKyuG+axTZmDhs8obF1Wub9NdP4qPIH +b4Vnjt4rueIXsDqg8A6iAJrf8xQVbrvIhVqYgPn/vnQdPfP+MCXRNzRn+qVxeTBh +KXLA4CxM+1bkOqhv5TJZUtt1KFBZDPgLGeSs2a+WjS9Q2wfD6h+rM+D1KzGJ +-----END CERTIFICATE----- + +Thawte Personal Freemail CA +=========================== + +-----BEGIN CERTIFICATE----- +MIIDLTCCApagAwIBAgIBADANBgkqhkiG9w0BAQQFADCB0TELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEkMCIGA1UEAxMbVGhhd3RlIFBlcnNvbmFsIEZyZWVt +YWlsIENBMSswKQYJKoZIhvcNAQkBFhxwZXJzb25hbC1mcmVlbWFpbEB0aGF3dGUu +Y29tMB4XDTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgdExCzAJBgNVBAYT +AlpBMRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEa +MBgGA1UEChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xJDAiBgNVBAMTG1RoYXd0ZSBQZXJzb25hbCBG +cmVlbWFpbCBDQTErMCkGCSqGSIb3DQEJARYccGVyc29uYWwtZnJlZW1haWxAdGhh +d3RlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1GnX1LCUZFtx6UfY +DFG26nKRsIRefS0Nj3sS34UldSh0OkIsYyeflXtL734Zhx2G6qPduc6WZBrCFG5E +rHzmj+hND3EfQDimAKOHePb5lIZererAXnbr2RSjXW56fAylS1V/Bhkpf56aJtVq +uzgkCGqYx7Hao5iR/Xnb5VrEHLkCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zAN +BgkqhkiG9w0BAQQFAAOBgQDH7JJ+Tvj1lqVnYiqk8E0RYNBvjWBYYawmu1I1XAjP +MPuoSpaKH2JCI4wXD/S6ZJwXrEcp352YXtJsYHFcoqzceePnbgBHH7UNKOgCneSa +/RP0ptl8sfjcXyMmCZGAc9AUG95DqYMl8uacLxXK/qarigd1iwzdUYRr5PjRznei +gQ== +-----END CERTIFICATE----- + +Thawte Server CA +================ + +-----BEGIN CERTIFICATE----- +MIIDEzCCAnygAwIBAgIBATANBgkqhkiG9w0BAQQFADCBxDELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEm +MCQGCSqGSIb3DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wHhcNOTYwODAx +MDAwMDAwWhcNMjAxMjMxMjM1OTU5WjCBxDELMAkGA1UEBhMCWkExFTATBgNVBAgT +DFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYDVQQKExRUaGF3 +dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNl +cyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEmMCQGCSqGSIb3 +DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD +gY0AMIGJAoGBANOkUG7I/1Zr5s9dtuoMaHVHoqrC2oQl/Kj0R1HahbUgdJSGHg91 +yekIYfUGbTBuFRkC6VLAYttNmZ7iagxEOM3+vuNkCXDF/rFrKbYvScg71CcEJRCX +L+eQbcAoQpnXTEPew/UhbVSfXcNY4cDk2VuwuNy0e982OsK1ZiIS1ocNAgMBAAGj +EzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEEBQADgYEAB/pMaVz7lcxG +7oWDTSEwjsrZqG9JGubaUeNgcGyEYRGhGshIPllDfU+VPaGLtwtimHp1it2ITk6e +QNuozDJ0uW8NxuOzRAvZim+aKZuZGCg70eNAKJpaPNW15yAbi8qkq43pUdniTCxZ +qdq5snUb9kLy78fyGPmJvKP/iiMucEc= +-----END CERTIFICATE----- + +Thawte Premium Server CA +======================== + +-----BEGIN CERTIFICATE----- +MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy +dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t +MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB +MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG +A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl +cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv +bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE +VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ +ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR +uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG +9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI +hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM +pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg== +-----END CERTIFICATE----- + +Equifax Secure CA +================= + +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV +UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy +dGlmaWNhdGUgQXV0aG9yaXR5MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1 +MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VxdWlmYXgxLTArBgNVBAsTJEVx +dWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPRfM6f +BeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+A +cJkVV5MW8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kC +AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQ +MA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlm +aWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTgw +ODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvSspXXR9gj +IBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQF +MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA +A4GBAFjOKer89961zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y +7qj/WsjTVbJmcVfewCHrPSqnI0kBBIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh +1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee9570+sB3c4 +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPTCCAaYCEQDNun9W8N/kvFT+IqyzcqpVMA0GCSqGSIb3DQEBAgUAMF8xCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xh +c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05 +NjAxMjkwMDAwMDBaFw0yODA4MDEyMzU5NTlaMF8xCzAJBgNVBAYTAlVTMRcwFQYD +VQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xhc3MgMSBQdWJsaWMgUHJp +bWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOB +jQAwgYkCgYEA5Rm/baNWYS2ZSHH2Z965jeu3noaACpEO+jglr0aIguVzqKCbJF0N +H8xlbgyw0FaEGIeaBpsQoXPftFg5a27B9hXVqKg/qhIGjTGsf7A01480Z4gJzRQR +4k5FVmkfeAKA2txHkSm7NsljXMXg1y2He6G3MrB7MLoqLzGq7qNn2tsCAwEAATAN +BgkqhkiG9w0BAQIFAAOBgQBMP7iLxmjf7kMzDl3ppssHhE16M/+SG/Q2rdiVIjZo +EWx8QszznC7EBz8UsA9P/5CSdvnivErpj82ggAr3xSnxgiJduLHdgSOjeyUVRjB5 +FvjqBUuUfx3CHMjjt/QQQDwTw18fU+hI5Ia0e6E1sHslurjTjqs/OJ0ANACY89Fx +lA== +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEC0b/EoXjaOR6+f/9YtFvgswDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAyIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAyIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQC2WoujDWojg4BrzzmH9CETMwZMJaLtVRKXxaeAufqDwSCg+i8VDXyh +YGt+eSz6Bg86rvYbb7HS/y8oUl+DfUvEerf4Zh+AVPy3wo5ZShRXRtGak75BkQO7 +FYCTXOvnzAhsPz6zSvz/S2wj1VCCJkQZjiPDceoZJEcEnnW/yKYAHwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBAIobK/o5wXTXXtgZZKJYSi034DNHD6zt96rbHuSLBlxg +J8pFUs4W7z8GZOeUaHxgMxURaa+dYo2jA1Rrpr7l7gUYYAS/QoD90KioHgE796Nc +r6Pc5iaAIzy4RHT3Cq5Ji2F4zCS/iIqnDupzGUH9TQPwiNHleI2lKk/2lw0Xd8rY +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE +BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is +I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do +lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc +AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEEzH6qqYPnHTkxD4PTqJkZIwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMSBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQCq0Lq+Fi24g9TK0g+8djHKlNgdk4xWArzZbxpvUjZudVYK +VdPfQ4chEWWKfo+9Id5rMj8bhDSVBZ1BNeuS65bdqlk/AVNtmU/t5eIqWpDBucSm +Fc/IReumXY6cPvBkJHalzasab7bYe1FhbqZ/h8jit+U03EGI6glAvnOSPWvndQID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAKlPww3HZ74sy9mozS11534Vnjty637rXC0J +h9ZrbWB85a7FkCMMXErQr7Fd88e2CtvgFZMN3QO8x3aKtd1Pw5sTdbgBwObJW2ul +uIncrKTdcu1OofdPvAbT6shkdHvClUGcZXNY8ZCaPGqxmMnEh7zPRW1F4m4iP/68 +DzFc6PLZ +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAzCCAmwCEQC5L2DMiJ+hekYJuFtwbIqvMA0GCSqGSIb3DQEBBQUAMIHBMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0Ns +YXNzIDIgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH +MjE6MDgGA1UECxMxKGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9y +aXplZCB1c2Ugb25seTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazAe +Fw05ODA1MTgwMDAwMDBaFw0yODA4MDEyMzU5NTlaMIHBMQswCQYDVQQGEwJVUzEX +MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0NsYXNzIDIgUHVibGlj +IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBHMjE6MDgGA1UECxMx +KGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s +eTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEAp4gBIXQs5xoD8JjhlzwPIQjxnNuX6Zr8wgQGE75fUsjM +HiwSViy4AWkszJkfrbCWrnkE8hM5wXuYuggs6MKEEyyqaekJ9MepAqRCwiNPStjw +DqL7MWzJ5m+ZJwf15vRMeJ5t60aG+rmGyVTyssSv1EYcWskVMP8NbPUtDm3Of3cC +AwEAATANBgkqhkiG9w0BAQUFAAOBgQByLvl/0fFx+8Se9sVeUYpAmLho+Jscg9ji +nb3/7aHmZuovCfTK1+qlK5X2JGCGTUQug6XELaDTrnhpb3LabK4I8GOSN+a7xDAX +rXfMSTWqz9iP0b63GJZHc2pUIjRkLbYWm1lbtFFZOrMLFPQS32eg9K0yZF6xRnIn +jBJ7xUS0rg== +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMyBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMyBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQDMXtERXVxp0KvTuWpMmR9ZmDCOFoUgRm1HP9SFIIThbbP4 +pO0M8RcPO/mn+SXXwc+EY/J8Y8+iR/LGWzOOZEAEaMGAuWQcRXfH2G71lSk8UOg0 +13gfqLptQ5GVj0VXXn7F+8qkBOvqlzdUMG+7AUcyM83cV5tkaWH4mx0ciU9cZwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAFFNzb5cy5gZnBWyATl4Lk0PZ3BwmcYQWpSk +U01UbSuvDV1Ai2TT1+7eVmGSX6bEHRBhNtMsJzzoKQm5EWR0zLVznxxIqbxhAe7i +F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY +oJ2daZH9 +-----END CERTIFICATE----- + +Verisign Class 4 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEDKIjprS9esTR/h/xCA3JfgwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgNCBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgNCBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQC68OTP+cSuhVS5B1f5j8V/aBH4xBewRNzjMHPVKmIquNDM +HO0oW369atyzkSTKQWI8/AIBvxwWMZQFl3Zuoq29YRdsTjCG8FE3KlDHqGKB3FtK +qsGgtG7rL+VXxbErQHDbWk2hjh+9Ax/YA9SPTJlxvOKCzFjomDqG04Y48wApHwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAIWMEsGnuVAVess+rLhDityq3RS6iYF+ATwj +cSGIL4LcY/oCRaxFWdcqWERbt5+BO5JoPeI3JPV7bI92NZYJqFmduc4jq3TWg/0y +cyfYaT5DdPauxYma51N86Xv2S/PBZYPejYqcPIiNOVn8qj8ijaHBZlCBckztImRP +T8qAkbYp +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCLW3VWhFSFCwDPrzhIzrGkMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN2E1Lm0+afY8wR4 +nN493GwTFtl63SRRZsDHJlkNrAYIwpTRMx/wgzUfbhvI3qpuFU5UJ+/EbRrsC+MO +8ESlV8dAWB6jRx9x7GD2bZTIGDnt/kIYVt/kTEkQeE4BdjVjEjbdZrwBBDajVWjV +ojYJrKshJlQGrT/KFOCsyq0GHZXi+J3x4GD/wn91K0zM2v6HmSHquv4+VNfSWXjb +PG7PoBMAGrgnoeS+Z5bKoMWznN3JdZ7rMJpfo83ZrngZPyPpXNspva1VyBtUjGP2 +6KbqxzcSXKMpHgLZ2x87tNcPVkeBFQRKr4Mn0cVYiMHd9qqnoxjaaKptEVHhv2Vr +n5Z20T0CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAq2aN17O6x5q25lXQBfGfMY1a +qtmqRiYPce2lrVNWYgFHKkTp/j90CxObufRNG7LRX7K20ohcs5/Ny9Sn2WCVhDr4 +wTcdYcrnsMXlkdpUpqwxga6X3s0IrLjAl4B/bnKk52kTlWUfxJM8/XmPBNQ+T+r3 +ns7NZ3xPZQL/kYVUc8f/NveGLezQXk//EZ9yBta4GvFMDSZl4kSAHsef493oCtrs +pSCAaWihT37ha88HQfqDjrw43bAuEbFrskLMmrz5SCJ5ShkPshw+IHTZasO+8ih4 +E1Z5T21Q6huwtVexN2ZYI/PcD98Kh8TvhgXVOBRgmaNL3gaWcSzy27YfpO8/7g== +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGTCCAwECEGFwy0mMX5hFKeewptlQW3owDQYJKoZIhvcNAQEFBQAwgcoxCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVy +aVNpZ24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24s +IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNp +Z24gQ2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eSAtIEczMB4XDTk5MTAwMTAwMDAwMFoXDTM2MDcxNjIzNTk1OVowgcoxCzAJBgNV +BAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVyaVNp +Z24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24sIElu +Yy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNpZ24g +Q2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAt +IEczMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArwoNwtUs22e5LeWU +J92lvuCwTY+zYVY81nzD9M0+hsuiiOLh2KRpxbXiv8GmR1BeRjmL1Za6tW8UvxDO +JxOeBUebMXoT2B/Z0wI3i60sR/COgQanDTAM6/c8DyAd3HJG7qUCyFvDyVZpTMUY +wZF7C9UTAJu878NIPkZgIIUq1ZC2zYugzDLdt/1AVbJQHFauzI13TccgTacxdu9o +koqQHgiBVrKtaaNS0MscxCM9H5n+TOgWY47GCI72MfbS+uV23bUckqNJzc0BzWjN +qWm6o+sdDZykIKbBoMXRRkwXbdKsZj+WjOCE1Db/IlnF+RFgqF8EffIa9iVCYQ/E +Srg+iQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQA0JhU8wI1NQ0kdvekhktdmnLfe +xbjQ5F1fdiLAJvmEOjr5jLX77GDx6M4EsMjdpwOPMPOY36TmpDHf0xwLRtxyID+u +7gU8pDM/CzmscHhzS5kr3zDCVLCoO1Wh/hYozUK9dG6A2ydEp85EXdQbkJgNHkKU +sQAsBNB0owIFImNjzYO1+8FtYmtpdf1dcEG59b98377BMnMiIYtYgXsVkXq642RI +sH/7NiXaldDxJBQX3RiAa0YjOVT1jmIJBB2UkKab5iXiQkWquJCtvgiPqQtCGJTP +cjnhsUPgKM+351psE2tJs//jGHyJizNdrDPXp/naOlXJWBD5qu9ats9LS98q +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b +N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t +KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu +kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm +CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ +Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu +imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te +2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe +DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC +/Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p +F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt +TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ== +-----END CERTIFICATE----- + +Verisign Class 4 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQDsoKeLbnVqAc/EfMwvlF7XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3LpRFpxlmr8Y+1 +GQ9Wzsy1HyDkniYlS+BzZYlZ3tCD5PUPtbut8XzoIfzk6AzufEUiGXaStBO3IFsJ ++mGuqPKljYXCKtbeZjbSmwL0qJJgfJxptI8kHtCGUvYynEFYHiK9zUVilQhu0Gbd +U6LM8BDcVHOLBKFGMzNcF0C5nk3T875Vg+ixiY5afJqWIpA7iCXy0lOIAgwLePLm +NxdLMEYH5IBtptiWLugs+BGzOA1mppvqySNb247i8xOOGlktqgLw7KSHZtzBP/XY +ufTsgsbSPZUd5cBPhMnZo0QoBmrXRazwa2rvTl/4EYIeOGM0ZlDUPpNz+jDDZq3/ +ky2X7wMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAj/ola09b5KROJ1WrIhVZPMq1 +CtRK26vdoV9TxaBXOcLORyu+OshWv8LZJxA6sQU8wHcxuzrTBXttmhwwjIDLk5Mq +g6sFUYICABFna/OIYUdfA5PVWw3g8dShMjWFsjrbsIKr0csKvE+MW8VLADsfKoKm +fjaF3H48ZwC15DtS4KjrXRX5xm3wrR0OhbepmnMUWluPQSjA1egtTaRezarZ7c7c +2NU8Qh0XwRJdRTjDOPP8hS6DRkiy1yBfkjaP53kPmF6Z6PDQpLv1U70qzlmwr25/ +bLvSHgCwIe34QWKCudiyxLtGUPMxxY8BqHTr9Xgn2uf3ZkPznoM+IKrDNWCRzg== +-----END CERTIFICATE----- + +Equifax Secure Global eBusiness CA +================================== + +-----BEGIN CERTIFICATE----- +MIICkDCCAfmgAwIBAgIBATANBgkqhkiG9w0BAQQFADBaMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEtMCsGA1UEAxMkRXF1aWZheCBT +ZWN1cmUgR2xvYmFsIGVCdXNpbmVzcyBDQS0xMB4XDTk5MDYyMTA0MDAwMFoXDTIw +MDYyMTA0MDAwMFowWjELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0VxdWlmYXggU2Vj +dXJlIEluYy4xLTArBgNVBAMTJEVxdWlmYXggU2VjdXJlIEdsb2JhbCBlQnVzaW5l +c3MgQ0EtMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAuucXkAJlsTRVPEnC +UdXfp9E3j9HngXNBUmCbnaEXJnitx7HoJpQytd4zjTov2/KaelpzmKNc6fuKcxtc +58O/gGzNqfTWK8D3+ZmqY6KxRwIP1ORROhI8bIpaVIRw28HFkM9yRcuoWcDNM50/ +o5brhTMhHD4ePmBudpxnhcXIw2ECAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgAH +MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUvqigdHJQa0S3ySPY+6j/s1dr +aGwwHQYDVR0OBBYEFL6ooHRyUGtEt8kj2Puo/7NXa2hsMA0GCSqGSIb3DQEBBAUA +A4GBADDiAVGqx+pf2rnQZQ8w1j7aDRRJbpGTJxQx78T3LUX47Me/okENI7SS+RkA +Z70Br83gcfxaz2TE4JaY0KNA4gGK7ycH8WUBikQtBmV1UsCGECAhX2xrD2yuCRyv +8qIYNMR1pHMc8Y3c7635s3a0kr/clRAevsvIO1qEYBlWlKlV +-----END CERTIFICATE----- + +Equifax Secure eBusiness CA 1 +============================= + +-----BEGIN CERTIFICATE----- +MIICgjCCAeugAwIBAgIBBDANBgkqhkiG9w0BAQQFADBTMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEmMCQGA1UEAxMdRXF1aWZheCBT +ZWN1cmUgZUJ1c2luZXNzIENBLTEwHhcNOTkwNjIxMDQwMDAwWhcNMjAwNjIxMDQw +MDAwWjBTMQswCQYDVQQGEwJVUzEcMBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5j +LjEmMCQGA1UEAxMdRXF1aWZheCBTZWN1cmUgZUJ1c2luZXNzIENBLTEwgZ8wDQYJ +KoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vGbwXt3fek6lfWg0XTzQaDJj0ItlZ1MRo +RvC0NcWFAyDGr0WlIVFFQesWWDYyb+JQYmT5/VGcqiTZ9J2DKocKIdMSODRsjQBu +WqDZQu4aIZX5UkxVWsUPOE9G+m34LjXWHXzr4vCwdYDIqROsvojvOm6rXyo4YgKw +Env+j6YDAgMBAAGjZjBkMBEGCWCGSAGG+EIBAQQEAwIABzAPBgNVHRMBAf8EBTAD +AQH/MB8GA1UdIwQYMBaAFEp4MlIR21kWNl7fwRQ2QGpHfEyhMB0GA1UdDgQWBBRK +eDJSEdtZFjZe38EUNkBqR3xMoTANBgkqhkiG9w0BAQQFAAOBgQB1W6ibAxHm6VZM +zfmpTMANmvPMZWnmJXbMWbfWVMMdzZmsGd20hdXgPfxiIKeES1hl8eL5lSE/9dR+ +WB5Hh1Q+WKG1tfgq73HnvMP2sUlG4tega+VWeponmHxGYhTnyfxuAxJ5gDgdSIKN +/Bf+KpYrtWKmpj29f5JZzVoqgrI3eQ== +-----END CERTIFICATE----- + +Equifax Secure eBusiness CA 2 +============================= + +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIEN3DPtTANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2Vj +dXJlIGVCdXNpbmVzcyBDQS0yMB4XDTk5MDYyMzEyMTQ0NVoXDTE5MDYyMzEyMTQ0 +NVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkVxdWlmYXggU2VjdXJlMSYwJAYD +VQQLEx1FcXVpZmF4IFNlY3VyZSBlQnVzaW5lc3MgQ0EtMjCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEA5Dk5kx5SBhsoNviyoynF7Y6yEb3+6+e0dMKP/wXn2Z0G +vxLIPw7y1tEkshHe0XMJitSxLJgJDR5QRrKDpkWNYmi7hRsgcDKqQM2mll/EcTc/ +BPO3QSQ5BxoeLmFYoBIL5aXfxavqN3HMHMg3OrmXUqesxWoklE6ce8/AatbfIb0C +AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEX +MBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2VjdXJl +IGVCdXNpbmVzcyBDQS0yMQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTkw +NjIzMTIxNDQ1WjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUUJ4L6q9euSBIplBq +y/3YIHqngnYwHQYDVR0OBBYEFFCeC+qvXrkgSKZQasv92CB6p4J2MAwGA1UdEwQF +MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA +A4GBAAyGgq3oThr1jokn4jVYPSm0B482UJW/bsGe68SQsoWou7dC4A8HOd/7npCy +0cE+U58DRLB+S/Rv5Hwf5+Kx5Lia78O9zt4LMjTZ3ijtM2vE1Nc9ElirfQkty3D1 +E4qUoSek1nDFbZS1yX2doNLGCEnZZpum0/QL3MUmV+GRMOrN +-----END CERTIFICATE----- + +Thawte Time Stamping CA +======================= + +-----BEGIN CERTIFICATE----- +MIICoTCCAgqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBizELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzAN +BgNVBAoTBlRoYXd0ZTEdMBsGA1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAd +BgNVBAMTFlRoYXd0ZSBUaW1lc3RhbXBpbmcgQ0EwHhcNOTcwMTAxMDAwMDAwWhcN +MjAxMjMxMjM1OTU5WjCBizELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4g +Q2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzANBgNVBAoTBlRoYXd0ZTEdMBsG +A1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAdBgNVBAMTFlRoYXd0ZSBUaW1l +c3RhbXBpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYrWHhhRYZT +6jR7UZztsOYuGA7+4F+oJ9O0yeB8WU4WDnNUYMF/9p8u6TqFJBU820cEY8OexJQa +Wt9MevPZQx08EHp5JduQ/vBR5zDWQQD9nyjfeb6Uu522FOMjhdepQeBMpHmwKxqL +8vg7ij5FrHGSALSQQZj7X+36ty6K+Ig3AgMBAAGjEzARMA8GA1UdEwEB/wQFMAMB +Af8wDQYJKoZIhvcNAQEEBQADgYEAZ9viwuaHPUCDhjc1fR/OmsMMZiCouqoEiYbC +9RAIDb/LogWK0E02PvTX72nGXuSwlG9KuefeW4i2e9vjJ+V2w/A1wcu1J5szedyQ +pgCed/r8zSeUQhac0xxo7L9c3eWpexAKMnRUEzGLhQOEkbdYATAUOK8oyvyxUBkZ +CayJSdM= +-----END CERTIFICATE----- + +thawte Primary Root CA +====================== + +-----BEGIN CERTIFICATE----- +MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB +qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV +BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw +NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j +LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG +A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl +IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs +W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta +3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk +6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6 +Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J +NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA +MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP +r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU +DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz +YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX +xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2 +/qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/ +LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7 +jVaMaA== +-----END CERTIFICATE----- + +VeriSign Class 3 Public Primary Certification Authority - G5 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB +yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp +U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW +ZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL +MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW +ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln +biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp +U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y +aXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1 +nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex +t0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz +SdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG +BO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+ +rCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/ +NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH +BgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy +aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv +MzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE +p6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y +5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK +WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ +4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N +hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq +-----END CERTIFICATE----- + +Entrust.net Secure Server Certification Authority +================================================= + +-----BEGIN CERTIFICATE----- +MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC +VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u +ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc +KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u +ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1 +MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE +ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j +b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF +bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg +U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA +A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/ +I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3 +wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC +AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb +oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5 +BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p +dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk +MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp +b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu +dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0 +MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi +E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa +MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI +hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN +95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd +2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI= +-----END CERTIFICATE----- + +Go Daddy Certification Authority Root Certificate Bundle +======================================================== + +-----BEGIN CERTIFICATE----- +MIIE3jCCA8agAwIBAgICAwEwDQYJKoZIhvcNAQEFBQAwYzELMAkGA1UEBhMCVVMx +ITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28g +RGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjExMTYw +MTU0MzdaFw0yNjExMTYwMTU0MzdaMIHKMQswCQYDVQQGEwJVUzEQMA4GA1UECBMH +QXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5j +b20sIEluYy4xMzAxBgNVBAsTKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5j +b20vcmVwb3NpdG9yeTEwMC4GA1UEAxMnR28gRGFkZHkgU2VjdXJlIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5MREwDwYDVQQFEwgwNzk2OTI4NzCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMQt1RWMnCZM7DI161+4WQFapmGBWTtwY6vj3D3H +KrjJM9N55DrtPDAjhI6zMBS2sofDPZVUBJ7fmd0LJR4h3mUpfjWoqVTr9vcyOdQm +VZWt7/v+WIbXnvQAjYwqDL1CBM6nPwT27oDyqu9SoWlm2r4arV3aLGbqGmu75RpR +SgAvSMeYddi5Kcju+GZtCpyz8/x4fKL4o/K1w/O5epHBp+YlLpyo7RJlbmr2EkRT +cDCVw5wrWCs9CHRK8r5RsL+H0EwnWGu1NcWdrxcx+AuP7q2BNgWJCJjPOq8lh8BJ +6qf9Z/dFjpfMFDniNoW1fho3/Rb2cRGadDAW/hOUoz+EDU8CAwEAAaOCATIwggEu +MB0GA1UdDgQWBBT9rGEyk2xF1uLuhV+auud2mWjM5zAfBgNVHSMEGDAWgBTSxLDS +kdRMEXGzYcs9of7dqGrU4zASBgNVHRMBAf8ECDAGAQH/AgEAMDMGCCsGAQUFBwEB +BCcwJTAjBggrBgEFBQcwAYYXaHR0cDovL29jc3AuZ29kYWRkeS5jb20wRgYDVR0f +BD8wPTA7oDmgN4Y1aHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNvbS9yZXBv +c2l0b3J5L2dkcm9vdC5jcmwwSwYDVR0gBEQwQjBABgRVHSAAMDgwNgYIKwYBBQUH +AgEWKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5jb20vcmVwb3NpdG9yeTAO +BgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBANKGwOy9+aG2Z+5mC6IG +OgRQjhVyrEp0lVPLN8tESe8HkGsz2ZbwlFalEzAFPIUyIXvJxwqoJKSQ3kbTJSMU +A2fCENZvD117esyfxVgqwcSeIaha86ykRvOe5GPLL5CkKSkB2XIsKd83ASe8T+5o +0yGPwLPk9Qnt0hCqU7S+8MxZC9Y7lhyVJEnfzuz9p0iRFEUOOjZv2kWzRaJBydTX +RE4+uXR21aITVSzGh6O1mawGhId/dQb8vxRMDsxuxN89txJx9OjxUUAiKEngHUuH +qDTMBqLdElrRhjZkAzVvb3du6/KFUJheqwNTrZEjYx8WnM25sgVjOuH0aBsXBTWV +U+4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIE+zCCBGSgAwIBAgICAQ0wDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1Zh +bGlDZXJ0IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIElu +Yy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24g +QXV0aG9yaXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAe +BgkqhkiG9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTA0MDYyOTE3MDYyMFoX +DTI0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBE +YWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3MgMiBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgC +ggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv +2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+q +N1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiO +r18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lN +f4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+YihfukEH +U1jPEX44dMX4/7VpkI+EdOqXG68CAQOjggHhMIIB3TAdBgNVHQ4EFgQU0sSw0pHU +TBFxs2HLPaH+3ahq1OMwgdIGA1UdIwSByjCBx6GBwaSBvjCBuzEkMCIGA1UEBxMb +VmFsaUNlcnQgVmFsaWRhdGlvbiBOZXR3b3JrMRcwFQYDVQQKEw5WYWxpQ2VydCwg +SW5jLjE1MDMGA1UECxMsVmFsaUNlcnQgQ2xhc3MgMiBQb2xpY3kgVmFsaWRhdGlv +biBBdXRob3JpdHkxITAfBgNVBAMTGGh0dHA6Ly93d3cudmFsaWNlcnQuY29tLzEg +MB4GCSqGSIb3DQEJARYRaW5mb0B2YWxpY2VydC5jb22CAQEwDwYDVR0TAQH/BAUw +AwEB/zAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3NwLmdv +ZGFkZHkuY29tMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jZXJ0aWZpY2F0ZXMu +Z29kYWRkeS5jb20vcmVwb3NpdG9yeS9yb290LmNybDBLBgNVHSAERDBCMEAGBFUd +IAAwODA2BggrBgEFBQcCARYqaHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNv +bS9yZXBvc2l0b3J5MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOBgQC1 +QPmnHfbq/qQaQlpE9xXUhUaJwL6e4+PrxeNYiY+Sn1eocSxI0YGyeR+sBjUZsE4O +WBsUs5iB0QQeyAfJg594RAoYC5jcdnplDQ1tgMQLARzLrUc+cb53S8wGd9D0Vmsf +SxOaFIqII6hR8INMqzW/Rn453HWkrugp++85j09VZw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 +IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz +BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y +aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG +9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy +NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y +azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs +YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw +Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl +cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY +dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9 +WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS +v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v +UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu +IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC +W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd +-----END CERTIFICATE----- + +GeoTrust Global CA +================== + +-----BEGIN CERTIFICATE----- +MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT +MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 +aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAw +WjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UE +AxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9m +OSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIu +T8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6c +JmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmR +Cw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5asz +PeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjm +aPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrM +TjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+g +LaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBO +BgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2Vv +dHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GB +AHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrL +NhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1W +b8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S +-----END CERTIFICATE----- + diff --git a/libs/httplib2/iri2uri.py b/libs/httplib2/iri2uri.py index 70667edf..d88c91fd 100644 --- a/libs/httplib2/iri2uri.py +++ b/libs/httplib2/iri2uri.py @@ -16,7 +16,7 @@ import urlparse # Convert an IRI to a URI following the rules in RFC 3987 -# +# # The characters we need to enocde and escape are defined in the spec: # # iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD @@ -28,28 +28,28 @@ import urlparse # / %xD0000-DFFFD / %xE1000-EFFFD escape_range = [ - (0xA0, 0xD7FF ), - (0xE000, 0xF8FF ), - (0xF900, 0xFDCF ), - (0xFDF0, 0xFFEF), - (0x10000, 0x1FFFD ), - (0x20000, 0x2FFFD ), - (0x30000, 0x3FFFD), - (0x40000, 0x4FFFD ), - (0x50000, 0x5FFFD ), - (0x60000, 0x6FFFD), - (0x70000, 0x7FFFD ), - (0x80000, 0x8FFFD ), - (0x90000, 0x9FFFD), - (0xA0000, 0xAFFFD ), - (0xB0000, 0xBFFFD ), - (0xC0000, 0xCFFFD), - (0xD0000, 0xDFFFD ), - (0xE1000, 0xEFFFD), - (0xF0000, 0xFFFFD ), - (0x100000, 0x10FFFD) + (0xA0, 0xD7FF), + (0xE000, 0xF8FF), + (0xF900, 0xFDCF), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD), + (0x20000, 0x2FFFD), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD), + (0x50000, 0x5FFFD), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD), + (0x80000, 0x8FFFD), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD), + (0xB0000, 0xBFFFD), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD), + (0x100000, 0x10FFFD), ] - + def encode(c): retval = c i = ord(c) @@ -63,19 +63,19 @@ def encode(c): def iri2uri(uri): - """Convert an IRI to a URI. Note that IRIs must be + """Convert an IRI to a URI. Note that IRIs must be passed in a unicode strings. That is, do not utf-8 encode - the IRI before passing it into the function.""" + the IRI before passing it into the function.""" if isinstance(uri ,unicode): (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) authority = authority.encode('idna') # For each character in 'ucschar' or 'iprivate' # 1. encode as utf-8 - # 2. then %-encode each octet of that utf-8 + # 2. then %-encode each octet of that utf-8 uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) uri = "".join([encode(c) for c in uri]) return uri - + if __name__ == "__main__": import unittest @@ -83,7 +83,7 @@ if __name__ == "__main__": def test_uris(self): """Test that URIs are invariant under the transformation.""" - invariant = [ + invariant = [ u"ftp://ftp.is.co.za/rfc/rfc1808.txt", u"http://www.ietf.org/rfc/rfc2396.txt", u"ldap://[2001:db8::7]/c=GB?objectClass?one", @@ -94,7 +94,7 @@ if __name__ == "__main__": u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] for uri in invariant: self.assertEqual(uri, iri2uri(uri)) - + def test_iri(self): """ Test that the right type of escaping is done for each part of the URI.""" self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) @@ -107,4 +107,4 @@ if __name__ == "__main__": unittest.main() - + diff --git a/libs/httplib2/socks.py b/libs/httplib2/socks.py new file mode 100644 index 00000000..0991f4cf --- /dev/null +++ b/libs/httplib2/socks.py @@ -0,0 +1,438 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +""" + +import base64 +import socket +import struct +import sys + +if getattr(socket, 'socket', None) is None: + raise ImportError('socket.socket missing, proxy support unusable') + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 +PROXY_TYPE_HTTP_NO_TUNNEL = 4 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): pass +class GeneralProxyError(ProxyError): pass +class Socks5AuthError(ProxyError): pass +class Socks5Error(ProxyError): pass +class Socks4Error(ProxyError): pass +class HTTPError(ProxyError): pass + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + +def wrapmodule(module): + """wrapmodule(module) + Attempts to replace a module's socket library with a SOCKS socket. Must set + a default proxy using setdefaultproxy(...) first. + This will only work on modules that import socket directly into the namespace; + most of the Python Standard Library falls into this category. + """ + if _defaultproxy != None: + module.socket.socket = socksocket + else: + raise GeneralProxyError((4, "no proxy specified")) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + self.__httptunnel = True + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = self.recv(count) + while len(data) < count: + d = self.recv(count-len(data)) + if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) + data = data + d + return data + + def sendall(self, content, *args): + """ override socket.socket.sendall method to rewrite the header + for non-tunneling proxies if needed + """ + if not self.__httptunnel: + content = self.__rewriteproxy(content) + return super(socksocket, self).sendall(content, *args) + + def __rewriteproxy(self, header): + """ rewrite HTTP request headers to support non-tunneling proxies + (i.e. those which do not support the CONNECT method). + This only works for HTTP (not HTTPS) since HTTPS requires tunneling. + """ + host, endpt = None, None + hdrs = header.split("\r\n") + for hdr in hdrs: + if hdr.lower().startswith("host:"): + host = hdr + elif hdr.lower().startswith("get") or hdr.lower().startswith("post"): + endpt = hdr + if host and endpt: + hdrs.remove(host) + hdrs.remove(endpt) + host = host.split(" ")[1] + endpt = endpt.split(" ") + if (self.__proxy[4] != None and self.__proxy[5] != None): + hdrs.insert(0, self.__getauthheader()) + hdrs.insert(0, "Host: %s" % host) + hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2])) + return "\r\n".join(hdrs) + + def __getauthheader(self): + auth = self.__proxy[4] + ":" + self.__proxy[5] + return "Proxy-Authorization: Basic " + base64.b64encode(auth) + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02)) + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00)) + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1:2] == chr(0x00).encode(): + # No authentication is required + pass + elif chosenauth[1:2] == chr(0x02).encode(): + # Okay, we need to perform a basic username/password + # authentication. + self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0:1] != chr(0x01).encode(): + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1:2] != chr(0x00).encode(): + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == chr(0xFF).encode(): + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = struct.pack('BBB', 0x05, 0x01, 0x00) + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + chr(0x01).encode() + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + chr(0x01).encode() + ipaddr + req = req + struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1:2] != chr(0x00).encode(): + # Connection failed + self.close() + if ord(resp[1:2])<=8: + raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3:4] == chr(0x01).encode(): + boundaddr = self.__recvall(4) + elif resp[3:4] == chr(0x03).encode(): + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4:5])) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01) + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + chr(0x00).encode() + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + chr(0x00).encode() + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0:1] != chr(0x00).encode(): + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1:2] != chr(0x5A).encode(): + # Server returned an error + self.close() + if ord(resp[1:2]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"] + headers += ["Host: ", destaddr, "\r\n"] + if (self.__proxy[4] != None and self.__proxy[5] != None): + headers += [self.__getauthheader(), "\r\n"] + headers.append("\r\n") + self.sendall("".join(headers).encode()) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n".encode()) == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ".encode(), 2) + if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self, despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + if destpair[1] == 443: + self.__negotiatehttp(destpair[0],destpair[1]) + else: + self.__httptunnel = False + elif self.__proxy[0] == None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/libs/pyasn1/__init__.py b/libs/pyasn1/__init__.py index 7de39fe5..88aff79c 100644 --- a/libs/pyasn1/__init__.py +++ b/libs/pyasn1/__init__.py @@ -1 +1,8 @@ -majorVersionId = '1' +import sys + +# http://www.python.org/dev/peps/pep-0396/ +__version__ = '0.1.7' + +if sys.version_info[:2] < (2, 4): + raise RuntimeError('PyASN1 requires Python 2.4 or later') + diff --git a/libs/pyasn1/codec/__init__.py b/libs/pyasn1/codec/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/__init__.py +++ b/libs/pyasn1/codec/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/ber/__init__.py b/libs/pyasn1/codec/ber/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/ber/__init__.py +++ b/libs/pyasn1/codec/ber/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/ber/decoder.py b/libs/pyasn1/codec/ber/decoder.py index ae9311cb..be0cf490 100644 --- a/libs/pyasn1/codec/ber/decoder.py +++ b/libs/pyasn1/codec/ber/decoder.py @@ -1,21 +1,24 @@ # BER decoder from pyasn1.type import tag, base, univ, char, useful, tagmap from pyasn1.codec.ber import eoo -from pyasn1.compat.octets import oct2int, octs2ints -from pyasn1 import error +from pyasn1.compat.octets import oct2int, octs2ints, isOctetsType +from pyasn1 import debug, error class AbstractDecoder: protoComponent = None def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - raise error.PyAsn1Error('Decoder not implemented for %s' % tagSet) + length, state, decodeFun, substrateFun): + raise error.PyAsn1Error('Decoder not implemented for %s' % (tagSet,)) def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % tagSet) + length, state, decodeFun, substrateFun): + raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,)) class AbstractSimpleDecoder(AbstractDecoder): + tagFormats = (tag.tagFormatSimple,) def _createComponent(self, asn1Spec, tagSet, value=None): + if tagSet[0][1] not in self.tagFormats: + raise error.PyAsn1Error('Invalid tag format %r for %r' % (tagSet[0], self.protoComponent,)) if asn1Spec is None: return self.protoComponent.clone(value, tagSet) elif value is None: @@ -24,7 +27,10 @@ class AbstractSimpleDecoder(AbstractDecoder): return asn1Spec.clone(value) class AbstractConstructedDecoder(AbstractDecoder): + tagFormats = (tag.tagFormatConstructed,) def _createComponent(self, asn1Spec, tagSet, value=None): + if tagSet[0][1] not in self.tagFormats: + raise error.PyAsn1Error('Invalid tag format %r for %r' % (tagSet[0], self.protoComponent,)) if asn1Spec is None: return self.protoComponent.clone(tagSet) else: @@ -32,19 +38,34 @@ class AbstractConstructedDecoder(AbstractDecoder): class EndOfOctetsDecoder(AbstractSimpleDecoder): def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - return eoo.endOfOctets, substrate[:length] + length, state, decodeFun, substrateFun): + return eoo.endOfOctets, substrate[length:] class ExplicitTagDecoder(AbstractSimpleDecoder): + protoComponent = univ.Any('') + tagFormats = (tag.tagFormatConstructed,) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - return decodeFun(substrate[:length], asn1Spec, tagSet, length) + length, state, decodeFun, substrateFun): + if substrateFun: + return substrateFun( + self._createComponent(asn1Spec, tagSet, ''), + substrate, length + ) + head, tail = substrate[:length], substrate[length:] + value, _ = decodeFun(head, asn1Spec, tagSet, length) + return value, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): + if substrateFun: + return substrateFun( + self._createComponent(asn1Spec, tagSet, ''), + substrate, length + ) value, substrate = decodeFun(substrate, asn1Spec, tagSet, length) terminator, substrate = decodeFun(substrate) - if terminator == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(terminator) and \ + terminator == eoo.endOfOctets: return value, substrate else: raise error.PyAsn1Error('Missing end-of-octets terminator') @@ -71,79 +92,71 @@ class IntegerDecoder(AbstractSimpleDecoder): '\xfb': -5 } - def _valueFilter(self, value): - try: - return int(value) - except OverflowError: - return value - def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: - raise error.PyAsn1Error('Empty substrate') - if substrate in self.precomputedValues: - value = self.precomputedValues[substrate] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: + return self._createComponent(asn1Spec, tagSet, 0), tail + if head in self.precomputedValues: + value = self.precomputedValues[head] else: - firstOctet = oct2int(substrate[0]) + firstOctet = oct2int(head[0]) if firstOctet & 0x80: value = -1 else: value = 0 - for octet in substrate: + for octet in head: value = value << 8 | oct2int(octet) - value = self._valueFilter(value) - return self._createComponent(asn1Spec, tagSet, value), substrate + return self._createComponent(asn1Spec, tagSet, value), tail class BooleanDecoder(IntegerDecoder): protoComponent = univ.Boolean(0) - def _valueFilter(self, value): - if value: - return 1 - else: - return 0 + def _createComponent(self, asn1Spec, tagSet, value=None): + return IntegerDecoder._createComponent(self, asn1Spec, tagSet, value and 1 or 0) class BitStringDecoder(AbstractSimpleDecoder): protoComponent = univ.BitString(()) + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] if tagSet[0][1] == tag.tagFormatSimple: # XXX what tag to check? - if not substrate: - raise error.PyAsn1Error('Missing initial octet') - trailingBits = oct2int(substrate[0]) + if not head: + raise error.PyAsn1Error('Empty substrate') + trailingBits = oct2int(head[0]) if trailingBits > 7: raise error.PyAsn1Error( 'Trailing bits overflow %s' % trailingBits ) - substrate = substrate[1:] - lsb = p = 0; l = len(substrate)-1; b = () + head = head[1:] + lsb = p = 0; l = len(head)-1; b = () while p <= l: if p == l: lsb = trailingBits j = 7 - o = oct2int(substrate[p]) + o = oct2int(head[p]) while j >= lsb: b = b + ((o>>j)&0x01,) j = j - 1 p = p + 1 - return self._createComponent(asn1Spec, tagSet, b), '' + return self._createComponent(asn1Spec, tagSet, b), tail r = self._createComponent(asn1Spec, tagSet, ()) - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun(substrate) + if substrateFun: + return substrateFun(r, substrate, length) + while head: + component, head = decodeFun(head) r = r + component - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -154,27 +167,29 @@ class BitStringDecoder(AbstractSimpleDecoder): class OctetStringDecoder(AbstractSimpleDecoder): protoComponent = univ.OctetString('') + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] if tagSet[0][1] == tag.tagFormatSimple: # XXX what tag to check? - return self._createComponent(asn1Spec, tagSet, substrate), '' + return self._createComponent(asn1Spec, tagSet, head), tail r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun(substrate) + if substrateFun: + return substrateFun(r, substrate, length) + while head: + component, head = decodeFun(head) r = r + component - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -186,93 +201,89 @@ class OctetStringDecoder(AbstractSimpleDecoder): class NullDecoder(AbstractSimpleDecoder): protoComponent = univ.Null('') def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) - if substrate: - raise error.PyAsn1Error('Unexpected substrate for Null') - return r, substrate + if head: + raise error.PyAsn1Error('Unexpected %d-octet substrate for Null' % length) + return r, tail class ObjectIdentifierDecoder(AbstractSimpleDecoder): protoComponent = univ.ObjectIdentifier(()) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: raise error.PyAsn1Error('Empty substrate') - oid = (); index = 0 - # Get the first subid - subId = oct2int(substrate[index]) - oid = oid + divmod(subId, 40) - index = index + 1 - substrateLen = len(substrate) - + # Get the first subid + subId = oct2int(head[0]) + oid = divmod(subId, 40) + + index = 1 + substrateLen = len(head) while index < substrateLen: - subId = oct2int(substrate[index]) - if subId < 128: - oid = oid + (subId,) - index = index + 1 - else: + subId = oct2int(head[index]) + index = index + 1 + if subId == 128: + # ASN.1 spec forbids leading zeros (0x80) in sub-ID OID + # encoding, tolerating it opens a vulnerability. + # See http://www.cosic.esat.kuleuven.be/publications/article-1432.pdf page 7 + raise error.PyAsn1Error('Invalid leading 0x80 in sub-OID') + elif subId > 128: # Construct subid from a number of octets nextSubId = subId subId = 0 - while nextSubId >= 128 and index < substrateLen: + while nextSubId >= 128: subId = (subId << 7) + (nextSubId & 0x7F) + if index >= substrateLen: + raise error.SubstrateUnderrunError( + 'Short substrate for sub-OID past %s' % (oid,) + ) + nextSubId = oct2int(head[index]) index = index + 1 - nextSubId = oct2int(substrate[index]) - if index == substrateLen: - raise error.SubstrateUnderrunError( - 'Short substrate for OID %s' % oid - ) subId = (subId << 7) + nextSubId - oid = oid + (subId,) - index = index + 1 - return self._createComponent(asn1Spec, tagSet, oid), substrate[index:] + oid = oid + (subId,) + return self._createComponent(asn1Spec, tagSet, oid), tail class RealDecoder(AbstractSimpleDecoder): protoComponent = univ.Real() def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] - if not length: - raise error.SubstrateUnderrunError('Short substrate for Real') - fo = oct2int(substrate[0]); substrate = substrate[1:] - if fo & 0x40: # infinite value - value = fo & 0x01 and '-inf' or 'inf' - elif fo & 0x80: # binary enoding - if fo & 0x11 == 0: - n = 1 - elif fo & 0x01: - n = 2 - elif fo & 0x02: - n = 3 - else: - n = oct2int(substrate[0]) - eo, substrate = substrate[:n], substrate[n:] - if not eo or not substrate: + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: + return self._createComponent(asn1Spec, tagSet, 0.0), tail + fo = oct2int(head[0]); head = head[1:] + if fo & 0x80: # binary enoding + n = (fo & 0x03) + 1 + if n == 4: + n = oct2int(head[0]) + eo, head = head[:n], head[n:] + if not eo or not head: raise error.PyAsn1Error('Real exponent screwed') - e = 0 + e = oct2int(eo[0]) & 0x80 and -1 or 0 while eo: # exponent e <<= 8 e |= oct2int(eo[0]) eo = eo[1:] p = 0 - while substrate: # value + while head: # value p <<= 8 - p |= oct2int(substrate[0]) - substrate = substrate[1:] + p |= oct2int(head[0]) + head = head[1:] if fo & 0x40: # sign bit p = -p value = (p, 2, e) + elif fo & 0x40: # infinite value + value = fo & 0x01 and '-inf' or 'inf' elif fo & 0xc0 == 0: # character encoding try: if fo & 0x3 == 0x1: # NR1 - value = (int(substrate), 10, 0) + value = (int(head), 10, 0) elif fo & 0x3 == 0x2: # NR2 - value = float(substrate) + value = float(head) elif fo & 0x3 == 0x3: # NR3 - value = float(substrate) + value = float(head) else: raise error.SubstrateUnderrunError( 'Unknown NR (tag %s)' % fo @@ -281,13 +292,11 @@ class RealDecoder(AbstractSimpleDecoder): raise error.SubstrateUnderrunError( 'Bad character Real syntax' ) - elif fo & 0xc0 == 0x40: # special real value - pass else: raise error.SubstrateUnderrunError( 'Unknown encoding (tag %s)' % fo ) - return self._createComponent(asn1Spec, tagSet, value), substrate + return self._createComponent(asn1Spec, tagSet, value), tail class SequenceDecoder(AbstractConstructedDecoder): protoComponent = univ.Sequence() @@ -301,17 +310,15 @@ class SequenceDecoder(AbstractConstructedDecoder): return r.getComponentPositionNearType(t, idx) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) idx = 0 - if not decodeFun: - return r, substrate - while substrate: + if substrateFun: + return substrateFun(r, substrate, length) + while head: asn1Spec = self._getComponentTagMap(r, idx) - component, substrate = decodeFun( - substrate, asn1Spec - ) + component, head = decodeFun(head, asn1Spec) idx = self._getComponentPositionByType( r, component.getEffectiveTagSet(), idx ) @@ -319,18 +326,19 @@ class SequenceDecoder(AbstractConstructedDecoder): idx = idx + 1 r.setDefaultComponents() r.verifySizeSpec() - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) idx = 0 while substrate: asn1Spec = self._getComponentTagMap(r, idx) - if not decodeFun: - return r, substrate component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break idx = self._getComponentPositionByType( r, component.getEffectiveTagSet(), idx @@ -348,32 +356,31 @@ class SequenceDecoder(AbstractConstructedDecoder): class SequenceOfDecoder(AbstractConstructedDecoder): protoComponent = univ.SequenceOf() def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) asn1Spec = r.getComponentType() idx = 0 - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun( - substrate, asn1Spec - ) + while head: + component, head = decodeFun(head, asn1Spec) r.setComponentByPosition(idx, component, asn1Spec is None) idx = idx + 1 r.verifySizeSpec() - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) asn1Spec = r.getComponentType() idx = 0 - if not decodeFun: - return r, substrate while substrate: component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r.setComponentByPosition(idx, component, asn1Spec is None) idx = idx + 1 @@ -401,43 +408,68 @@ class SetOfDecoder(SequenceOfDecoder): class ChoiceDecoder(AbstractConstructedDecoder): protoComponent = univ.Choice() + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) if r.getTagSet() == tagSet: # explicitly tagged Choice - component, substrate = decodeFun( - substrate, r.getComponentTagMap() + component, head = decodeFun( + head, r.getComponentTagMap() ) else: - component, substrate = decodeFun( - substrate, r.getComponentTagMap(), tagSet, length, state + component, head = decodeFun( + head, r.getComponentTagMap(), tagSet, length, state ) if isinstance(component, univ.Choice): effectiveTagSet = component.getEffectiveTagSet() else: effectiveTagSet = component.getTagSet() r.setComponentByType(effectiveTagSet, component, 0, asn1Spec is None) + return r, tail + + def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, + length, state, decodeFun, substrateFun): + r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) + if r.getTagSet() == tagSet: # explicitly tagged Choice + component, substrate = decodeFun(substrate, r.getComponentTagMap()) + eooMarker, substrate = decodeFun(substrate) # eat up EOO marker + if not eoo.endOfOctets.isSameTypeWith(eooMarker) or \ + eooMarker != eoo.endOfOctets: + raise error.PyAsn1Error('No EOO seen before substrate ends') + else: + component, substrate= decodeFun( + substrate, r.getComponentTagMap(), tagSet, length, state + ) + if isinstance(component, univ.Choice): + effectiveTagSet = component.getEffectiveTagSet() + else: + effectiveTagSet = component.getTagSet() + r.setComponentByType(effectiveTagSet, component, 0, asn1Spec is None) return r, substrate - indefLenValueDecoder = valueDecoder - class AnyDecoder(AbstractSimpleDecoder): protoComponent = univ.Any() + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): if asn1Spec is None or \ asn1Spec is not None and tagSet != asn1Spec.getTagSet(): # untagged Any container, recover inner header substrate length = length + len(fullSubstrate) - len(substrate) substrate = fullSubstrate - substrate = substrate[:length] - return self._createComponent(asn1Spec, tagSet, value=substrate), '' + if substrateFun: + return substrateFun(self._createComponent(asn1Spec, tagSet), + substrate, length) + head, tail = substrate[:length], substrate[length:] + return self._createComponent(asn1Spec, tagSet, value=head), tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): if asn1Spec is not None and tagSet == asn1Spec.getTagSet(): # tagged Any type -- consume header substrate header = '' @@ -450,11 +482,12 @@ class AnyDecoder(AbstractSimpleDecoder): # Any components do not inherit initial tag asn1Spec = self.protoComponent - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -550,7 +583,10 @@ class Decoder: self.__tagSetCache = {} def __call__(self, substrate, asn1Spec=None, tagSet=None, - length=None, state=stDecodeTag, recursiveFlag=1): + length=None, state=stDecodeTag, recursiveFlag=1, + substrateFun=None): + if debug.logger & debug.flagDecoder: + debug.logger('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate))) fullSubstrate = substrate while state != stStop: if state == stDecodeTag: @@ -559,6 +595,9 @@ class Decoder: raise error.SubstrateUnderrunError( 'Short octet stream on tag decoding' ) + if not isOctetsType(substrate) and \ + not isinstance(substrate, univ.OctetString): + raise error.PyAsn1Error('Bad octet stream type') firstOctet = substrate[0] substrate = substrate[1:] @@ -598,6 +637,7 @@ class Decoder: else: tagSet = lastTag + tagSet state = stDecodeLength + debug.logger and debug.logger & debug.flagDecoder and debug.logger('tag decoded into %r, decoding length' % tagSet) if state == stDecodeLength: # Decode length if not substrate: @@ -625,12 +665,13 @@ class Decoder: for char in lengthString: length = (length << 8) | oct2int(char) size = size + 1 - state = stGetValueDecoder substrate = substrate[size:] if length != -1 and len(substrate) < length: raise error.SubstrateUnderrunError( '%d-octet short' % (length - len(substrate)) ) + state = stGetValueDecoder + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length]))) if state == stGetValueDecoder: if asn1Spec is None: state = stGetValueDecoderByTag @@ -669,14 +710,27 @@ class Decoder: state = stDecodeValue else: state = stTryAsExplicitTag + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('codec %s chosen by a built-in type, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as explicit tag')) + debug.scope.push(concreteDecoder is None and '?' or concreteDecoder.protoComponent.__class__.__name__) if state == stGetValueDecoderByAsn1Spec: if isinstance(asn1Spec, (dict, tagmap.TagMap)): if tagSet in asn1Spec: __chosenSpec = asn1Spec[tagSet] else: __chosenSpec = None + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('candidate ASN.1 spec is a map of:') + for t, v in asn1Spec.getPosMap().items(): + debug.logger(' %r -> %s' % (t, v.__class__.__name__)) + if asn1Spec.getNegMap(): + debug.logger('but neither of: ') + for i in asn1Spec.getNegMap().items(): + debug.logger(' %r -> %s' % (t, v.__class__.__name__)) + debug.logger('new candidate ASN.1 spec is %s, chosen by %r' % (__chosenSpec is None and '<none>' or __chosenSpec.__class__.__name__, tagSet)) else: __chosenSpec = asn1Spec + debug.logger and debug.logger & debug.flagDecoder and debug.logger('candidate ASN.1 spec is %s' % asn1Spec.__class__.__name__) if __chosenSpec is not None and ( tagSet == __chosenSpec.getTagSet() or \ tagSet in __chosenSpec.getTagMap() @@ -687,9 +741,11 @@ class Decoder: __chosenSpec.typeId in self.__typeMap: # ambiguous type concreteDecoder = self.__typeMap[__chosenSpec.typeId] + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value decoder chosen for an ambiguous type by type ID %s' % (__chosenSpec.typeId,)) elif baseTagSet in self.__tagMap: # base type or tagged subtype concreteDecoder = self.__tagMap[baseTagSet] + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value decoder chosen by base %r' % (baseTagSet,)) else: concreteDecoder = None if concreteDecoder: @@ -700,8 +756,13 @@ class Decoder: elif tagSet == self.__endOfOctetsTagSet: concreteDecoder = self.__tagMap[tagSet] state = stDecodeValue + debug.logger and debug.logger & debug.flagDecoder and debug.logger('end-of-octets found') else: + concreteDecoder = None state = stTryAsExplicitTag + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('codec %s chosen by ASN.1 spec, decoding %s' % (state == stDecodeValue and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as explicit tag')) + debug.scope.push(__chosenSpec is None and '?' or __chosenSpec.__class__.__name__) if state == stTryAsExplicitTag: if tagSet and \ tagSet[0][1] == tag.tagFormatConstructed and \ @@ -710,34 +771,35 @@ class Decoder: concreteDecoder = explicitTagDecoder state = stDecodeValue else: + concreteDecoder = None state = self.defaultErrorState + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s chosen, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as failure')) if state == stDumpRawValue: concreteDecoder = self.defaultRawDecoder + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s chosen, decoding value' % concreteDecoder.__class__.__name__) state = stDecodeValue if state == stDecodeValue: - if recursiveFlag: - decodeFun = self - else: - decodeFun = None + if recursiveFlag == 0 and not substrateFun: # legacy + substrateFun = lambda a,b,c: (a,b[:c]) if length == -1: # indef length value, substrate = concreteDecoder.indefLenValueDecoder( fullSubstrate, substrate, asn1Spec, tagSet, length, - stGetValueDecoder, decodeFun + stGetValueDecoder, self, substrateFun ) else: - value, _substrate = concreteDecoder.valueDecoder( + value, substrate = concreteDecoder.valueDecoder( fullSubstrate, substrate, asn1Spec, tagSet, length, - stGetValueDecoder, decodeFun + stGetValueDecoder, self, substrateFun ) - if recursiveFlag: - substrate = substrate[length:] - else: - substrate = _substrate state = stStop + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, value.prettyPrint(), substrate and debug.hexdump(substrate) or '<none>')) if state == stErrorCondition: raise error.PyAsn1Error( '%r not in asn1Spec: %r' % (tagSet, asn1Spec) ) + if debug.logger and debug.logger & debug.flagDecoder: + debug.scope.pop() + debug.logger('decoder left scope %s, call completed' % debug.scope) return value, substrate decode = Decoder(tagMap, typeMap) diff --git a/libs/pyasn1/codec/ber/encoder.py b/libs/pyasn1/codec/ber/encoder.py index 2149b0ba..173949d0 100644 --- a/libs/pyasn1/codec/ber/encoder.py +++ b/libs/pyasn1/codec/ber/encoder.py @@ -1,8 +1,8 @@ # BER encoder from pyasn1.type import base, tag, univ, char, useful from pyasn1.codec.ber import eoo -from pyasn1.compat.octets import int2oct, ints2octs, null, str2octs -from pyasn1 import error +from pyasn1.compat.octets import int2oct, oct2int, ints2octs, null, str2octs +from pyasn1 import debug, error class Error(Exception): pass @@ -78,9 +78,24 @@ class ExplicitlyTaggedItemEncoder(AbstractItemEncoder): explicitlyTaggedItemEncoder = ExplicitlyTaggedItemEncoder() +class BooleanEncoder(AbstractItemEncoder): + supportIndefLenMode = 0 + _true = ints2octs((1,)) + _false = ints2octs((0,)) + def encodeValue(self, encodeFun, value, defMode, maxChunkSize): + return value and self._true or self._false, 0 + class IntegerEncoder(AbstractItemEncoder): supportIndefLenMode = 0 + supportCompactZero = False def encodeValue(self, encodeFun, value, defMode, maxChunkSize): + if value == 0: # shortcut for zero value + if self.supportCompactZero: + # this seems to be a correct way for encoding zeros + return null, 0 + else: + # this seems to be a widespread way for encoding zeros + return ints2octs((0,)), 0 octets = [] value = int(value) # to save on ops on asn1 type while 1: @@ -149,18 +164,15 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): index = 5 else: if len(oid) < 2: - raise error.PyAsn1Error('Short OID %s' % value) + raise error.PyAsn1Error('Short OID %s' % (value,)) # Build the first twos - index = 0 - subid = oid[index] * 40 - subid = subid + oid[index+1] - if subid < 0 or subid > 0xff: + if oid[0] > 6 or oid[1] > 39 or oid[0] == 6 and oid[1] > 15: raise error.PyAsn1Error( - 'Initial sub-ID overflow %s in OID %s' % (oid[index:], value) + 'Initial sub-ID overflow %s in OID %s' % (oid[:2], value) ) - octets = (subid,) - index = index + 2 + octets = (oid[0] * 40 + oid[1],) + index = 2 # Cycle through subids for subid in oid[index:]: @@ -184,6 +196,7 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): return ints2octs(octets), 0 class RealEncoder(AbstractItemEncoder): + supportIndefLenMode = 0 def encodeValue(self, encodeFun, value, defMode, maxChunkSize): if value.isPlusInfinity(): return int2oct(0x40), 0 @@ -206,9 +219,11 @@ class RealEncoder(AbstractItemEncoder): m >>= 1 e += 1 eo = null - while e: + while e not in (0, -1): eo = int2oct(e&0xff) + eo e >>= 8 + if e == 0 and eo and oct2int(eo[0]) & 0x80: + eo = int2oct(0) + eo n = len(eo) if n > 0xff: raise error.PyAsn1Error('Real exponent overflow') @@ -268,7 +283,7 @@ class AnyEncoder(OctetStringEncoder): tagMap = { eoo.endOfOctets.tagSet: EndOfOctetsEncoder(), - univ.Boolean.tagSet: IntegerEncoder(), + univ.Boolean.tagSet: BooleanEncoder(), univ.Integer.tagSet: IntegerEncoder(), univ.BitString.tagSet: BitStringEncoder(), univ.OctetString.tagSet: OctetStringEncoder(), @@ -313,6 +328,7 @@ class Encoder: self.__typeMap = typeMap def __call__(self, value, defMode=1, maxChunkSize=0): + debug.logger & debug.flagEncoder and debug.logger('encoder called in %sdef mode, chunk size %s for type %s, value:\n%s' % (not defMode and 'in' or '', maxChunkSize, value.__class__.__name__, value.prettyPrint())) tagSet = value.getTagSet() if len(tagSet) > 1: concreteEncoder = explicitlyTaggedItemEncoder @@ -322,13 +338,16 @@ class Encoder: elif tagSet in self.__tagMap: concreteEncoder = self.__tagMap[tagSet] else: - baseTagSet = value.baseTagSet - if baseTagSet in self.__tagMap: - concreteEncoder = self.__tagMap[baseTagSet] + tagSet = value.baseTagSet + if tagSet in self.__tagMap: + concreteEncoder = self.__tagMap[tagSet] else: - raise Error('No encoder for %s' % value) - return concreteEncoder.encode( + raise Error('No encoder for %s' % (value,)) + debug.logger & debug.flagEncoder and debug.logger('using value codec %s chosen by %r' % (concreteEncoder.__class__.__name__, tagSet)) + substrate = concreteEncoder.encode( self, value, defMode, maxChunkSize ) + debug.logger & debug.flagEncoder and debug.logger('built %s octets of substrate: %s\nencoder completed' % (len(substrate), debug.hexdump(substrate))) + return substrate encode = Encoder(tagMap, typeMap) diff --git a/libs/pyasn1/codec/cer/__init__.py b/libs/pyasn1/codec/cer/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/cer/__init__.py +++ b/libs/pyasn1/codec/cer/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/cer/decoder.py b/libs/pyasn1/codec/cer/decoder.py index 71395d22..9fd37c13 100644 --- a/libs/pyasn1/codec/cer/decoder.py +++ b/libs/pyasn1/codec/cer/decoder.py @@ -7,22 +7,25 @@ from pyasn1 import error class BooleanDecoder(decoder.AbstractSimpleDecoder): protoComponent = univ.Boolean(0) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: raise error.PyAsn1Error('Empty substrate') - byte = oct2int(substrate[0]) + byte = oct2int(head[0]) + # CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while + # BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1 + # in http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf if byte == 0xff: value = 1 elif byte == 0x00: value = 0 else: raise error.PyAsn1Error('Boolean CER violation: %s' % byte) - return self._createComponent(asn1Spec, tagSet, value), substrate[1:] + return self._createComponent(asn1Spec, tagSet, value), tail tagMap = decoder.tagMap.copy() tagMap.update({ - univ.Boolean.tagSet: BooleanDecoder(), + univ.Boolean.tagSet: BooleanDecoder() }) typeMap = decoder.typeMap diff --git a/libs/pyasn1/codec/der/__init__.py b/libs/pyasn1/codec/der/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/der/__init__.py +++ b/libs/pyasn1/codec/der/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/der/decoder.py b/libs/pyasn1/codec/der/decoder.py index 0f5a24ca..604abec2 100644 --- a/libs/pyasn1/codec/der/decoder.py +++ b/libs/pyasn1/codec/der/decoder.py @@ -2,4 +2,8 @@ from pyasn1.type import univ from pyasn1.codec.cer import decoder -decode = decoder.Decoder(decoder.tagMap, decoder.typeMap) +tagMap = decoder.tagMap +typeMap = decoder.typeMap +Decoder = decoder.Decoder + +decode = Decoder(tagMap, typeMap) diff --git a/libs/pyasn1/compat/__init__.py b/libs/pyasn1/compat/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/compat/__init__.py +++ b/libs/pyasn1/compat/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/compat/octets.py b/libs/pyasn1/compat/octets.py index d0303eaa..f7f2a29b 100644 --- a/libs/pyasn1/compat/octets.py +++ b/libs/pyasn1/compat/octets.py @@ -8,6 +8,7 @@ if version_info[0] <= 2: octs2ints = lambda s: [ oct2int(x) for x in s ] str2octs = lambda x: x octs2str = lambda x: x + isOctetsType = lambda s: isinstance(s, str) else: ints2octs = bytes int2oct = lambda x: ints2octs((x,)) @@ -16,3 +17,4 @@ else: octs2ints = lambda s: [ x for x in s ] str2octs = lambda x: x.encode() octs2str = lambda x: x.decode() + isOctetsType = lambda s: isinstance(s, bytes) diff --git a/libs/pyasn1/debug.py b/libs/pyasn1/debug.py new file mode 100644 index 00000000..c27cb1d4 --- /dev/null +++ b/libs/pyasn1/debug.py @@ -0,0 +1,65 @@ +import sys +from pyasn1.compat.octets import octs2ints +from pyasn1 import error +from pyasn1 import __version__ + +flagNone = 0x0000 +flagEncoder = 0x0001 +flagDecoder = 0x0002 +flagAll = 0xffff + +flagMap = { + 'encoder': flagEncoder, + 'decoder': flagDecoder, + 'all': flagAll + } + +class Debug: + defaultPrinter = sys.stderr.write + def __init__(self, *flags): + self._flags = flagNone + self._printer = self.defaultPrinter + self('running pyasn1 version %s' % __version__) + for f in flags: + if f not in flagMap: + raise error.PyAsn1Error('bad debug flag %s' % (f,)) + self._flags = self._flags | flagMap[f] + self('debug category \'%s\' enabled' % f) + + def __str__(self): + return 'logger %s, flags %x' % (self._printer, self._flags) + + def __call__(self, msg): + self._printer('DBG: %s\n' % msg) + + def __and__(self, flag): + return self._flags & flag + + def __rand__(self, flag): + return flag & self._flags + +logger = 0 + +def setLogger(l): + global logger + logger = l + +def hexdump(octets): + return ' '.join( + [ '%s%.2X' % (n%16 == 0 and ('\n%.5d: ' % n) or '', x) + for n,x in zip(range(len(octets)), octs2ints(octets)) ] + ) + +class Scope: + def __init__(self): + self._list = [] + + def __str__(self): return '.'.join(self._list) + + def push(self, token): + self._list.append(token) + + def pop(self): + return self._list.pop() + +scope = Scope() diff --git a/libs/pyasn1/type/__init__.py b/libs/pyasn1/type/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/type/__init__.py +++ b/libs/pyasn1/type/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/type/base.py b/libs/pyasn1/type/base.py index db31671e..40873719 100644 --- a/libs/pyasn1/type/base.py +++ b/libs/pyasn1/type/base.py @@ -120,7 +120,12 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): def prettyIn(self, value): return value def prettyOut(self, value): return str(value) - def prettyPrint(self, scope=0): return self.prettyOut(self._value) + def prettyPrint(self, scope=0): + if self._value is noValue: + return '<no value>' + else: + return self.prettyOut(self._value) + # XXX Compatibility stub def prettyPrinter(self, scope=0): return self.prettyPrint(scope) diff --git a/libs/pyasn1/type/namedtype.py b/libs/pyasn1/type/namedtype.py index aa9c5678..48967a5f 100644 --- a/libs/pyasn1/type/namedtype.py +++ b/libs/pyasn1/type/namedtype.py @@ -60,12 +60,12 @@ class NamedTypes: tagMap = self.__namedTypes[idx].getType().getTagMap() for t in tagMap.getPosMap(): if t in self.__tagToPosIdx: - raise error.PyAsn1Error('Duplicate type %s' % t) + raise error.PyAsn1Error('Duplicate type %s' % (t,)) self.__tagToPosIdx[t] = idx try: return self.__tagToPosIdx[tagSet] except KeyError: - raise error.PyAsn1Error('Type %s not found' % tagSet) + raise error.PyAsn1Error('Type %s not found' % (tagSet,)) def getNameByPosition(self, idx): try: @@ -79,12 +79,12 @@ class NamedTypes: idx = idx - 1 n = self.__namedTypes[idx].getName() if n in self.__nameToPosIdx: - raise error.PyAsn1Error('Duplicate name %s' % n) + raise error.PyAsn1Error('Duplicate name %s' % (n,)) self.__nameToPosIdx[n] = idx try: return self.__nameToPosIdx[name] except KeyError: - raise error.PyAsn1Error('Name %s not found' % name) + raise error.PyAsn1Error('Name %s not found' % (name,)) def __buildAmbigiousTagMap(self): ambigiousTypes = () diff --git a/libs/pyasn1/type/namedval.py b/libs/pyasn1/type/namedval.py index 815e2d42..d0fea7cc 100644 --- a/libs/pyasn1/type/namedval.py +++ b/libs/pyasn1/type/namedval.py @@ -15,10 +15,10 @@ class NamedValues: name = namedValue val = automaticVal if name in self.nameToValIdx: - raise error.PyAsn1Error('Duplicate name %s' % name) + raise error.PyAsn1Error('Duplicate name %s' % (name,)) self.nameToValIdx[name] = val if val in self.valToNameIdx: - raise error.PyAsn1Error('Duplicate value %s' % name) + raise error.PyAsn1Error('Duplicate value %s=%s' % (name, val)) self.valToNameIdx[val] = name self.namedValues = self.namedValues + ((name, val),) automaticVal = automaticVal + 1 diff --git a/libs/pyasn1/type/tag.py b/libs/pyasn1/type/tag.py index 0cf67ebd..1144907f 100644 --- a/libs/pyasn1/type/tag.py +++ b/libs/pyasn1/type/tag.py @@ -18,7 +18,7 @@ class Tag: def __init__(self, tagClass, tagFormat, tagId): if tagId < 0: raise error.PyAsn1Error( - 'Negative tag ID (%s) not allowed' % tagId + 'Negative tag ID (%s) not allowed' % (tagId,) ) self.__tag = (tagClass, tagFormat, tagId) self.uniq = (tagClass, tagId) diff --git a/libs/pyasn1/type/tagmap.py b/libs/pyasn1/type/tagmap.py index 53e1791a..7cec3a10 100644 --- a/libs/pyasn1/type/tagmap.py +++ b/libs/pyasn1/type/tagmap.py @@ -28,7 +28,7 @@ class TagMap: def clone(self, parentType, tagMap, uniq=False): if self.__defType is not None and tagMap.getDef() is not None: - raise error.PyAsn1Error('Duplicate default value at %s' % self) + raise error.PyAsn1Error('Duplicate default value at %s' % (self,)) if tagMap.getDef() is not None: defType = tagMap.getDef() else: @@ -37,7 +37,7 @@ class TagMap: posMap = self.__posMap.copy() for k in tagMap.getPosMap(): if uniq and k in posMap: - raise error.PyAsn1Error('Duplicate positive key %s' % k) + raise error.PyAsn1Error('Duplicate positive key %s' % (k,)) posMap[k] = parentType negMap = self.__negMap.copy() diff --git a/libs/pyasn1/type/univ.py b/libs/pyasn1/type/univ.py index cb4f49b7..9cd16f8a 100644 --- a/libs/pyasn1/type/univ.py +++ b/libs/pyasn1/type/univ.py @@ -69,13 +69,18 @@ class Integer(base.AbstractSimpleAsn1Item): def prettyIn(self, value): if not isinstance(value, str): - return int(value) + try: + return int(value) + except: + raise error.PyAsn1Error( + 'Can\'t coerce %s into integer: %s' % (value, sys.exc_info()[1]) + ) r = self.__namedValues.getValue(value) if r is not None: return r try: return int(value) - except ValueError: + except: raise error.PyAsn1Error( 'Can\'t coerce %s into integer: %s' % (value, sys.exc_info()[1]) ) @@ -224,14 +229,14 @@ class BitString(base.AbstractSimpleAsn1Item): return tuple(r) else: raise error.PyAsn1Error( - 'Bad BIT STRING value notation %s' % value + 'Bad BIT STRING value notation %s' % (value,) ) else: for i in value.split(','): j = self.__namedValues.getValue(i) if j is None: raise error.PyAsn1Error( - 'Unknown bit identifier \'%s\'' % i + 'Unknown bit identifier \'%s\'' % (i,) ) if j >= len(r): r.extend([0]*(j-len(r)+1)) @@ -528,7 +533,7 @@ class Real(base.AbstractSimpleAsn1Item): ) if value[1] not in (2, 10): raise error.PyAsn1Error( - 'Prohibited base for Real value: %s' % value[1] + 'Prohibited base for Real value: %s' % (value[1],) ) if value[1] == 10: value = self.__normalizeBase10(value) @@ -648,7 +653,7 @@ class SetOf(base.AbstractConstructedAsn1Item): def _verifyComponent(self, idx, value): if self._componentType is not None and \ not self._componentType.isSuperTypeOf(value): - raise error.PyAsn1Error('Component type error %s' % value) + raise error.PyAsn1Error('Component type error %s' % (value,)) def getComponentByPosition(self, idx): return self._componentValues[idx] def setComponentByPosition(self, idx, value=None, verifyConstraints=True): @@ -924,9 +929,9 @@ class Choice(Set): return self._componentValues[self._currentIdx] >= other return NotImplemented if sys.version_info[0] <= 2: - def __nonzero__(self, other): return bool(self._componentValues) + def __nonzero__(self): return bool(self._componentValues) else: - def __bool__(self, other): return bool(self._componentValues) + def __bool__(self): return bool(self._componentValues) def __len__(self): return self._currentIdx is not None and 1 or 0 diff --git a/libs/pyutil/_version.py b/libs/pyutil/_version.py index 617d2205..376b2b9b 100644 --- a/libs/pyutil/_version.py +++ b/libs/pyutil/_version.py @@ -6,7 +6,7 @@ # pyutil.version_class for a description of what the different fields mean. __pkgname__ = "pyutil" -verstr = "1.9.3" +verstr = "1.9.7" try: from pyutil.version_class import Version as pyutil_Version __version__ = pyutil_Version(verstr) diff --git a/libs/pyutil/benchutil.py b/libs/pyutil/benchutil.py index 3e773a63..6c286346 100644 --- a/libs/pyutil/benchutil.py +++ b/libs/pyutil/benchutil.py @@ -1,4 +1,4 @@ -# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn +# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn # This file is part of pyutil; see README.rst for licensing terms. """ @@ -21,10 +21,10 @@ the second, e.g.: >>> rep_bench(fib, 25, UNITS_PER_SECOND=1000) best: 1.968e+00, 3th-best: 1.987e+00, mean: 2.118e+00, 3th-worst: 2.175e+00, worst: 2.503e+00 (of 10) -The output is reporting the number of milliseconds that executing the function -took, divided by N, from ten different invocations of fib(). It reports the -best, worst, M-th best, M-th worst, and mean, where "M" is the natural log of -the number of invocations (in this case 10). +The output is reporting the number of milliseconds that executing the +function took, divided by N, from ten different invocations of +fib(). It reports the best, worst, M-th best, M-th worst, and mean, +where "M" is 1/4 of the number of invocations (in this case 10). 2. Now run it with different values of N and look for patterns: @@ -74,10 +74,12 @@ and the main function is to make them be methods of the same object, e.g.: 4. Things to fix: - a. I used to have it hooked up to use the "hotshot" profiler on the code being - measured. I recently tried to change it to use the newer cProfile profiler - instead, but I don't understand the interface to cProfiler so it just gives an - exception if you pass profile=True. Please fix this and send me a patch. + a. I used to have it hooked up to use the "hotshot" profiler on the + code being measured. I recently tried to change it to use the newer + cProfile profiler instead, but I don't understand the interface to + cProfiler so it just gives an exception if you pass + profile=True. Please fix this and send me a patch. xxx change it to + statprof b. Wouldn't it be great if this script emitted results in a json format that was understood by a tool to make pretty interactive explorable graphs? The @@ -122,7 +124,7 @@ def mult(a, b): except TypeError: return to_decimal(a) * to_decimal(b) -def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): +def rep_bench(func, n, runtime=1.0, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ Will run the func up to MAXREPS times, but won't start a new run if MAXTIME (wall-clock time) has already elapsed (unless MAXTIME is None). @@ -130,33 +132,43 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p @param quiet Don't print anything--just return the results dict. """ assert isinstance(n, int), (n, type(n)) + global worstemptymeasure + emsta = clock() + do_nothing(2**32) + emstop = clock() + empty = emstop - emsta + if empty > worstemptymeasure: + worstemptymeasure = empty + if (worstemptymeasure*2) >= runtime: + raise BadMeasure("Apparently simply invoking an empty Python function can take as long as %0.10f seconds, and we were running iterations for only about %0.10f seconds. So the measurement of the runtime of the code under benchmark is not reliable. Please pass a higher number for the 'runtime' argument to bench_it().") + startwallclocktime = time.time() - tls = [] # elapsed time in seconds + tls = [] # (elapsed time per iter in seconds, iters) bmes = [] while ((len(tls) < MAXREPS) or (MAXREPS is None)) and ((MAXTIME is None) or ((time.time() - startwallclocktime) < MAXTIME)): if initfunc: initfunc(n) try: - tl = bench_it(func, n, profile=profile, profresults=profresults) + tl, iters = bench_it(func, n, runtime=runtime, profile=profile, profresults=profresults) except BadMeasure, bme: bmes.append(bme) else: - tls.append(tl) + tls.append((tl, iters)) if len(tls) == 0: raise Exception("Couldn't get any measurements within time limits or number-of-attempts limits. Maybe something is wrong with your clock? %s" % (bmes,)) - sumtls = reduce(operator.__add__, tls) + sumtls = sum([tl for (tl, iters) in tls]) mean = sumtls / len(tls) tls.sort() - worst = tls[-1] - best = tls[0] - _assert(best > worstemptymeasure*MARGINOFERROR, "%s(n=%s) took %0.10f seconds, but we cannot measure times much less than about %0.10f seconds. Try a more time-consuming variant (such as higher n)." % (func, n, best, worstemptymeasure*MARGINOFERROR,)) + worst = tls[-1][0] + best = tls[0][0] + m = len(tls)/4 if m > 0: - mthbest = tls[m-1] - mthworst = tls[-m] + mthbest = tls[m-1][0] + mthworst = tls[-m][0] else: - mthbest = tls[0] - mthworst = tls[-1] + mthbest = tls[0][0] + mthworst = tls[-1][0] # The +/-0 index is the best/worst, the +/-1 index is the 2nd-best/worst, # etc, so we use mp1 to name it. @@ -196,26 +208,22 @@ class BadMeasure(Exception): def do_nothing(n): pass -def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"): +def bench_it(func, n, runtime=1.0, profile=False, profresults="pyutil-benchutil.prof"): if profile: - st = clock() - cProfile.run('func(n)', profresults) - sto = clock() + raise NotImplementedException() else: + iters = 0 st = clock() - func(n) + deadline = st + runtime sto = clock() + while sto < deadline: + func(n) + iters += 1 + sto = clock() timeelapsed = sto - st - if timeelapsed <= 0: - raise BadMeasure(timeelapsed) - global worstemptymeasure - emsta = clock() - do_nothing(2**32) - emstop = clock() - empty = emstop - emsta - if empty > worstemptymeasure: - worstemptymeasure = empty - return timeelapsed + if (timeelapsed <= 0) or (iters == 0): + raise BadMeasure((timeelapsed, iters)) + return (timeelapsed / iters, iters) def bench(func, initfunc=None, TOPXP=21, MAXREPS=5, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", outputjson=False, jsonresultsfname="pyutil-benchutil-results.json", UNITS_PER_SECOND=1): BSIZES = [] diff --git a/libs/pyutil/benchutil.py~ b/libs/pyutil/benchutil.py~ index 3ec323eb..a33111e8 100644 --- a/libs/pyutil/benchutil.py~ +++ b/libs/pyutil/benchutil.py~ @@ -1,4 +1,4 @@ -# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn +# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn # This file is part of pyutil; see README.rst for licensing terms. """ @@ -104,6 +104,24 @@ def makeg(func): func() return blah +def to_decimal(x): + """ + See if D(x) returns something. If instead it raises TypeError, x must have been a float, so convert it to Decimal by way of string. (In Python >= 2.7, D(x) does this automatically. + """ + try: + return D(x) + except TypeError: + return D("%0.54f" % (x,)) + +def mult(a, b): + """ + If we get TypeError from * (possibly because one is float and the other is Decimal), then promote them both to Decimal. + """ + try: + return a * b + except TypeError: + return to_decimal(a) * to_decimal(b) + def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ Will run the func up to MAXREPS times, but won't start a new run if MAXTIME @@ -144,12 +162,12 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p # etc, so we use mp1 to name it. mp1 = m+1 res = { - 'worst': (worst*UNITS_PER_SECOND)/n, - 'best': (best*UNITS_PER_SECOND)/n, + 'worst': mult(worst, UNITS_PER_SECOND)/n, + 'best': mult(best, UNITS_PER_SECOND)/n, 'mp1': mp1, - 'mth-best': (mthbest*UNITS_PER_SECOND)/n, - 'mth-worst': (mthworst*UNITS_PER_SECOND)/n, - 'mean': (mean*UNITS_PER_SECOND)/n, + 'mth-best': mult(mthbest, UNITS_PER_SECOND)/n, + 'mth-worst': mult(mthworst, UNITS_PER_SECOND)/n, + 'mean': mult(mean, UNITS_PER_SECOND)/n, 'num': len(tls), } @@ -178,7 +196,10 @@ class BadMeasure(Exception): def do_nothing(n): pass -def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"): +def bench_it(func, n, runtime=0.1, profile=False, profresults="pyutil-benchutil.prof"): + """ + runtime is how many seconds to + """ if profile: st = clock() cProfile.run('func(n)', profresults) diff --git a/libs/pyutil/data/wordlist.txt b/libs/pyutil/data/wordlist.txt new file mode 100644 index 00000000..e1048b99 --- /dev/null +++ b/libs/pyutil/data/wordlist.txt @@ -0,0 +1,7248 @@ +fawn +yellow +four +prefix +payoff +scold +outwit +lore +lord +swivel +deli +pigment +foul +fur +disturb +prize +broiler +wooden +satchel +crotch +fritter +charter +tired +miller +bacon +second +tether +ruthless +thunder +fossil +succumb +cull +specialist +hero +avert +herb +splinter +here +herd +china +dogwood +cult +shriek +chink +pancreas +robin +neurologist +climber +diplomat +golden +gridiron +lengthen +summons +remnant +stern +unit +spoke +exhort +statesmanship +music +bedrock +passport +strike +teaspoon +relay +relax +hurt +meteorologist +glass +hurl +hole +hold +unpack +sweeten +blade +locker +locket +plunger +wand +wane +unjust +household +digit +malign +caution +want +rayon +hog +hoe +travel +copious +cutback +revisit +how +hot +hop +cheetah +diagram +possum +modest +antonym +pigtail +revolt +alias +decoy +wing +squint +wine +feedback +misdemeanor +kickoff +foodstuff +butcher +dreamer +fir +bowlder +fix +fib +fig +fin +undercut +enrich +slate +interrupt +sixteen +silver +scholar +thyme +seamstress +debut +arrow +debug +volcano +burial +whim +concord +knockout +garment +allah +spider +crocus +turnip +yiddish +fortnight +allay +whir +whip +diction +smirk +mason +semiconductor +re +adapt +outburst +knit +scruff +silicon +miaow +thumbtack +shopper +wasp +wash +instruct +rhododendron +tango +master +architect +bitter +listen +wisdom +swish +sulphur +crawl +trek +peril +outlay +coward +tree +shower +pneumonia +sheen +acclaim +entail +girder +runner +spectrum +headland +increment +quay +dozen +kidnap +gripe +hum +greenback +tipi +matriarch +stirrup +object +toil +microsecond +mouth +addict +letter +fluster +drought +thriller +expound +singer +upend +grove +professor +camp +detriment +nineteenth +scream +marvel +bomb +reactor +heckler +ulcer +caper +layout +menu +bust +cougar +bush +bliss +rich +mend +rice +plate +pocket +cushion +fetish +relish +jaguar +boarder +pretzel +patch +hasten +respond +fair +heirloom +radium +radius +result +fail +crouch +clef +best +irk +yogurt +ire +wage +extend +vestment +souvenir +extent +wheelbarrow +carbon +debt +roller +accident +trickster +veer +disdain +cup +logic +genus +rehash +gopher +canyon +bewilder +chrome +onomatopoeia +advert +grapefruit +stadium +jackass +counterattack +life +retrospect +spit +worker +wish +lift +toboggan +chile +child +chili +spin +wildcat +dissect +employ +calcium +delicatessen +locksmith +letdown +player +elicit +eighteen +violin +doorman +specter +hone +toaster +honk +rebellion +split +bid +european +typhoid +boiler +ownership +supper +tuna +tune +furlough +noblewoman +unhook +abound +bellow +beset +plight +brandish +previous +ham +hag +hay +prison +falter +east +hat +quirk +birth +shadow +gangplank +remind +pavement +battlefield +attorney +right +old +creek +crowd +creed +crown +glove +billboard +creep +chorus +okra +bottom +circumvent +inhuman +fox +foe +fog +binder +yoke +slither +recollect +despair +rebut +eightieth +sob +sod +overshadow +honeymoon +overgrow +sop +sow +wrap +fabric +panorama +support +tame +avail +width +hothead +call +overhand +overhang +telegraph +offer +thesaurus +beech +squalid +safeguard +otter +duel +misinform +paprika +vanguard +pest +duet +proud +tournament +proven +exist +quintuplet +dealer +leer +floor +glacier +actor +flood +role +entomologist +sunset +smell +leek +intend +glutton +ointment +asterisk +taurus +intent +cleaver +entrust +windscreen +puss +lowdown +time +push +gown +chain +viaduct +skate +chair +midst +millisecond +ballet +uneven +vex +crater +oversight +jerk +ameba +embark +flora +mourn +knapsack +southpaw +exact +epic +judaism +tear +teas +teat +crustacean +subway +team +skewer +prevent +meadow +gremlin +attic +sigh +milligram +heavyweight +crescent +playpen +crackpot +melt +current +boost +abscond +gnaw +splice +address +brilliant +endow +queue +influx +love +radish +prefer +piranha +fake +instal +forefront +sky +homesick +turret +wicker +wicket +scope +prosecutor +wicked +afford +refrain +visual +appendix +behalf +mascot +lumberjack +pretend +descriptor +dispossess +stole +winter +savor +sputter +meddler +slush +spot +textual +date +suck +dove +pulley +stress +conscious +bluster +wheelchair +quadrant +mango +so +skirmish +truce +drunken +archeologist +footstep +yearn +jig +disconnect +thumb +accordion +nearsighted +councillor +hubbub +suspicion +thump +apron +civilian +insomnia +nation +amulet +twilight +ketchup +handiwork +revert +fisherman +quarter +quartet +receipt +fireproof +breakthrough +sponsor +troll +naked +canvas +onrush +trauma +formula +dumbfound +million +envelop +vicious +disrespect +mime +plea +byte +workmanship +punk +wrong +ostrich +punt +footwear +neglect +gunshot +potter +one +reopen +chide +conifer +vote +paleontologist +languish +boulevard +wrath +convent +bite +extortion +shiver +draft +cite +starfish +shawl +artifact +snatch +antic +boyfriend +iceberg +rival +stammer +counselor +janitor +prospect +sac +greyhound +argument +alley +sad +say +borough +saw +handicraft +tulip +general +knead +zoo +note +take +destroy +printer +buffer +squalor +compress +buffet +crochet +knee +byway +lawn +enamel +blockhead +sale +cocoanut +wind +axe +salt +cobra +homespun +lotus +friction +bright +slot +slow +slop +unkind +gourd +transact +cloak +debunk +slog +hockey +slob +robe +clank +dissimilar +psychiatrist +clang +outlet +prime +artist +saliva +borrow +soloist +carrion +handcuff +primp +landlord +tortilla +where +xmas +vision +gout +gangster +cheesecloth +diver +bugler +mutton +plummet +bootleg +teacup +bureau +mope +vender +jumper +spars +screen +dome +supermarket +adept +jovial +spare +spark +quack +oust +fit +madcap +mane +flipper +backpack +twin +boar +supervisor +extinct +twig +boat +companionship +stretch +west +breath +reflex +gist +thousand +photon +cloudburst +turtleneck +former +jute +scarlet +straighten +spotlight +girth +brow +canon +dubious +monk +blab +fame +spunk +breakdown +hideaway +deft +barber +disown +booster +driftwood +veal +pewter +dimension +scholarship +summer +manifold +poach +disconcert +slime +rest +invalid +alarmist +mandolin +instrument +overthrow +stopwatch +haystack +joyful +sportsmanship +rejoin +dart +dark +brazier +snarl +traffic +cranium +vacuum +world +snare +dare +clan +clam +stranger +shutter +glamor +clay +claw +inter +kennel +clap +auditorium +obstruct +grub +potion +lobster +racial +endeavor +tote +tube +moslem +tuba +nook +exit +refer +zest +ration +leadership +standpoint +stone +ace +slender +meal +tumor +neighbor +act +mean +invert +braggart +homeless +wade +hypnotist +racquet +hew +burglar +her +gleam +glean +mindless +harpsichord +italic +hem +hen +defrost +epilog +pull +regimen +darken +wafer +rage +hooray +tripe +ruse +flirt +reprimand +whiz +torso +pulp +rust +ohm +gong +ad +fright +certain +epaulet +catchup +hoodlum +ay +ax +tranquil +jargon +slobber +cream +yoga +collector +abolish +tight +backgammon +congress +annex +slant +midget +brotherhood +slang +rostrum +neuter +thorn +groom +mask +kilogram +mash +mimic +mast +mass +ringworm +waiver +retch +gingham +influenza +consider +neigh +upkeep +taxicab +tinsel +to +tail +smile +norm +debit +baton +candid +salesperson +cobalt +strand +laud +pedant +sand +adjust +small +mammal +peon +ninetieth +plaid +past +burnish +gossip +canvass +healer +hick +offbeat +clock +section +succinct +method +contrast +full +hash +lobbyist +saleswoman +dramatist +backlash +brutal +prior +hamster +skyrocket +social +action +welder +raze +depart +vie +sherbet +regiment +captor +coercion +entrap +select +casket +enliven +petroleum +maltreat +pearl +sitter +morn +ballad +more +teen +teem +door +tester +signpost +nomad +doom +cunning +fatal +malt +chisel +patriarch +knocker +midstream +mall +learn +grope +male +stewardess +prompt +taunt +gallop +scab +accept +autumn +gallon +scar +rustler +condemn +huge +speedboat +fruition +cling +clink +plant +anoint +blotter +variant +unsound +plane +waver +flutter +pucker +wrench +trellis +patio +pant +instep +trade +paper +pang +brim +mislay +hearsay +buttercup +epoch +coarsen +bypass +motley +sucker +gadget +consign +imperil +skipper +harrow +nugget +fount +found +lantern +status +eyelash +clockwork +scribe +penicillin +lipstick +research +highway +bungler +belief +porcelain +bedlam +cockpit +loafer +suntan +acorn +riser +reproach +prefab +drivel +sicken +bumper +testament +clump +major +purport +limerick +number +feeder +slipper +footprint +florist +glitter +guess +guest +jet +swipe +vocalist +saint +gnash +relationship +tightwad +typhoon +mural +consult +grace +frock +getaway +vocal +video +defect +waft +pedestrian +graffito +caress +blond +gasket +sell +ballerina +ragamuffin +tarnish +spaghetti +self +trowel +poplar +brace +bobbin +kneecap +hypochondriac +blackboard +nasal +twine +raucous +virus +plan +wive +foyer +oyster +unequal +arson +covet +cover +barren +barrel +bulletin +chowder +golf +cruiser +affix +session +freight +impact +condor +writer +peculiar +condom +factor +downpour +dandelion +streamer +resent +actress +compass +banner +tumult +sojourn +caramel +enema +weaver +river +outlaw +prospectus +manger +set +creator +overwhelm +jade +sex +see +sea +contour +analog +project +urchin +fission +crossword +pickup +crosswalk +kneel +candor +mildew +hardship +disallow +incident +dividend +pagan +scatterbrain +lass +last +thou +opal +feminist +amoeba +lash +whole +load +loaf +electrician +pendulum +bell +loam +loan +hollow +scallop +church +psychoanalyst +underlay +napalm +airfield +devil +filth +imbed +proprietor +veneer +firm +sweetheart +champion +fire +infect +upstart +fund +deport +hostess +straight +budget +error +outskirt +real +pound +moth +vow +chasm +vanish +chase +starlight +seek +shorten +wasteland +specimen +commune +snail +teeter +cigar +epithet +alert +opinion +stack +recent +expend +clime +person +sixtieth +crayfish +telegram +aroma +belabor +amp +demerit +sandal +goblet +chest +eager +horseradish +homeland +wrongdoer +input +limp +cordon +format +bureaucrat +quest +cataclysm +blackjack +falcon +abduct +flannel +spine +consensus +crescendo +spring +beckon +palm +pall +sight +curious +sprint +battalion +pale +gruel +benefit +religion +be +odor +agreement +carol +by +scepter +coexist +hatchet +sacrament +ambush +biennial +repair +contributor +next +span +sock +submit +custom +spay +suit +spar +spat +blueprint +perplex +poster +lint +slump +pastor +overbear +link +atom +line +up +slander +foist +hornet +insignia +genial +aerial +nationalist +haunt +char +chap +chat +parsec +breather +phantom +paradox +tuft +uranium +scrape +parakeet +swirl +freighter +tart +tedium +scapegoat +trouser +scrub +gardenia +hackney +lane +land +fighter +algorithm +scotch +age +feud +summit +walker +fresh +crowbar +rescuer +hello +essay +code +partial +serviceman +scratch +broaden +totem +soften +leggin +renown +prim +flashback +young +send +moor +tremor +garret +armament +garden +quadruplet +llama +precinct +wipe +magic +harbor +eve +anxious +race +rack +mishap +crook +croon +odd +ode +victor +index +yock +sauerkraut +apparatus +indian +proffer +bird +inspect +leg +punch +acquit +let +fifteen +vinegar +great +casino +screech +scatter +survey +insulin +grandchild +buss +popcorn +mussel +maker +grower +sire +disobey +causal +zip +archbishop +theme +aeon +eleven +doubt +yardstick +midday +pencil +babe +shipwreck +patrol +rubbish +central +hoard +pout +pour +thin +drill +coffin +cherubim +bent +pawn +process +lock +slim +high +slit +bend +slip +pelvis +martyr +trumpet +weaken +rhubarb +delay +blackhead +luster +stow +halter +singular +await +wristwatch +notebook +tier +marrow +hawk +autograph +tomato +counter +robot +element +writ +allot +allow +alloy +thigh +mute +insight +spatula +comma +mutt +warren +perfect +decay +shudder +garnet +derelict +prosper +python +belch +bat +launder +dock +snake +kiss +bar +cage +wrangler +truth +scorch +subset +bump +static +thirteen +mete +jagged +disco +tenth +wander +matrix +bag +fatherland +venom +czar +oblong +lob +shut +perish +tempo +graze +tempt +shun +embarrass +minstrel +chilli +mainland +spill +length +stickler +scare +scarf +manuscript +scene +cobweb +owner +scent +prank +lop +opossum +sergeant +spaceship +painful +stomach +chagrin +vouch +rotunda +haven +steel +wet +bother +aggressor +psalm +disband +unman +steep +torrent +misunderstand +beggar +viewer +partnership +correspond +tonight +receptionist +fourteenth +mischief +depict +soak +bacterium +bassoon +hammer +adjunct +lilt +soap +soar +calculus +manor +raindrop +cipher +vise +segment +fervent +instil +locust +enlist +soprano +fiasco +brew +fact +bring +brine +bedroom +rough +asylum +trivial +brink +redirect +disillusion +planter +jay +jaw +jar +jam +tape +bourbon +flinch +hope +jackpot +move +familiar +scorn +sinus +wring +antagonist +smash +shaver +summon +stuff +rein +withstand +pronoun +packer +frame +packet +bellhop +airmail +dungeon +wire +mien +partisan +unravel +piston +pistol +email +browbeat +fetich +physicist +courtyard +lawsuit +tantrum +drum +quitter +ramp +drug +doorway +puff +roughen +medallion +revamp +migrant +distil +javelin +indict +chromium +lectern +mailman +gondola +quaver +blatant +feather +ballast +sheepish +crisscross +federalist +mannequin +altruism +banish +laser +runway +bathtub +maul +groin +ripe +lush +site +lust +mockingbird +tenor +passbook +ransom +tattoo +inquest +terrorist +buffoon +outbreak +android +balm +ball +balk +dusk +fiesta +bale +bald +dust +broccoli +mosaic +audit +off +shotgun +polyp +command +diphtheria +audio +maggot +compel +glut +glue +rambler +web +generous +clergyman +wee +wed +arrest +crack +scoundrel +government +chancellor +crux +haul +cedar +desk +password +recurs +placenta +crisp +onion +resin +alkali +stagger +imprison +nymph +sprain +overcast +foray +habitat +thief +daylight +flush +wisecrack +ballot +transport +henchman +disbelief +hoarder +avoid +disk +doer +passion +saucepan +stairway +putt +drift +ornithologist +stage +iris +sister +adverb +peal +ingest +union +artefact +parsley +assess +lung +mere +muck +commission +caviar +watchman +stamina +much +function +funnel +cosmopolitan +frisk +shyness +grate +rectum +count +congresswoman +smooth +monument +problem +baptism +cordial +kowtow +sirloin +retina +inn +replica +ink +anesthesia +furl +sexual +saviour +behold +reckless +chum +monday +repeal +veil +vein +ghost +eon +rule +dynamo +torrid +pension +tryout +abhor +buoy +inning +dote +rapid +mansion +defraud +voter +spew +bludgeon +bike +daze +regal +chill +regalia +whack +ale +compassion +blanket +distort +mania +chauvinist +chapel +whisk +daydream +pinch +scalar +handout +roadblock +unblock +math +triumph +chew +paperback +phoney +speck +heliport +sabbath +horn +chef +aristocrat +panda +stardom +lizard +walkout +toll +crunch +dustpan +pursuit +paraffin +sorceress +hairdo +daughter +envoy +adopt +tankard +smoke +loincloth +bunker +anarchist +envious +sanatorium +infield +spigot +thrust +hindsight +total +bra +plot +plow +plop +sweater +gloss +ploy +insult +plod +knoll +beeswax +solicit +award +yard +tariff +overrun +word +err +crest +work +grovel +tinder +era +elbow +spendthrift +quiver +serpent +flunk +impair +liter +chameleon +sever +moron +disappoint +beach +pizza +fever +lad +ladder +lag +lab +lay +law +arch +cosmonaut +retort +greet +greek +green +south +worst +order +greed +salon +gumption +devote +muffin +misconduct +mayor +sheaf +avocado +valor +carton +shear +then +fragment +safe +break +band +bang +coffer +overprint +tzar +bank +bread +crock +gallows +lisp +iguana +schemer +transient +prawn +sled +flock +slew +hostel +burlap +network +diesel +fellowship +amethyst +marigold +barrier +veto +standard +stencil +lollipop +morass +drench +ticket +maniac +raisin +flawless +renew +sprig +regress +vanquish +kin +render +system +hamstring +chopper +disembark +comic +overs +neck +upshot +tourniquet +kiwi +emblem +luncheon +cereal +rebuff +minibus +guild +target +tavern +hike +medley +iota +guilt +iron +minus +pessimist +lull +ponder +strength +realm +widen +silversmith +latter +hamper +transmit +curfew +maiden +boxcar +sue +negro +phase +proverb +grave +deacon +swamp +bracket +aunt +rickshaw +oppress +mitten +crust +boyhood +nephew +toast +geyser +layman +geologist +predecessor +do +ardor +ecologist +roundabout +slack +rebirth +runt +rune +rung +crucifixion +steak +steal +steam +ghoul +reread +misdirect +christian +goulash +pastel +gentleman +cellist +contraband +drawl +accord +unfold +kitchen +cop +cot +cow +brat +excrement +ill +cob +brag +cod +cog +bran +coo +con +emporium +eyesight +tone +spear +royal +trunk +nonconformist +infirm +speak +charisma +scarecrow +warmth +leech +baud +hacksaw +millionth +hoist +spellbind +gracious +physician +inhibit +gnu +launcher +air +aim +ail +abrupt +thrash +aid +stink +have +sticker +sting +throat +brake +cone +hebrew +uplift +stint +descent +perform +descend +decibel +wheel +raid +fuss +nil +swell +hang +evil +hand +fuse +nip +nit +scenario +drip +ragged +client +mamma +kinship +indigo +photo +victim +extol +thyroid +exalt +shout +cognac +board +zillion +righteous +plasma +intercom +fusion +boxer +cape +retreat +cooler +night +flatter +born +rile +flatten +bore +orchid +cede +humor +peek +peel +pose +confer +peer +peep +chafe +foreskin +chaff +diner +coral +visa +banker +horizon +cherish +gingerbread +octopus +croak +faint +dilemma +tetanus +float +profession +bound +curios +sedan +loin +beet +piggyback +wag +bookend +wad +frill +sovereign +fight +gybe +way +wax +burro +war +fizz +peninsula +holdup +boon +true +reset +absent +nursemaid +smidgeon +maximum +crystal +veterinarian +emir +emit +aorta +flat +abstract +molt +flaw +postscript +subsist +prayer +cacao +face +mold +mole +stake +shrine +test +upholster +unwilling +frolic +shrink +heyday +chairperson +hemophilia +faze +affidavit +loyal +longshoreman +igloo +concept +matron +consul +fulcrum +datum +horseback +supplement +toothpick +varnish +grape +zone +mallet +flask +graph +hump +flash +manicurist +glad +rhythm +tusk +terror +idealist +southwest +brown +congest +kitten +blast +brows +ophthalmologist +gun +gum +gut +guy +diarrhoea +upper +brave +regret +bravo +thinker +cost +helpless +tempest +cargo +appear +economist +menthol +medal +havoc +uniform +tarantula +appeal +caveat +genes +gawk +jester +disclaim +goldfish +teacher +buck +merriment +fogey +precursor +plotter +eke +disavow +trial +convertor +pillow +bolt +extra +paragon +keeper +marker +firearm +market +streetcar +prove +subvert +live +matador +club +cluck +clue +logarithm +prepay +graphic +slogan +car +cap +caw +cat +meow +can +cab +heart +hears +chip +sake +bridesmaid +abort +chin +chic +serum +bankrupt +freezer +write +lobe +storeroom +criterion +entrench +afternoon +product +dive +southern +bawl +motorway +pave +drastic +flourish +crepe +grandson +explicit +offend +barnyard +forfeit +haircut +ledger +brain +nitrogen +cold +braid +ethic +willow +theorem +window +artisan +factual +tiara +gizzard +nought +halt +fling +nod +rake +overcrowd +dishearten +hale +half +recap +courtship +taillight +provision +discuss +halo +wont +concerto +servant +drop +domain +supplant +year +operand +wavelength +happen +album +accomplish +space +thirst +rational +thong +carp +cart +virtuoso +quart +rebel +marina +prospector +card +care +fungus +tomahawk +british +honest +nonprofit +profess +blind +madam +blink +rink +rind +ring +drove +tomorrow +size +sheep +sheer +sheet +silent +bookmark +breed +callous +traction +checker +tragic +heartburn +friend +pomp +courier +that +peck +scalpel +rugged +recruit +magnesium +optimist +extinguish +angel +slay +slat +premier +slap +racetrack +slam +anger +breakfast +recover +slab +upbeat +veteran +shore +snout +siesta +begin +prick +halibut +price +foothold +dream +tooth +aerosol +washcloth +fifth +ground +gnat +snack +ratio +stair +proportion +jolt +stain +juror +shrill +pumpkin +cannon +loath +stroll +leather +thermal +husband +druggist +concert +burst +spore +whitewash +unfit +staunch +sport +incisor +concern +crawfish +glaze +complexion +import +clench +pluck +blame +broil +impromptu +whisker +guffaw +pertain +priestess +temper +aura +comet +evict +adroit +dispatch +exploit +semicolon +lioness +harmless +rebuild +toss +sisterhood +textbook +bloodhound +crumb +these +trick +scum +cherub +fool +marksman +zenith +mucus +soil +agnostic +laggard +bias +eras +bestial +beaver +waterway +petunia +helium +develop +media +pester +poetic +document +sweeper +finish +thunderbolt +foal +foam +cymbal +fruit +volley +trawler +smelt +quartz +theater +framework +patchwork +demean +taxi +livestock +fester +battleship +typhus +neon +touch +speed +death +refurbish +treatment +baloney +momentum +lade +ream +hover +frown +spectacular +larva +read +ruler +swig +leapfrog +earnest +detract +stunt +execution +reap +hovel +rear +postcard +incest +roll +engross +oblivion +output +downward +falsehood +laugh +verbal +landslid +squirm +garland +putter +cleanser +deficit +squirt +wheat +sadden +throb +sixth +tuition +strainer +bazaar +throw +comparison +placard +hiatus +chop +fell +wolf +parson +chow +ruff +assassin +processor +heater +outlook +earring +watchdog +your +restless +stare +grater +log +area +start +stealth +low +lot +wigwam +groan +pitcher +rump +posterior +recoil +omelet +lymph +thaw +corduroy +wholes +hire +fraud +default +bucket +draftsman +cornmeal +gibberish +charlatan +scanner +pickax +sheath +mover +antenna +housework +valid +colt +you +houseboat +poor +polar +poop +peat +pear +peas +overreact +podium +peak +pool +fiscal +assert +moonlight +repay +forswear +mourner +skeleton +breadth +groundwork +angler +month +unrest +bequeath +carpet +corps +gymnast +foster +spearhead +fervor +fountain +washroom +horror +verb +minded +heaven +ceaseless +saboteur +tendril +blackout +smock +homonym +resound +exodus +casual +bomber +protractor +lurch +milk +turmoil +vet +excess +strong +arena +divisor +noticeboard +outgrowth +vegetarian +whine +soldier +amount +base +trainer +put +haemoglobin +seventeenth +taker +helper +pup +titbit +assessor +chestnut +yuck +suction +grill +nine +parasol +transcend +pusher +boycott +archipelago +tract +phrase +magenta +frostbit +sheikh +warhead +spreadsheet +lyre +snippet +reject +gash +circuit +rude +sneak +denial +gasp +reelect +undergo +figment +overnight +ego +dread +egg +lynch +earthworm +help +reservoir +slouch +auburn +reclaim +preempt +soot +helm +hell +clarinet +limelight +prowler +lateral +heron +astonish +forbear +dim +food +musket +terrain +vomit +foot +stopper +holiday +payer +twelfth +bless +radial +trailer +pamper +thirteenth +talisman +event +magnet +vertigo +wedlock +teak +publish +eardrum +sustain +shrivel +outrun +ass +pun +drink +bass +dirt +dung +dune +cornstarch +reason +heroin +ask +ash +turnout +bask +bash +pus +launch +curtsey +round +bridegroom +caption +liaison +heartbeat +blush +assign +arsenal +demagog +elder +effortless +mist +miss +blossom +minion +station +expand +fingernail +scheme +banana +merciless +slosh +behind +bowel +trapper +boredom +sign +leotard +tunic +lament +ouster +hyaena +bride +bobsled +currant +toward +weapon +phobia +chipmunk +yowl +null +sensual +lid +lie +koala +cave +lip +useless +honeycomb +popular +quota +plunder +mace +watt +smoulder +clear +cleat +succor +clean +skein +blend +humid +phenomenon +cowgirl +flicker +sheik +crayon +copyright +paranoid +scooter +less +ikon +ampersand +waterproof +custodian +outlast +strut +strum +basement +chimney +monogram +fluff +chasten +geld +courteous +cramp +backtrack +grey +close +despatch +bandwagon +haddock +aqueduct +wow +grocer +won +woodchuck +wok +woe +stalk +bettor +wreath +philanthropist +spray +distinguish +zipper +garrison +delimit +eggplant +buzz +vault +protector +mausoleum +onward +oversleep +liken +proton +header +badminton +vessel +catwalk +stamp +damp +nape +damn +threaten +dame +alto +liven +exempt +deter +liver +hobnob +furrow +pact +loom +utmost +look +socialist +governor +rope +pace +while +smart +fleet +loot +loop +pack +petal +hoax +grant +belong +makeshift +discredit +grand +conflict +sham +hallway +optic +dime +bonus +banter +overweight +user +boa +grind +auditor +five +ambassador +chore +abstain +bearer +morsel +tick +botch +pier +carat +march +albino +game +jibe +banjo +optician +signal +manifest +eel +sleigh +sketch +creation +undress +yolk +urgent +impoverish +mustang +clothespin +fundamentalist +gild +simmer +slash +slapstick +run +rum +rub +booklet +benefactor +rug +stem +step +stew +taboo +subtract +rut +discus +shine +faith +pigpen +letup +portico +reappear +aye +congressman +block +foreswear +misbehavior +dude +within +gilt +pentagon +connector +syllabus +palomino +harem +frost +reed +womankind +reef +reek +reel +dull +skulk +swagger +chiropractor +ringlet +foresight +similar +psychopath +kidney +straitjacket +nab +sullen +nag +objection +obelisk +nap +department +nay +draw +resign +drag +tundra +drab +formal +horseplay +outing +orbit +depth +bribe +pinion +underbrush +cheerful +go +emboss +dunk +ammonia +compact +aquarium +baron +aria +stave +ameer +shack +geranium +warpath +epitaph +velour +schooner +virtuous +fond +wave +trough +cellular +tenet +nausea +stiff +asparagus +gender +button +hive +verdict +cloister +pilfer +picket +blitz +jump +booth +ardent +languor +cartel +click +poke +wallet +colonel +valet +cell +rotten +experiment +stanch +brooch +fifteenth +quell +weirdo +convert +chant +gent +repel +behead +ricksha +wig +daybreak +danger +win +clout +wit +ligament +infest +gimmick +wiz +cloud +metaphor +snoop +copperhead +crag +crab +cram +expressway +compatriot +mismatch +starter +salad +consort +ride +donut +archer +meet +control +wharf +halloween +glade +skirt +bandana +filament +circular +fare +farm +thunderstorm +canker +foment +corral +scoop +encyclopedia +scoot +agenda +american +cadet +sperm +gunman +hock +brood +broom +brook +walnut +youngster +skater +frond +auto +dike +relentless +snorkel +placid +stout +hands +front +refuel +muff +perfectionist +mode +upward +commonwealth +unwind +chunk +mollusc +seesaw +apartheid +mollusk +special +gallant +armor +confess +fathom +remiss +activist +playground +wick +obsess +umbrella +hopscotch +watermark +jilt +undo +advisor +sneer +princess +shrew +timer +keep +counterpart +elector +keel +mad +blogger +seventeen +bitch +drinker +equinox +dump +wrapper +chintz +attach +attack +jellyfish +final +beard +introvert +punish +feint +noun +plough +piecework +waist +photograph +spurn +cartoonist +beg +bed +bee +discolor +swindler +firework +spurt +bet +are +exhibit +fume +tabu +torment +sundown +portrait +need +border +rotor +bastard +sprinkler +gunner +jaunt +tactic +truck +detector +visor +brand +african +camper +rigor +awe +plumber +eject +spleen +urn +upset +snapshot +businessman +constrain +skunk +affair +indoor +crate +molest +cavort +sheriff +fiberglass +winner +wreak +rash +earner +rasp +gradual +fuel +sulfur +joint +fallout +buyer +endless +gray +tobacco +gust +ordain +topaz +nutmeg +she +contain +recoup +grab +conduct +widow +hardwood +shake +orphan +portend +computer +driveway +equip +portent +unearth +southward +tend +state +lug +tens +antler +tent +bleed +castoff +blinker +keg +bemoan +key +overhaul +thank +sniff +career +admit +spatter +plankton +jersey +christen +tuesday +poem +sari +tread +shaikh +yap +cent +quiz +yam +treat +yak +whisper +poet +fibber +spaniel +nuptial +brunch +debtor +novel +ripen +pandemonium +harden +neuron +hearten +steer +generic +balloon +speaker +northwest +blight +fireman +flesh +absorb +powwow +inbreed +spree +magician +rift +weld +surfboard +lunch +glower +well +drone +welt +underdog +discord +mistaken +dose +distant +laurel +skill +cinder +jackal +dais +ovum +snicker +stratum +possess +warrant +homework +canter +rafter +fate +burden +loss +clown +tablespoons +lose +divest +satirist +rote +page +backlog +shed +glare +twitter +hush +redress +home +peter +competitor +pinpoint +overlay +broad +overlap +hinder +individualist +journal +offset +instinct +smidgen +refuge +freedom +cleans +nightclub +rodeo +dominion +wallop +buzzard +cocoa +pointless +gerbil +snowplow +mastermind +museum +poinsettia +drumstick +mohair +jinx +backhand +cricket +north +gait +admonish +neutral +ho +technician +overflow +ear +eat +he +leper +limit +cello +display +wringer +twist +entreat +contest +meteor +finch +chemist +fodder +star +stay +stag +foil +stab +phosphorus +appoint +sphinx +shunt +broncho +atheist +atheism +portion +pardon +mackerel +demand +unfurl +protest +asian +captain +gamut +swab +swan +sinew +swat +swap +anticlimax +sway +loon +appal +void +vase +smack +govern +affect +hitch +vast +pilgrim +naturalist +vector +washout +whirlwind +quilt +crave +yack +cactus +quill +pander +wreck +orchestra +bikini +spokesman +haze +new +net +maverick +seventh +mew +onus +cardboard +interpret +taper +credit +harass +jamb +permit +prolog +menial +hunch +campaign +bayonet +moral +handlebar +overhear +ore +overheat +calk +overhead +calm +intrust +recommend +type +tell +calf +demon +wart +warp +warn +dogma +warm +pecan +adult +qualm +flotilla +ward +blindfold +confound +rook +room +candlestick +worth +bungalow +headway +root +defer +vodka +give +climax +assent +honey +surveyor +quail +freshen +polio +rib +stockyard +answer +abdomen +plank +coup +fracas +passageway +waterfront +lesbian +guerilla +attempt +third +maintain +capitalist +fetus +deck +keyboard +windshield +furlong +harmonica +crew +better +persist +pass +workout +microfilm +caterpillar +grammar +meat +mistrust +roast +side +bone +luck +caustic +aids +dawn +extract +jell +contend +velvet +gradient +open +crucial +content +reader +linear +whiff +bestow +mistress +needlework +steward +fleck +loud +skinflint +playwright +grade +hoop +hoot +buttock +hook +ditch +hoof +hood +hydrant +acquaint +spinach +historian +enthral +woodwind +brainwash +dwell +inferno +twister +gym +somewhat +gambler +symptom +preacher +affront +keyword +matter +loiter +mink +seep +quench +modern +mind +mine +ginger +seed +seem +churn +mint +unfasten +alibi +desist +chess +sleeper +quarterback +phonograph +chatterbox +regular +condominium +blacklist +don +alarm +impostor +dog +doe +solemn +digress +constrict +consumer +dot +hunger +visitor +probe +syntax +chord +sundial +northeast +explain +jailer +sugar +folder +inventor +edict +patter +smut +stop +coast +pincushion +watermelon +smug +earl +earn +peacock +bay +reload +bad +troop +cower +ban +stinger +linguist +enchant +attest +subject +snuff +scrap +sail +causeway +scram +baboon +warrior +triplet +vitamin +lotion +orthodontist +beautician +cousin +motto +sprinter +pate +typist +height +gusher +aftermath +arsenic +ether +accustom +tint +recur +three +erect +ting +chrysanthemum +trigger +interest +basic +basil +basin +idol +chug +mushroom +suppress +dismiss +deepen +encyclopaedia +unpopular +tank +affirm +tang +near +moratorium +neat +motorist +anchor +spawn +seven +cane +diaphragm +it +shame +jest +in +id +disappear +if +abscess +growl +sap +parish +make +kit +delight +squat +garlic +warden +unicorn +jealous +overt +bequest +kid +butter +romp +smoker +inherit +bedspread +diphthong +left +protocol +just +yen +unfair +psych +human +yes +cretin +yew +legion +character +wretch +save +opt +discreet +background +shoulder +nude +manual +pillar +dean +squander +deal +deaf +maxim +dead +revel +intern +dear +strife +sprawl +pail +collect +normal +councilor +flounder +bullfight +tartar +bold +statistician +burn +blackmail +cottontail +sift +protagonist +burp +burr +tartan +super +innuendo +crucifix +craftsman +commit +marshal +unsay +paunch +chimp +down +lieu +chime +initial +lampoon +editor +fraction +unseat +fork +undergrowth +form +fore +ford +diaper +overburden +analyst +fort +pavilion +whiskey +boomerang +cosmos +propaganda +shin +disciplinarian +classic +covert +sidestep +drive +gland +scrawl +fatherhood +ship +graft +vista +marxist +smidgin +excel +handed +venison +congeal +marxism +sling +faction +handicap +slink +felt +diet +parenthood +journey +reign +stoke +weekend +derail +billion +potato +jacket +gorilla +almanac +teeth +meander +befriend +proletariat +woodwork +skip +skit +invent +adjourn +mild +mile +skim +skin +mill +primer +proletarian +skid +surplus +seasick +misread +depend +swoon +father +countdown +deject +swoop +regatta +unburden +string +yeast +pathologist +merit +join +jettison +stiffen +hoorah +din +stapler +nectar +die +dig +democrat +noiseless +item +dip +blur +shave +thresher +villa +worm +slake +sunup +talker +fillet +suspect +drunkard +shoo +dwarf +dweller +wail +guardian +clerk +makeup +stallion +waif +detest +tangent +deceit +rue +wait +box +boy +cuckoo +shift +bow +dither +boo +raccoon +cyst +bob +nylon +bog +elect +plumb +kayak +surmount +transplant +saki +wealth +perk +visit +vineyard +somersault +sharpen +yoghurt +aspirin +labyrinth +curriculum +downtown +tandem +rigid +savior +effort +gnome +demolish +pageant +moccasin +melodrama +flu +soul +impel +soup +sour +claim +plaza +reflector +predict +agent +drawer +council +craze +pink +purr +arbor +tilt +clever +parch +pine +till +sunday +sword +tile +pathway +pint +map +mar +mat +may +gelatin +membership +mankind +tablecloth +grow +man +relinquish +aimless +hemlock +omen +tale +switch +jail +deposit +talc +unleash +basket +longhand +talk +shield +rabbi +moralist +lyric +pitch +solder +pointer +group +monitor +bedbug +maid +drummer +maim +mail +main +tonic +killer +shatter +minuet +safari +teller +rattler +outweigh +feverish +peasant +careless +rock +hijack +eyelid +latin +bookshop +poker +gavel +unlock +manifesto +girl +stitch +monolog +priest +dutch +blubber +sensor +correct +monster +zombi +vinyl +jaywalk +cough +underwear +waiter +buzzer +thing +registrar +blacksmith +think +frequent +first +lone +crib +long +extrovert +thoughtless +lap +autocrat +escort +daunt +mermaid +anus +yoghourt +memo +broadcast +butt +proofread +tractor +coconut +lick +piccolo +marijuana +dash +comedian +sulk +nazi +sherbert +stopgap +daredevil +acumen +squad +interior +channel +pain +trace +roster +track +acrid +zigzag +whizz +assault +billow +pair +synonym +napkin +typeset +scowl +voodoo +toucan +amir +shop +lexicon +shot +show +cornea +veranda +therapist +shoe +threshold +corner +label +cornet +enthusiast +fend +objector +flapjack +dice +plume +enough +syphon +black +consent +enthusiasm +fiendish +plump +get +straggler +stomp +midriff +slyness +gee +gibber +neckerchief +gem +disinherit +beseech +skull +businesswoman +yield +stupid +nostril +tallow +kernel +sear +eighteenth +seat +seam +seal +stigma +calendar +wonder +puma +parent +limber +ornament +forecast +gage +pump +august +foreword +slingshot +tacit +wednesday +gauntlet +childbirth +tug +tuck +trader +tour +tout +delirium +stretcher +cancer +spank +cancel +tub +mare +underworld +imp +undershirt +mark +mart +workshop +rancher +fiftieth +chalet +graveyard +squash +wake +sound +litterbug +epidermis +slumber +cock +strait +strain +sudden +protein +par +pat +harsh +paw +pay +woodland +same +heartbreak +pad +cotton +pal +pan +exhaust +oil +chloroform +munch +companion +foghorn +polygon +drain +vertebra +soundproof +outdoor +suitor +money +imprint +leeway +aspect +flavor +asthma +godchild +comradeship +forgo +pile +pill +grip +grit +mop +mow +moo +mom +mob +railroad +grim +grin +oxygen +server +chamber +nose +hallelujah +fulfil +sneaker +afflict +witchcraft +ascend +dole +ascent +spasm +gross +confirm +pioneer +inject +gladden +highbrow +linoleum +intravenous +knife +raincoat +broker +squall +bravado +racoon +opium +contagion +roar +island +insect +mixer +thrive +partizan +road +checkup +dagger +coupon +splint +empress +whiten +strip +uptown +skillet +paraphernalia +jigsaw +totalitarian +madden +tycoon +tripod +striker +shroud +hiccup +gore +spice +ember +magnolia +grouch +conqueror +embed +deadlock +affection +deer +deep +fellow +planetarium +deem +file +girlfriend +deed +hound +film +fill +tedious +selfish +personnel +hybrid +repent +drouth +field +prism +astronaut +fruitless +lapel +shelter +gander +unload +burrow +god +gangway +oral +motel +represent +forger +pheasant +forget +founder +suburban +dollar +rebind +zinc +implement +crimson +hideous +premium +parcel +straightforward +scout +scour +fall +bottleneck +pueblo +hinterland +dampen +dictatorship +flyover +neighborhood +clinch +gnarl +burger +zero +cottonwood +lawyer +further +misrepresent +ribbon +dial +skeptic +stool +trinket +stoop +plush +movement +girlhood +malaria +intrench +twang +mule +ranger +beacon +bigamist +capacitor +search +stupor +margin +airport +chipper +chieftain +narrow +fatten +quotient +wizard +caravan +transit +sadist +sadism +establish +dachshund +hobgoblin +eye +score +distinct +two +splash +libel +furor +wiper +diamond +brisk +opportunist +particular +disfavor +nineteen +town +hour +cluster +fast +dew +remain +paragraph +den +abandon +stubborn +shark +buttress +onslaught +share +sphere +minimum +rainstorm +attain +junket +sharp +botanist +siren +awkward +comfort +rapport +stir +bleat +whopper +blacken +blood +bloom +chute +coax +orchard +coat +doctor +spiteful +electron +blunder +mislead +coal +sect +infant +setback +radar +dough +lava +suffer +hundredth +sodium +bosom +late +speech +clamor +lath +lookout +goof +good +goon +detour +frigid +compound +detach +complain +bombard +headroom +countersign +token +monsoon +clamp +harm +hark +mental +hare +hard +beret +banquet +connect +fist +callus +hart +orient +harp +flower +creditor +trooper +pigeon +seaport +granola +print +foreground +assist +cockroach +pleasant +gig +faucet +prophet +omit +wither +pure +corkscrew +copper +perturb +barbarian +shoal +cups +jabber +razor +construct +paint +leash +statement +mama +hummingbird +catapult +pare +park +selector +glycerin +dentist +part +pars +youth +totter +plead +hangout +cistern +blanch +mountain +cardigan +couch +onset +build +zucchini +flute +salmon +chart +most +charm +moss +eskimo +organist +humanitarian +mammoth +pennant +squelch +weigh +standoff +sector +sparrow +fine +find +giant +merger +nervous +ruin +fiend +boulder +prowess +paperweight +cholera +express +ferret +cheapen +batter +breast +theft +silk +pellet +restart +silo +huff +common +archaeologist +printout +vine +lion +overeat +tender +expert +burner +myriad +stowaway +subscript +hypochondria +premiss +egotist +complement +figurehead +mailbox +pagoda +aircraft +sultan +archway +annual +foreign +point +smother +newborn +pamphlet +dancer +esophagus +platinum +pocketbook +secret +amnesia +reformat +finalist +ram +gay +gas +gap +holler +gal +understand +gag +chatter +gab +bile +politician +metro +solid +bill +holocaust +crutch +fun +lingo +manner +mystic +astound +rancor +eczema +ingrain +anaesthesia +sociologist +dishonor +ewe +seminar +corridor +neutron +itch +leopard +yesterday +moment +stripe +unveil +timid +task +werewolf +withdraw +landmark +grid +recant +spend +howl +darn +shape +snot +timber +rundown +impetus +cut +cur +pollster +snag +forbid +cue +punter +cub +snap +bridal +easter +brainstorm +bin +squawk +rebound +bib +judgement +redeem +bit +knock +disrepair +blemish +flue +fagot +flux +bamboo +foolish +walrus +sequin +transgress +often +back +impeach +extremist +mirror +lightning +scald +scale +culprit +pet +pelt +pew +pep +pen +scalp +lard +lark +peg +pea +patient +fed +megaton +constraint +oatmeal +drama +catnip +pediatrician +offshoot +obstetrician +gambit +maelstrom +tiff +clack +lesson +jockey +few +doll +errand +camera +handbook +forward +nougat +sideshow +showman +switchboard +calico +lifeguard +planet +jumbo +azalea +constant +flow +possessor +lye +curd +cure +curb +curl +prevail +stagecoach +leaflet +crypt +underweight +cellar +lend +tablespoon +papa +lens +cater +desert +statesman +mantel +notion +uterus +anguish +caribou +stroller +seaman +golfer +strew +parrakeet +peanut +welter +mower +rudder +compost +blaze +atlas +gravel +queen +dessert +rhyme +claustrophobia +surgeon +molar +verandah +knight +shock +crow +queer +crop +append +power +junior +anthem +access +clipboard +bachelor +intercept +sink +sing +roof +bode +implicit +remark +talent +conceit +resurrect +weekday +climb +honor +blizzard +liqueur +talon +oval +scandal +gateway +sermon +lime +patron +asteroid +butler +charcoal +trait +kiosk +thatch +trail +train +armadillo +harvest +fan +account +tunnel +carrot +obvious +smear +parkway +unread +fetch +employe +truism +sanitarium +teamster +boney +spruce +serial +contempt +hangar +lamb +lame +lamp +forest +goner +stock +roam +leukemia +bluff +terrier +fray +drape +bind +guru +liner +linen +chief +poacher +furious +furnish +disarm +meter +bunch +marshmallow +decorum +labor +kindergarten +heroism +willing +marsh +dad +junction +dab +dam +spell +swordfish +mention +courtroom +sonata +day +strive +flail +snowdrift +thrill +slacken +cider +memoir +sawdust +disregard +flair +thwart +jailor +jugular +pivot +cupboard +lentil +salesman +hippopotamus +matt +defend +rev +repress +stub +mate +barley +stud +smog +stun +red +franc +frank +hanker +fourteen +salami +likelihood +afterward +squadron +indent +mortar +skateboard +yarn +mortal +workbook +retain +retail +waitress +suffix +overshoot +ethnic +sack +brute +whoop +puppet +guidebook +vandal +pauper +ancient +monkey +bologna +laps +vulgar +vagina +hexagon +scant +liquor +cabin +sixteenth +gear +eavesdrop +bulldog +smolder +forethought +springboard +nun +bodyguard +prune +shrapnel +shampoo +linchpin +lover +anthropologist +tide +cavern +pedlar +countryman +waken +optimum +mix +parka +spinster +meek +dryness +hazel +eight +clamber +handbag +hoodwink +transcript +payment +gullet +gulley +gather +request +absurd +rendezvous +occasion +thicken +recess +kite +text +hamlet +traitor +industrialist +sidetrack +portfolio +floodlight +thicket +staff +communism +scorpion +madman +prolong +resubmit +satan +oppressor +communist +inferior +equilibrium +gumdrop +starch +beat +rush +bear +beam +bean +october +beak +bead +organ +ashtray +nutriment +eyebrow +motherhood +mascara +conform +showdown +infidel +racket +interview +reform +pattern +nebula +brunt +hammock +progress +tailspin +sorrow +stratagem +deliver +blackbird +boloney +exclaim +instant +joke +equal +kosher +swim +swallow +highland +guerrilla +glorious +wear +comment +vent +denim +overcoat +commend +vend +harpoon +manhood +citizenship +copier +newscast +gaze +teamwork +gulch +curtain +curtail +hyacinth +juggler +censor +goddess +bulk +bull +bulb +skew +carburetor +cypher +plain +homey +bray +kinfolk +bicker +dissent +squid +blimp +creak +prose +partner +inspector +lynchpin +portray +whirl +grinder +matchbook +defiant +anorak +tumbler +infer +whirr +tighten +pockmark +sauna +ion +grandstand +sunburn +judgment +retard +center +builder +pickpocket +thought +starboard +usual +coaster +humdrum +fingerprint +storey +clinic +interim +surpass +tough +earshot +flashlight +tong +flee +lupin +lake +bench +add +citizen +ado +crossroad +ravel +match +raven +cantaloup +punctual +newsstand +dryer +insert +flamingo +like +success +sofa +journalist +heed +arraign +chick +soft +heel +outfield +propel +fuze +hail +hair +convey +proper +paddock +novelist +shrug +shrub +slide +tureen +regain +pepper +hose +slight +host +panel +beaker +actual +socket +flake +preen +toadstool +pickaback +discard +tomb +tome +snitch +chronic +guard +esteem +custard +underpass +glimmer +gene +maze +globe +buy +bus +coke +sequel +but +bun +bum +bug +bud +embargo +woodsman +wise +ecosystem +debrief +flip +wisp +wist +trapezoid +condiment +plutonium +pin +garter +domino +circus +pie +pig +pit +campus +gush +oaf +cashew +oak +detail +virtual +detain +sewer +oar +redden +dresser +wallow +nutrient +godsend +yelp +baker +jab +hiker +pupil +yell +cookbook +vermin +sleek +sleet +sleep +liar +hate +trolley +sallow +tweet +glider +under +tweed +pride +merchant +lure +risk +rise +lurk +jack +confetti +anemia +school +parrot +enjoy +overdo +cracker +almond +direct +nail +street +monorail +ransack +blue +hide +worsen +poison +beater +supplier +dashboard +wink +even +pontoon +studio +path +crossbow +enrol +connoisseur +forum +ravish +auction +settler +mentor +midway +blowtorch +stray +straw +strap +cassino +would +phlegm +bayou +asset +spike +preview +musk +mush +saber +muse +grief +phone +muss +pouch +must +shoot +hutch +ma +ms +mr +machinist +fortress +quarrel +loosen +joyous +hemoglobin +dolphin +mayhem +attract +end +trill +keen +bunk +vagrant +rhinoceros +shred +toxin +gate +ancestor +dialect +moisten +kilowatt +mess +lump +mesh +sparkler +parallel +stronghold +splendid +spout +patent +enter +vapor +hedgehog +fetter +deform +clapper +sprout +over +bleach +mallard +oven +caster +digest +forehead +theologian +womanhood +comprehend +tramp +drawback +fade +croquet +tourist +plaster +roost +knighthood +monarch +rental +gloom +chuck +choir +prohibit +hanger +unscrew +gymnasium +poncho +truant +saturday +depress +goo +lair +dinosaur +nitwit +psychic +tonsil +gob +emphysema +nite +washer +resistor +carcass +rail +free +rain +acrobat +fret +harpist +ritual +filter +hopeless +soda +rang +accent +puck +rank +restrict +rant +sober +toy +their +sarcasm +top +tow +tot +fiction +ton +duress +toe +urban +murder +overdraw +tool +hearth +embellish +solicitor +toot +incur +western +nonpartisan +lather +prong +flame +mirth +countess +rag +donkey +fashion +handkerchief +ruckus +taint +raw +rat +rap +protract +spade +ray +snow +thorough +contact +hatch +snob +cleft +extravert +quicken +rider +evangelist +shallow +milkman +coil +coin +glow +interject +flop +metal +freeway +policewoman +flog +yank +chariot +bait +endear +saga +alight +random +sage +dupe +radio +rector +earth +bail +shellfish +spite +stanza +disgust +axiom +waltz +gees +watch +fluid +ultimatum +report +reconstruct +noon +spokesperson +egoism +public +erupt +pacifist +pummel +habit +wrest +nut +resist +corrupt +hourglass +mull +mud +mug +finger +mum +approach +wean +weak +contort +boss +toothbrush +southeast +larynx +devour +devout +censorship +newt +protect +irregular +fault +papyrus +facet +elf +smuggler +trust +bingo +bathroom +beef +legend +beer +spread +communion +loft +bladder +uncommon +craft +spearmint +catch +snipe +teapot +misfit +lessen +thousandth +referendum +pyramid +handrail +broth +lollypop +exterior +suggest +wound +overstep +utilitarian +complex +papaya +screw +pick +deflect +suburb +portal +postmark +tassel +ocean +mother +bough +bugger +rodent +shorthand +enlighten +elk +elm +moonbeam +flutist +kelp +misprint +teetotal +upturn +ramrod +dismount +quicksand +spanner +authorship +roach +befit +rumor +apart +ditto +gift +zeal +contradict +hunt +dishonest +zoom +mongrel +hunk +mosquito +hunchback +sanction +excerpt +curio +accost +usher +indirect +intellect +doorstep +nobleman +cooper +combat +letterhead +ice +rhino +newsprint +skylight +convict +christmas +splendor +cord +core +khaki +brawl +corn +brawn +cork +discount +shuck +plug +census +cowboy +plum +choke +surround +caulk +dinner +plus +alga +duke +abet +civic +civil +bath +engulf +cafeteria +art +scamper +transform +sunlight +forbad +virgin +gin +head +medium +amateur +heal +stereo +heat +hear +heap +raft +counsel +muster +bargain +bide +latch +adorn +trim +trio +forearm +cobbler +trip +no +tit +when +junta +tin +whet +tie +implant +depot +pseudonym +evergreen +cleric +toad +geneticist +bullet +navel +yacht +withhold +fasten +backward +coach +impression +rob +rod +focus +livelihood +snip +yokel +rot +discern +environment +aplomb +melon +prop +coop +impend +plantain +cook +cool +looney +level +brother +encroach +quick +lever +pork +drier +trend +bullfrog +pore +inland +voucher +takeoff +bake +port +colic +hymn +choral +postman +spire +theist +thresh +tormentor +humorist +water +fluke +entertain +witch +twentieth +tire +boast +catnap +blotch +cinnamon +prude +weird +tweak +brighten +touchdown +post +panacea +concoct +scan +handler +prey +today +chapter +conductor +altar +cashier +drown +dismal +inhabit +judo +conceal +flagship +hullabaloo +fauna +laughter +streak +overpass +sandbag +trump +stream +despot +stroke +cube +hydrogen +bigot +dress +vital +fourth +dope +ballroom +fascist +clone +scoff +fascism +birthday +apprenticeship +eighth +repeat +classroom +twitch +liquid +inform +reaper +lagoon +superscript +refund +rye +midnight +blare +worship +thermostat +apex +platform +farmer +meridian +cutter +underneath +conquer +fern +rescind +wagon +term +name +realist +opera +bunion +bullion +realism +ailment +torch +zebra +distributor +hysteria +hacker +concur +profit +middleman +gram +clover +hull +hulk +flyer +tuner +flare +highjack +motion +turn +place +swine +swing +turf +preach +childhood +origin +pelican +feign +suspend +insist +scollop +bobcat +array +peddler +given +afterthought +district +opus +trillion +plastic +assort +white +hue +hug +hub +cope +season +hut +enigma +naught +grunt +holder +wide +bewitch +spokeswoman +oath +powder +rend +froth +pro +ani +ant +rent +dragon +stolid +marathon +ideal +blunt +surf +sure +aspen +equestrian +tornado +freshman +librarian +bluebird +icon +latex +tendon +annul +seafood +later +koran +readjust +senior +slope +perch +convoy +cheat +cheap +trespass +hack +broach +hustler +trot +woolen +gulf +genius +gull +shimmer +crime +gulp +woof +wood +deign +wool +entrant +viewpoint +lighten +jazz +festoon +tailor +lighter +dye +homestead +reveal +aluminum +workman +joker +dumfound +bison +picnic +pane +vizor +prowl +optometrist +detect +crooked +review +spoons +hiss +smokestack +caucus +fearless +guitar +coma +comb +come +zodiac +isthmus +reaction +superstar +region +quiet +contract +railway +penal +adjoin +color +armchair +pot +period +pop +pole +colon +polo +pod +poll +runaway +turkey +hobo +schoolboy +tiger +padlock +hernia +careful +spirit +robber +pilot +case +shaft +amend +mount +cash +cask +cast +mound +ventriloquist +vest +exult +clutter +helmet +projector +author +alphabet +fender +bowl +check +macaroni +catfish +bellboy +hermit +week +sang +applaud +nest +driver +weed +director +petticoat +lute +puke +vowel +muffler +weep +cartoon +ranch +relief +model +reward +sinner +clod +clog +tip +kilt +ninth +clot +lavish +violent +kill +kiln +kilo +polish +satin +blow +blot +hint +rose +regent +except +blog +bloc +blob +hind +scrapbook +disrupt +impound +kingdom +blowout +sandman +mugger +towel +bracelet +snort +friar +tower +node +deduct +wombat +interlock +canteen +slice +mood +slick +legal +moon +moos +teardrop +moot +heir +porter +metropolis +quit +unmask +slaughter +quip +ok +oh +of +jeer +shrimp +pistachio +karat +stand +ox +doze +accredit +amber +tribe +vicar +polka +garb +spinal +forewarn +feudal +whaler +there +racism +strict +racist +valley +fish +gibe +relic +jug +regard +cabinet +castaway +strenuous +jut +terminus +feeler +grasp +grass +toilet +ruffian +cinema +frighten +lichen +encompass +bishop +incorrect +abyss +fiddler +heather +idiot +diarrhea +rubber +idiom +heathen +trash +stalwart +championship +symbol +cove +nucleus +serious +brass +wife +invest +derrick +treason +apparel +platter +all +lace +duplex +lack +spacecraft +disc +dish +follow +settlement +titter +wanton +thursday +program +neglig +woman +song +fat +roe +psychologist +retract +ultraviolet +awful +dapper +fad +induct +stimulus +list +trench +align +flick +ten +tea +tee +rate +design +chalk +what +sub +sun +sum +whimper +rascal +brief +overload +crush +version +pulpit +intersect +row +womb +lacquer +pumpernickel +backer +goodby +thrift +misinterpret +heifer +jogger +cataract +haemophilia +murmur +snug +snub +herring +proceed +tarpaulin +wield +hurray +rustic +quash +inlay +garnish +hurrah +minor +ladybug +wretched +flap +mire +protestor +stutter +flit +flag +stick +mellow +chaplain +berth +wrestler +plagiarist +searchlight +sunscreen +pond +court +goal +goad +goat +sandwich +okay +algebra +headrest +embalm +reflect +catalog +numb +short +ricochet +tsar +shade +waylay +mission +scientist +flaunt +reconnect +pretext +stride +islam +thirtieth +style +glide +pray +inward +wilder +abbey +mattress +resort +airstrip +bout +soccer +might +alter +return +hunter +underground +abacus +mathematician +liniment +policeman +refresh +tactless +friendship +weight +needless +duchess +falsetto +expect +inflict +wager +alcohol +disquiet +hilt +dugout +loll +health +hill +shipment +fiber +solvent +friday +differ +effect +disinfect +octagon +physic +teach +sidewalk +jew +blister +thread +threat +bushel +feed +dine +feel +sailor +revolution +least +blank +cigaret +idea +moan +script +gourmet +interact +grime +stork +swarm +storm +moat +syrup +store +mainstay +imperfect +option +hotel +fidget +king +kind +vial +kink +stall +cuff +foreleg +stale +restful +amass +cleaner +exert +strengthen +shrewd +bookworm +gale +gala +gall +remodel +smallpox +toughen +bacteria +chairman +donor +pianist +buff +gill +foreman +rapist +reckon +english +reach +react +nothing +quorum +hyena +amphibian +saloon +notch +scaffold +asphalt +memorandum +felon +font +anvil +firewood +betray +hip +shepherd +hit +deaden +reprint +him +adulthood +snowstorm +forego +stump +martyrdom +arc +bare +bard +bark +ark +arm +barn +blurt +parchment +various +plywood +nincompoop +solo +muslim +sole +outfit +succeed +inertia +orangutang +blazer +bandit +context +bond +cynic +sloth +flier +distress +chaperon +sweet +wastebasket +sweep +weasel +rave +shaykh +bolster +dub +overlook +dud +due +buttermilk +pa +watchword +brick +pi +flight +quintet +dropout +marihuana +cinch +temperament +instructor +heighten +toga +shove +batch +pitchfork +kick +behavior +incognito +lodger +bluegrass +sirup +rip +shamrock +rim +frantic +rig +rid +reprogram +chauffeur +shirt +kimono +viola +shirk +sliver +straightjacket +restraint +painless +throwback +cement +birch +robust +knack +lower +earmark +cheek +cheep +cheer +pollen +facial +vigilant +cabaret +continent +tablet +contractor +plateau +tuxedo +complaint +vendor +foreshadow +awaken +confront +uproar +distrust +breeder +hallmark +play +global +litter +wonderland +butterscotch +saucer +prow +seller +prom +prod +sag +perpendicular +tinker +raider +vivid +cautious +undertow +yawn +ordeal +militia +dialog +tomboy +conquest +momma +piteous +holster +vagabond +stench +canal +pundit +question +swill +parsnip +etch +filet +potassium +glamour +cloth +crank +usurp +delta +upright +crane +outpost +penguin +patriot +consist +apricot +caller +peppermint +husk +cartwheel +highlight +dill +freak +dismay +sublet +sagebrush +rainbow +lemon +riot +peach +grouper +saffron +nick +parlor +ferment +bandstand +mock +nice +mustard +chirp +meaning +vigil +vice +ocular +remit +pyre +buffalo +scroll +pervert +lean +alien +dispel +gang +theorist +gold +uphold +floss +breach +sniper +materialist +toenail +spool +spoon +spook +spoof +harlot +outdo +pleas +pleat +trawl +procession +fold +reunion +acid +folk +sandstorm +outsmart +acronym +relent +kangaroo +gloat +miser +cyclist +barb +survivor +guarantor +orangutan +armpit +shovel +duct +ensign +apt +volt +motor +duck +thick +redo +ape +use +fee +fen +frog +germ +modicum +fez +sort +parliament +porch +musician +impress +sore +rabbit +recount +penis +sculptor +annoy +topic +augment +critic +lumber +executor +proof +bittersweet +tap +tar +tax +villain +tag +condescend +tab +spa +silt +tan +rape +counterfeit +sir +sip +scuff +sit +tamper +six +outclass +occur +sic +carrier +goldsmith +toddler +panic +sin +defeat +tension +lesion +attend +tact +hazard +discomfort +tack +wrist +taco +footpath +aftereffect +light +arduous +schoolchild +sailboat +stamped +minnow +damsel +accompanist +hemp +tyrant +badger +glen +superior +inlet +sill +glee +nostalgia +flank +restrain +glisten +turban +redhead +bye +flex +crash +citrus +flour +flout +emerald +flea +republican +investor +successor +easel +footstool +profound +edit +feast +fuzz +trap +blacktop +cocoon +tray +lilac +mincemeat +interplay +our +proclaim +out +semen +tabloid +cocktail +sentiment +frontier +vehement +disarray +clatter +impart +plural +proviso +planner +utensil +tenement +pendant +gospel +tenant +greenhorn +tanker +zoologist +rivet +uproot +embryo +sew +bouquet +echo +bonnet +eleventh +synagog +salient +droop +unknown +galley +snore +anaemia +drool +boil +tidbit +shell +shelf +transistor +woo +diminish +persecutor +goblin +institution +kickback +frugal +brazen +yodel +laughingstock +clip +fowl +splatter +flunkey +blip +footwork +outstrip +disjoint +pallor +catholic +clove +rout +outward +bagel +lope +divert +trivia +pharmacist +divers +clash +petrol +siphon +filch +fortieth +class +clasp +fang +dens +dent +pipe +vernacular +gain +son +stove +sonnet +utter +chicken +feat +winch +dandruff +rioter +herald +piano +local +counsellor +vigor +sued +skimp +plaintiff +spud +watercolor +barter +bronco +spur +rite +ghetto +bisect +compliment +ascertain +sediment +view +unison +workbench +ebb +expel +hymnal +distract +violet +still +closet +superb +favor +viper +crude +torpedo +avow +jot +exam +amen +joy +foetus +job +spoil +jog +swift +memento +lifeboat +april +grain +commando +wall +hyphen +walk +respect +unclean +decent +trademark +tutor +reindeer +mike +nickel +cypress +penmanship +dearth +overturn +present +kerchief +corset +wilt +vanilla +priesthood +will +fingertip +wild +whirlpool +layer +mutant +motif +apprehend +rooster +lightweight +thug +thud +whore +headlight +cross +member +pediatrist +inch +grandeur +slave +diploma +outcast +beast +student +pedal +whale +collar +gutter +masochist +overwork +scissor +twirl +flint +outgrow +bandanna +rocker +cameo +rocket +camel +boot +wren +obtain +replenish +biologist +daub +distend +smite +now +panther +drunk +smith +hall +book +ski +enact +knob +sick +myth +know +knot +press +redesign +doughnut +loser +cutlet +vortex +clutch +exceed +setter +flagrant +birthmark +demeanor +growth +export +leaf +lead +leak +miner +leap +belt +leader +trout +obey +slur +mitt +slut +slum +pasta +mite +slug +throne +pike +throng +rare +linger +column +biscuit +fear +swear +sweat +udder +emperor +owl +outset +own +owe +weather +champ +brush +billfold +gape +rowboat +van +platoon +transfer +spiral +grotto +cliff +vat +nourish +catsup +unwrap +saunter +mutter +brassier +assail +tomcat +daffodil +nightgown +record +cake +faggot +maroon +boardwalk +abbot +counteract +limb +squirrel +mutual +glint +boor +percent +other +boom +branch +cutthroat +junk +mulch +june +squeak +squeal +extort +jewel +gynecologist +vane +sash diff --git a/libs/pyutil/fileutil.py~ b/libs/pyutil/fileutil.py~ deleted file mode 100644 index e37eb792..00000000 --- a/libs/pyutil/fileutil.py~ +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn -# This file is part of pyutil; see README.rst for licensing terms. - -""" -Futz with files like a pro. -""" - -import errno, exceptions, os, stat, tempfile - -try: - import bsddb -except ImportError: - DBNoSuchFileError = None -else: - DBNoSuchFileError = bsddb.db.DBNoSuchFileError - -# read_file() and write_file() copied from Mark Seaborn's blog post. Please -# read it for complete rationale: -# http://lackingrhoticity.blogspot.com/2009/12/readfile-and-writefile-in-python.html - -def read_file(filename, mode='rb'): - """ Read the contents of the file named filename and return it in - a string. This function closes the file handle before it returns - (even if the underlying Python implementation's garbage collector - doesn't). """ - fh = open(filename, mode) - try: - return fh.read() - finally: - fh.close() - -def write_file(filename, data, mode='wb'): - """ Write the string data into a file named filename. This - function closes the file handle (ensuring that the written data is - flushed from the perspective of the Python implementation) before - it returns (even if the underlying Python implementation's garbage - collector doesn't).""" - fh = open(filename, mode) - try: - fh.write(data) - finally: - fh.close() - -# For backwards-compatibility in case someone is using these names. We used to -# have a superkludge in fileutil.py under these names. -def rename(src, dst, tries=4, basedelay=0.1): - return os.rename(src, dst) - -def remove(f, tries=4, basedelay=0.1): - return os.remove(f) - -def rmdir(f, tries=4, basedelay=0.1): - return os.rmdir(f) - -class _Dir(object): - """ - Hold a set of files and subdirs and clean them all up when asked to. - """ - def __init__(self, name, cleanup=True): - self.name = name - self.cleanup = cleanup - self.files = [] - self.subdirs = set() - - def file(self, fname, mode=None): - """ - Create a file in the tempdir and remember it so as to close() it - before attempting to cleanup the temp dir. - - @rtype: file - """ - ffn = os.path.join(self.name, fname) - if mode is not None: - fo = open(ffn, mode) - else: - fo = open(ffn) - self.register_file(fo) - return fo - - def subdir(self, dirname): - """ - Create a subdirectory in the tempdir and remember it so as to call - shutdown() on it before attempting to clean up. - - @rtype: _Dir instance - """ - ffn = os.path.join(self.name, dirname) - sd = _Dir(ffn, self.cleanup) - self.register_subdir(sd) - make_dirs(sd.name) - return sd - - def register_file(self, fileobj): - """ - Remember the file object and call close() on it before attempting to - clean up. - """ - self.files.append(fileobj) - - def register_subdir(self, dirobj): - """ - Remember the _Dir object and call shutdown() on it before attempting - to clean up. - """ - self.subdirs.add(dirobj) - - def shutdown(self): - if self.cleanup: - for subdir in hasattr(self, 'subdirs') and self.subdirs or []: - subdir.shutdown() - for fileobj in hasattr(self, 'files') and self.files or []: - if DBNoSuchFileError is None: - fileobj.close() # "close()" is idempotent so we don't need to catch exceptions here - else: - try: - fileobj.close() - except DBNoSuchFileError: - # Ah, except that the bsddb module's file-like object (a DB object) has a non-idempotent close... - pass - - if hasattr(self, 'name'): - rm_dir(self.name) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - try: - self.shutdown() - except: - import traceback - traceback.print_exc() - -class NamedTemporaryDirectory(_Dir): - """ - Call tempfile.mkdtemp(), store the name of the dir in self.name, and - rm_dir() when it gets garbage collected or "shutdown()". - - Also keep track of file objects for files within the tempdir and call - close() on them before rm_dir(). This is a convenient way to open temp - files within the directory, and it is very helpful on Windows because you - can't delete a directory which contains a file which is currently open. - """ - - def __init__(self, cleanup=True, *args, **kwargs): - """ If cleanup, then the directory will be rmrf'ed when the object is shutdown. """ - name = tempfile.mkdtemp(*args, **kwargs) - _Dir.__init__(self, name, cleanup) - -class ReopenableNamedTemporaryFile: - """ - This uses tempfile.mkstemp() to generate a secure temp file. It then closes - the file, leaving a zero-length file as a placeholder. You can get the - filename with ReopenableNamedTemporaryFile.name. When the - ReopenableNamedTemporaryFile instance is garbage collected or its shutdown() - method is called, it deletes the file. - """ - def __init__(self, *args, **kwargs): - fd, self.name = tempfile.mkstemp(*args, **kwargs) - os.close(fd) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - self.shutdown() - - def shutdown(self): - remove(self.name) - -def make_dirs(dirname, mode=0777): - """ - An idempotent version of os.makedirs(). If the dir already exists, do - nothing and return without raising an exception. If this call creates the - dir, return without raising an exception. If there is an error that - prevents creation or if the directory gets deleted after make_dirs() creates - it and before make_dirs() checks that it exists, raise an exception. - """ - tx = None - try: - os.makedirs(dirname, mode) - except OSError, x: - tx = x - - if not os.path.isdir(dirname): - if tx: - raise tx - raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning... - -def rmtree(dirname): - """ - A threadsafe and idempotent version of shutil.rmtree(). If the dir is - already gone, do nothing and return without raising an exception. If this - call removes the dir, return without raising an exception. If there is an - error that prevents deletion or if the directory gets created again after - rm_dir() deletes it and before rm_dir() checks that it is gone, raise an - exception. - """ - excs = [] - try: - os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - for f in os.listdir(dirname): - fullname = os.path.join(dirname, f) - if os.path.isdir(fullname): - rm_dir(fullname) - else: - remove(fullname) - os.rmdir(dirname) - except EnvironmentError, le: - # Ignore "No such file or directory", collect any other exception. - if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): - excs.append(le) - except Exception, le: - excs.append(le) - - # Okay, now we've recursively removed everything, ignoring any "No - # such file or directory" errors, and collecting any other errors. - - if os.path.exists(dirname): - if len(excs) == 1: - raise excs[0] - if len(excs) == 0: - raise OSError, "Failed to remove dir for unknown reason." - raise OSError, excs - -def rm_dir(dirname): - # Renamed to be like shutil.rmtree and unlike rmdir. - return rmtree(dirname) - -def remove_if_possible(f): - try: - remove(f) - except EnvironmentError: - pass - -def remove_if_present(f): - try: - remove(f) - except EnvironmentError, le: - # Ignore "No such file or directory", re-raise any other exception. - if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): - raise - -def rmdir_if_possible(f): - try: - rmdir(f) - except EnvironmentError: - pass - -def open_or_create(fname, binarymode=True): - try: - f = open(fname, binarymode and "r+b" or "r+") - except EnvironmentError: - f = open(fname, binarymode and "w+b" or "w+") - return f - -def du(basedir): - size = 0 - - for root, dirs, files in os.walk(basedir): - for f in files: - fn = os.path.join(root, f) - size += os.path.getsize(fn) - - return size diff --git a/libs/pyutil/iputil.py b/libs/pyutil/iputil.py index b8b96362..cb3e7c02 100644 --- a/libs/pyutil/iputil.py +++ b/libs/pyutil/iputil.py @@ -1,22 +1,12 @@ -# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith) -# portions adapted from nattraverso.ipdiscover -# portions authored by Brian Warner, working for Allmydata -# most recent version authored by Zooko O'Whielacronx, working for Allmydata - # from the Python Standard Library -import os, re, socket, sys +import os, re, socket, sys, subprocess # from Twisted -from twisted.internet import defer, reactor -from twisted.python import failure +from twisted.internet import defer, threads, reactor from twisted.internet.protocol import DatagramProtocol -from twisted.internet.utils import getProcessOutput from twisted.python.procutils import which from twisted.python import log -# from pyutil -import observer - try: import resource def increase_rlimits(): @@ -77,6 +67,7 @@ except ImportError: # since one might be shadowing the other. This hack appeases pyflakes. increase_rlimits = _increase_rlimits + def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad @@ -121,14 +112,16 @@ def get_local_ip_for(target): except socket.gaierror: # DNS isn't running, or somehow we encountered an error - # note: if an interface is configured and up, but nothing is connected to it, - # gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror - # . This is synchronous and occurs for each node being started, so users of certain unit - # tests will see something like 120s of delay, which may be enough to hit the default - # trial timeouts. For that reason, get_local_addresses_async() was changed to default to - # the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also - # makes node startup a tad faster. - + # note: if an interface is configured and up, but nothing is + # connected to it, gethostbyname("A.ROOT-SERVERS.NET") will take 20 + # seconds to raise socket.gaierror . This is synchronous and occurs + # for each node being started, so users of + # test.common.SystemTestMixin (like test_system) will see something + # like 120s of delay, which may be enough to hit the default trial + # timeouts. For that reason, get_local_addresses_async() was changed + # to default to the numerical ip address for A.ROOT-SERVERS.NET, to + # avoid this DNS lookup. This also makes node startup fractionally + # faster. return None udpprot = DatagramProtocol() port = reactor.listenUDP(0, udpprot) @@ -146,16 +139,29 @@ _platform_map = { "linux-i386": "linux", # redhat "linux-ppc": "linux", # redhat "linux2": "linux", # debian + "linux3": "linux", # debian "win32": "win32", "irix6-n32": "irix", "irix6-n64": "irix", "irix6": "irix", "openbsd2": "bsd", + "openbsd3": "bsd", + "openbsd4": "bsd", + "openbsd5": "bsd", "darwin": "bsd", # Mac OS X "freebsd4": "bsd", "freebsd5": "bsd", "freebsd6": "bsd", + "freebsd7": "bsd", + "freebsd8": "bsd", + "freebsd9": "bsd", "netbsd1": "bsd", + "netbsd2": "bsd", + "netbsd3": "bsd", + "netbsd4": "bsd", + "netbsd5": "bsd", + "netbsd6": "bsd", + "dragonfly2": "bsd", "sunos5": "sunos", "cygwin": "cygwin", } @@ -173,12 +179,12 @@ _win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\ # These work in Redhat 6.x and Debian 2.2 potato _linux_path = '/sbin/ifconfig' -_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) +_linux_re = re.compile('^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) -# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD -_bsd_path = '/sbin/ifconfig' -_bsd_args = ('-a',) -_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) +# NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X +_netbsd_path = '/sbin/ifconfig' +_netbsd_args = ('-a',) +_netbsd_re = re.compile('^\s+inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) # Irix 6.5 _irix_path = '/usr/etc/ifconfig' @@ -186,39 +192,6 @@ _irix_path = '/usr/etc/ifconfig' # Solaris 2.x _sunos_path = '/usr/sbin/ifconfig' -class SequentialTrier(object): - """ I hold a list of executables to try and try each one in turn - until one gives me a list of IP addresses.""" - - def __init__(self, exebasename, args, regex): - assert not os.path.isabs(exebasename) - self.exes_left_to_try = which(exebasename) - self.exes_left_to_try.reverse() - self.args = args - self.regex = regex - self.o = observer.OneShotObserverList() - self._try_next() - - def _try_next(self): - if not self.exes_left_to_try: - self.o.fire(None) - else: - exe = self.exes_left_to_try.pop() - d2 = _query(exe, self.args, self.regex) - - def cb(res): - if res: - self.o.fire(res) - else: - self._try_next() - - def eb(why): - self._try_next() - - d2.addCallbacks(cb, eb) - - def when_tried(self): - return self.o.when_fired() # k: platform string as provided in the value of _platform_map # v: tuple of (path_to_tool, args, regex,) @@ -226,19 +199,22 @@ _tool_map = { "linux": (_linux_path, (), _linux_re,), "win32": (_win32_path, _win32_args, _win32_re,), "cygwin": (_win32_path, _win32_args, _win32_re,), - "bsd": (_bsd_path, _bsd_args, _bsd_re,), - "irix": (_irix_path, _bsd_args, _bsd_re,), - "sunos": (_sunos_path, _bsd_args, _bsd_re,), + "bsd": (_netbsd_path, _netbsd_args, _netbsd_re,), + "irix": (_irix_path, _netbsd_args, _netbsd_re,), + "sunos": (_sunos_path, _netbsd_args, _netbsd_re,), } + def _find_addresses_via_config(): - # originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API. + return threads.deferToThread(_synchronously_find_addresses_via_config) + +def _synchronously_find_addresses_via_config(): + # originally by Greg Smith, hacked by Zooko to conform to Brian's API platform = _platform_map.get(sys.platform) - (pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,)) + if not platform: + raise UnsupportedPlatformError(sys.platform) - # If the platform isn't known then we attempt BSD-style ifconfig. If it - # turns out that we don't get anything resembling a dotted quad IPv4 address - # out of it, then we'll raise UnsupportedPlatformError. + (pathtotool, args, regex,) = _tool_map[platform] # If pathtotool is a fully qualified path then we just try that. # If it is merely an executable name then we use Twisted's @@ -246,34 +222,33 @@ def _find_addresses_via_config(): # gives us something that resembles a dotted-quad IPv4 address. if os.path.isabs(pathtotool): - d = _query(pathtotool, args, regex) + return _query(pathtotool, args, regex) else: - d = SequentialTrier(pathtotool, args, regex).when_tried() - - d.addCallback(_check_result) - return d - -def _check_result(result): - if not result and not _platform_map.has_key(sys.platform): - return failure.Failure(UnsupportedPlatformError(sys.platform)) - else: - return result + exes_to_try = which(pathtotool) + for exe in exes_to_try: + try: + addresses = _query(exe, args, regex) + except Exception: + addresses = [] + if addresses: + return addresses + return [] def _query(path, args, regex): - d = getProcessOutput(path, args) - def _parse(output): - addresses = [] - outputsplit = output.split('\n') - for outline in outputsplit: - m = regex.match(outline) - if m: - addr = m.groupdict()['address'] - if addr not in addresses: - addresses.append(addr) + env = {'LANG': 'en_US.UTF-8'} + p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + (output, err) = p.communicate() - return addresses - d.addCallback(_parse) - return d + addresses = [] + outputsplit = output.split('\n') + for outline in outputsplit: + m = regex.match(outline) + if m: + addr = m.groupdict()['address'] + if addr not in addresses: + addresses.append(addr) + + return addresses def _cygwin_hack_find_addresses(target): addresses = [] diff --git a/libs/pyutil/iputil.py~ b/libs/pyutil/iputil.py~ new file mode 100644 index 00000000..b8b96362 --- /dev/null +++ b/libs/pyutil/iputil.py~ @@ -0,0 +1,288 @@ +# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith) +# portions adapted from nattraverso.ipdiscover +# portions authored by Brian Warner, working for Allmydata +# most recent version authored by Zooko O'Whielacronx, working for Allmydata + +# from the Python Standard Library +import os, re, socket, sys + +# from Twisted +from twisted.internet import defer, reactor +from twisted.python import failure +from twisted.internet.protocol import DatagramProtocol +from twisted.internet.utils import getProcessOutput +from twisted.python.procutils import which +from twisted.python import log + +# from pyutil +import observer + +try: + import resource + def increase_rlimits(): + # We'd like to raise our soft resource.RLIMIT_NOFILE, since certain + # systems (OS-X, probably solaris) start with a relatively low limit + # (256), and some unit tests want to open up more sockets than this. + # Most linux systems start with both hard and soft limits at 1024, + # which is plenty. + + # unfortunately the values to pass to setrlimit() vary widely from + # one system to another. OS-X reports (256, HUGE), but the real hard + # limit is 10240, and accepts (-1,-1) to mean raise it to the + # maximum. Cygwin reports (256, -1), then ignores a request of + # (-1,-1): instead you have to guess at the hard limit (it appears to + # be 3200), so using (3200,-1) seems to work. Linux reports a + # sensible (1024,1024), then rejects (-1,-1) as trying to raise the + # maximum limit, so you could set it to (1024,1024) but you might as + # well leave it alone. + + try: + current = resource.getrlimit(resource.RLIMIT_NOFILE) + except AttributeError: + # we're probably missing RLIMIT_NOFILE + return + + if current[0] >= 1024: + # good enough, leave it alone + return + + try: + if current[1] > 0 and current[1] < 1000000: + # solaris reports (256, 65536) + resource.setrlimit(resource.RLIMIT_NOFILE, + (current[1], current[1])) + else: + # this one works on OS-X (bsd), and gives us 10240, but + # it doesn't work on linux (on which both the hard and + # soft limits are set to 1024 by default). + resource.setrlimit(resource.RLIMIT_NOFILE, (-1,-1)) + new = resource.getrlimit(resource.RLIMIT_NOFILE) + if new[0] == current[0]: + # probably cygwin, which ignores -1. Use a real value. + resource.setrlimit(resource.RLIMIT_NOFILE, (3200,-1)) + + except ValueError: + log.msg("unable to set RLIMIT_NOFILE: current value %s" + % (resource.getrlimit(resource.RLIMIT_NOFILE),)) + except: + # who knows what. It isn't very important, so log it and continue + log.err() +except ImportError: + def _increase_rlimits(): + # TODO: implement this for Windows. Although I suspect the + # solution might be "be running under the iocp reactor and + # make this function be a no-op". + pass + # pyflakes complains about two 'def FOO' statements in the same time, + # since one might be shadowing the other. This hack appeases pyflakes. + increase_rlimits = _increase_rlimits + +def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET + """ + Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad + strings) that are currently configured on this host, sorted in descending + order of how likely we think they are to work. + + @param target: we want to learn an IP address they could try using to + connect to us; The default value is fine, but it might help if you + pass the address of a host that you are actually trying to be + reachable to. + """ + addresses = [] + local_ip = get_local_ip_for(target) + if local_ip: + addresses.append(local_ip) + + if sys.platform == "cygwin": + d = _cygwin_hack_find_addresses(target) + else: + d = _find_addresses_via_config() + + def _collect(res): + for addr in res: + if addr != "0.0.0.0" and not addr in addresses: + addresses.append(addr) + return addresses + d.addCallback(_collect) + + return d + +def get_local_ip_for(target): + """Find out what our IP address is for use by a given target. + + @return: the IP address as a dotted-quad string which could be used by + to connect to us. It might work for them, it might not. If + there is no suitable address (perhaps we don't currently have an + externally-visible interface), this will return None. + """ + + try: + target_ipaddr = socket.gethostbyname(target) + except socket.gaierror: + # DNS isn't running, or somehow we encountered an error + + # note: if an interface is configured and up, but nothing is connected to it, + # gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror + # . This is synchronous and occurs for each node being started, so users of certain unit + # tests will see something like 120s of delay, which may be enough to hit the default + # trial timeouts. For that reason, get_local_addresses_async() was changed to default to + # the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also + # makes node startup a tad faster. + + return None + udpprot = DatagramProtocol() + port = reactor.listenUDP(0, udpprot) + try: + udpprot.transport.connect(target_ipaddr, 7) + localip = udpprot.transport.getHost().host + except socket.error: + # no route to that host + localip = None + port.stopListening() # note, this returns a Deferred + return localip + +# k: result of sys.platform, v: which kind of IP configuration reader we use +_platform_map = { + "linux-i386": "linux", # redhat + "linux-ppc": "linux", # redhat + "linux2": "linux", # debian + "win32": "win32", + "irix6-n32": "irix", + "irix6-n64": "irix", + "irix6": "irix", + "openbsd2": "bsd", + "darwin": "bsd", # Mac OS X + "freebsd4": "bsd", + "freebsd5": "bsd", + "freebsd6": "bsd", + "netbsd1": "bsd", + "sunos5": "sunos", + "cygwin": "cygwin", + } + +class UnsupportedPlatformError(Exception): + pass + +# Wow, I'm really amazed at home much mileage we've gotten out of calling +# the external route.exe program on windows... It appears to work on all +# versions so far. Still, the real system calls would much be preferred... +# ... thus wrote Greg Smith in time immemorial... +_win32_path = 'route.exe' +_win32_args = ('print',) +_win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S) + +# These work in Redhat 6.x and Debian 2.2 potato +_linux_path = '/sbin/ifconfig' +_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) + +# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD +_bsd_path = '/sbin/ifconfig' +_bsd_args = ('-a',) +_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) + +# Irix 6.5 +_irix_path = '/usr/etc/ifconfig' + +# Solaris 2.x +_sunos_path = '/usr/sbin/ifconfig' + +class SequentialTrier(object): + """ I hold a list of executables to try and try each one in turn + until one gives me a list of IP addresses.""" + + def __init__(self, exebasename, args, regex): + assert not os.path.isabs(exebasename) + self.exes_left_to_try = which(exebasename) + self.exes_left_to_try.reverse() + self.args = args + self.regex = regex + self.o = observer.OneShotObserverList() + self._try_next() + + def _try_next(self): + if not self.exes_left_to_try: + self.o.fire(None) + else: + exe = self.exes_left_to_try.pop() + d2 = _query(exe, self.args, self.regex) + + def cb(res): + if res: + self.o.fire(res) + else: + self._try_next() + + def eb(why): + self._try_next() + + d2.addCallbacks(cb, eb) + + def when_tried(self): + return self.o.when_fired() + +# k: platform string as provided in the value of _platform_map +# v: tuple of (path_to_tool, args, regex,) +_tool_map = { + "linux": (_linux_path, (), _linux_re,), + "win32": (_win32_path, _win32_args, _win32_re,), + "cygwin": (_win32_path, _win32_args, _win32_re,), + "bsd": (_bsd_path, _bsd_args, _bsd_re,), + "irix": (_irix_path, _bsd_args, _bsd_re,), + "sunos": (_sunos_path, _bsd_args, _bsd_re,), + } +def _find_addresses_via_config(): + # originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API. + + platform = _platform_map.get(sys.platform) + (pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,)) + + # If the platform isn't known then we attempt BSD-style ifconfig. If it + # turns out that we don't get anything resembling a dotted quad IPv4 address + # out of it, then we'll raise UnsupportedPlatformError. + + # If pathtotool is a fully qualified path then we just try that. + # If it is merely an executable name then we use Twisted's + # "which()" utility and try each executable in turn until one + # gives us something that resembles a dotted-quad IPv4 address. + + if os.path.isabs(pathtotool): + d = _query(pathtotool, args, regex) + else: + d = SequentialTrier(pathtotool, args, regex).when_tried() + + d.addCallback(_check_result) + return d + +def _check_result(result): + if not result and not _platform_map.has_key(sys.platform): + return failure.Failure(UnsupportedPlatformError(sys.platform)) + else: + return result + +def _query(path, args, regex): + d = getProcessOutput(path, args) + def _parse(output): + addresses = [] + outputsplit = output.split('\n') + for outline in outputsplit: + m = regex.match(outline) + if m: + addr = m.groupdict()['address'] + if addr not in addresses: + addresses.append(addr) + + return addresses + d.addCallback(_parse) + return d + +def _cygwin_hack_find_addresses(target): + addresses = [] + for h in [target, "localhost", "127.0.0.1",]: + try: + addr = get_local_ip_for(h) + if addr not in addresses: + addresses.append(addr) + except socket.gaierror: + pass + + return defer.succeed(addresses) diff --git a/libs/pyutil/mathutil.py b/libs/pyutil/mathutil.py index 46781b0c..9c169801 100644 --- a/libs/pyutil/mathutil.py +++ b/libs/pyutil/mathutil.py @@ -11,7 +11,7 @@ def div_ceil(n, d): """ The smallest integer k such that k*d >= n. """ - return (n/d) + (n%d != 0) + return int((n//d) + (n%d != 0)) def next_multiple(n, k): """ diff --git a/libs/pyutil/mathutil.py~ b/libs/pyutil/mathutil.py~ new file mode 100644 index 00000000..46781b0c --- /dev/null +++ b/libs/pyutil/mathutil.py~ @@ -0,0 +1,106 @@ +# Copyright (c) 2005-2010 Zooko Wilcox-O'Hearn +# This file is part of pyutil; see README.rst for licensing terms. + +""" +A few commonly needed functions. +""" + +import math + +def div_ceil(n, d): + """ + The smallest integer k such that k*d >= n. + """ + return (n/d) + (n%d != 0) + +def next_multiple(n, k): + """ + The smallest multiple of k which is >= n. Note that if n is 0 then the + answer is 0. + """ + return div_ceil(n, k) * k + +def pad_size(n, k): + """ + The smallest number that has to be added to n to equal a multiple of k. + """ + if n%k: + return k - n%k + else: + return 0 + +def is_power_of_k(n, k): + return k**int(math.log(n, k) + 0.5) == n + +def next_power_of_k(n, k): + p = 1 + while p < n: + p *= k + return p + +def ave(l): + return sum(l) / len(l) + +def log_ceil(n, b): + """ + The smallest integer k such that b^k >= n. + + log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. + the number of bits needed to store any of 128 possible values is 7. + """ + p = 1 + k = 0 + while p < n: + p *= b + k += 1 + return k + +def log_floor(n, b): + """ + The largest integer k such that b^k <= n. + """ + p = 1 + k = 0 + while p <= n: + p *= b + k += 1 + return k - 1 + +def linear_fit_slope(ps): + """ + Single-independent-variable linear regression -- least squares method. + + At least, I *think* this function computes that answer. I no longer + remember where I learned this trick and at the moment I can't prove to + myself that this is correct. + + @param ps a sequence of tuples of (x, y) + """ + avex = ave([x for (x, y) in ps]) + avey = ave([y for (x, y) in ps]) + sxy = sum([ (x - avex) * (y - avey) for (x, y) in ps ]) + sxx = sum([ (x - avex) ** 2 for (x, y) in ps ]) + if sxx == 0: + return None + return sxy / sxx + +def permute(l): + """ + Return all possible permutations of l. + + @type l: sequence + @rtype a set of sequences + """ + if len(l) == 1: + return [l,] + + res = [] + for i in range(len(l)): + l2 = list(l[:]) + x = l2.pop(i) + for l3 in permute(l2): + l3.append(x) + res.append(l3) + + return res + diff --git a/libs/pyutil/odict.py~ b/libs/pyutil/odict.py~ deleted file mode 100644 index 0ed5ce7b..00000000 --- a/libs/pyutil/odict.py~ +++ /dev/null @@ -1,552 +0,0 @@ -# Copyright (c) 2002-2009 Zooko "Zooko" Wilcox-O'Hearn - -""" -This module offers a Ordered Dict, which is a dict that preserves -insertion order. See PEP 372 for description of the problem. This -implementation uses a linked-list to get good O(1) asymptotic -performance. (Actually it is O(hashtable-update-cost), but whatever.) - -Warning: if -O optimizations are not turned on then OrderedDict performs -extensive self-analysis in every function call, which can take minutes -and minutes for a large cache. Turn on -O, or comment out assert -self._assert_invariants() -""" - -import operator - -from assertutil import _assert, precondition -from humanreadable import hr - -class OrderedDict: - """ - An efficient ordered dict. - - Adding an item that is already in the dict *does not* make it the - most- recently-added item although it may change the state of the - dict itself (if the value is different than the previous value). - - See also SmallOrderedDict (below), which is faster in some cases. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - k = self.i - precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) - (v, p, n,) = self.c.d[k] - self.i = p - return (k, v,) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - k = self.i - precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) - (v, p, n,) = self.c.d[k] - self.i = p - return k - - class ValIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - precondition(self.c.d.has_key(self.i), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - (v, p, n,) = self.c.d[self.i] - self.i = p - return v - - class Sentinel: - def __init__(self, msg): - self.msg = msg - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.msg,) - - def __init__(self, initialdata={}): - self.d = {} # k: k, v: [v, prev, next,] # the dict - self.hs = OrderedDict.Sentinel("hs") - self.ts = OrderedDict.Sentinel("ts") - self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.update(initialdata) - - assert self._assert_invariants() - - def __repr_n__(self, n=None): - s = ["{",] - try: - iter = self.iteritems() - x = iter.next() - s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - i = 1 - while (n is None) or (i < n): - x = iter.next() - s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - except StopIteration: - pass - s.append("}") - return ''.join(s) - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) - - def _assert_invariants(self): - _assert((len(self.d) > 2) == (self.d[self.hs][2] is not self.ts) == (self.d[self.ts][1] is not self.hs), "Head and tail point to something other than each other if and only if there is at least one element in the dictionary.", self.hs, self.ts, len(self.d)) - foundprevsentinel = 0 - foundnextsentinel = 0 - for (k, (v, p, n,)) in self.d.iteritems(): - _assert(v not in (self.hs, self.ts,)) - _assert(p is not self.ts, "A reference to the tail sentinel may not appear in prev.", k, v, p, n) - _assert(n is not self.hs, "A reference to the head sentinel may not appear in next.", k, v, p, n) - _assert(p in self.d, "Each prev is required to appear as a key in the dict.", k, v, p, n) - _assert(n in self.d, "Each next is required to appear as a key in the dict.", k, v, p, n) - if p is self.hs: - foundprevsentinel += 1 - _assert(foundprevsentinel <= 2, "No more than two references to the head sentinel may appear as a prev.", k, v, p, n) - if n is self.ts: - foundnextsentinel += 1 - _assert(foundnextsentinel <= 2, "No more than one reference to the tail sentinel may appear as a next.", k, v, p, n) - _assert(foundprevsentinel == 2, "A reference to the head sentinel is required appear as a prev (plus a self-referential reference).") - _assert(foundnextsentinel == 2, "A reference to the tail sentinel is required appear as a next (plus a self-referential reference).") - - count = 0 - for (k, v,) in self.iteritems(): - _assert(k not in (self.hs, self.ts,), k, self.hs, self.ts) - count += 1 - _assert(count == len(self.d)-2, count, len(self.d)) # -2 for the sentinels - - return True - - def move_to_most_recent(self, k, strictkey=False): - assert self._assert_invariants() - - if not self.d.has_key(k): - if strictkey: - raise KeyError, k - return - - node = self.d[k] - - # relink - self.d[node[1]][2] = node[2] - self.d[node[2]][1] = node[1] - - # move to front - hnode = self.d[self.hs] - - node[1] = self.hs - node[2] = hnode[2] - hnode[2] = k - self.d[node[2]][1] = k - - assert self._assert_invariants() - - def iteritems(self): - return OrderedDict.ItemIterator(self) - - def itervalues(self): - return OrderedDict.ValIterator(self) - - def iterkeys(self): - return self.__iter__() - - def __iter__(self): - return OrderedDict.KeyIterator(self) - - def __getitem__(self, key, default=None, strictkey=True): - node = self.d.get(key) - if not node: - if strictkey: - raise KeyError, key - return default - return node[0] - - def __setitem__(self, k, v=None): - assert self._assert_invariants() - - node = self.d.get(k) - if node: - node[0] = v - return - - hnode = self.d[self.hs] - n = hnode[2] - self.d[k] = [v, self.hs, n,] - hnode[2] = k - self.d[n][1] = k - - assert self._assert_invariants() - return v - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the value removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if self.d.has_key(key): - node = self.d[key] - # relink - self.d[node[1]][2] = node[2] - self.d[node[2]][1] = node[1] - del self.d[key] - assert self._assert_invariants() - return node[0] - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def has_key(self, key): - assert self._assert_invariants() - if self.d.has_key(key): - assert self._assert_invariants() - return True - else: - assert self._assert_invariants() - return False - - def clear(self): - assert self._assert_invariants() - self.d.clear() - self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - - for (k, v,) in otherdict.iteritems(): - assert self._assert_invariants() - self[k] = v - assert self._assert_invariants() - - def pop(self): - assert self._assert_invariants() - if len(self.d) < 2: # the +2 is for the sentinels - raise KeyError, 'popitem(): dictionary is empty' - k = self.d[self.hs][2] - self.remove(k) - assert self._assert_invariants() - return k - - def popitem(self): - assert self._assert_invariants() - if len(self.d) < 2: # the +2 is for the sentinels - raise KeyError, 'popitem(): dictionary is empty' - k = self.d[self.hs][2] - val = self.remove(k) - assert self._assert_invariants() - return (k, val,) - - def keys_unsorted(self): - assert self._assert_invariants() - t = self.d.copy() - del t[self.hs] - del t[self.ts] - assert self._assert_invariants() - return t.keys() - - def keys(self): - res = [None] * len(self) - i = 0 - for k in self.iterkeys(): - res[i] = k - i += 1 - return res - - def values_unsorted(self): - assert self._assert_invariants() - t = self.d.copy() - del t[self.hs] - del t[self.ts] - assert self._assert_invariants() - return map(operator.__getitem__, t.values(), [0]*len(t)) - - def values(self): - res = [None] * len(self) - i = 0 - for v in self.itervalues(): - res[i] = v - i += 1 - return res - - def items(self): - res = [None] * len(self) - i = 0 - for it in self.iteritems(): - res[i] = it - i += 1 - return res - - def __len__(self): - return len(self.d) - 2 - - def insert(self, key, val=None): - assert self._assert_invariants() - result = self.__setitem__(key, val) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def get(self, key, default=None): - return self.__getitem__(key, default, strictkey=False) - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - -class SmallOrderedDict(dict): - """ - SmallOrderedDict is faster than OrderedDict for small sets. How small? That - depends on your machine and which operations you use most often. Use - performance profiling to determine whether the cache class that you are - using makes any difference to the performance of your program, and if it - does, then run "quick_bench()" in test/test_cache.py to see which cache - implementation is faster for the size of your datasets. - - A simple least-recently-used cache. It keeps an LRU queue, and - when the number of items in the cache reaches maxsize, it removes - the least recently used item. - - "Looking" at an item or a key such as with "has_key()" makes that - item become the most recently used item. - - You can also use "refresh()" to explicitly make an item become the most - recently used item. - - Adding an item that is already in the dict *does* make it the - most- recently-used item although it does not change the state of - the dict itself. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return (k, dict.__getitem__(self.c, k),) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return k - - class ValueIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return dict.__getitem__(self.c, k) - - def __init__(self, initialdata={}, maxsize=128): - dict.__init__(self, initialdata) - self._lru = initialdata.keys() # contains keys - self._maxsize = maxsize - over = len(self) - self._maxsize - if over > 0: - map(dict.__delitem__, [self]*over, self._lru[:over]) - del self._lru[:over] - assert self._assert_invariants() - - def _assert_invariants(self): - _assert(len(self._lru) <= self._maxsize, "Size is required to be <= maxsize.") - _assert(len(filter(lambda x: dict.has_key(self, x), self._lru)) == len(self._lru), "Each key in self._lru is required to be in dict.", filter(lambda x: not dict.has_key(self, x), self._lru), len(self._lru), self._lru, len(self), self) - _assert(len(filter(lambda x: x in self._lru, self.keys())) == len(self), "Each key in dict is required to be in self._lru.", filter(lambda x: x not in self._lru, self.keys()), len(self._lru), self._lru, len(self), self) - _assert(len(self._lru) == len(self), "internal consistency", filter(lambda x: x not in self.keys(), self._lru), len(self._lru), self._lru, len(self), self) - _assert(len(self._lru) <= self._maxsize, "internal consistency", len(self._lru), self._lru, self._maxsize) - return True - - def insert(self, key, item=None): - assert self._assert_invariants() - result = self.__setitem__(key, item) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def __setitem__(self, key, item=None): - assert self._assert_invariants() - if dict.has_key(self, key): - self._lru.remove(key) - else: - if len(self._lru) == self._maxsize: - # If this insert is going to increase the size of the cache to bigger than maxsize: - killkey = self._lru.pop(0) - dict.__delitem__(self, killkey) - dict.__setitem__(self, key, item) - self._lru.append(key) - assert self._assert_invariants() - return item - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the object removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if dict.has_key(self, key): - val = dict.__getitem__(self, key) - dict.__delitem__(self, key) - self._lru.remove(key) - assert self._assert_invariants() - return val - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def clear(self): - assert self._assert_invariants() - dict.clear(self) - self._lru = [] - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - if len(otherdict) > self._maxsize: - # Handling this special case here makes it possible to implement the - # other more common cases faster below. - dict.clear(self) - self._lru = [] - if self._maxsize > (len(otherdict) - self._maxsize): - dict.update(self, otherdict) - while len(self) > self._maxsize: - dict.popitem(self) - else: - for k, v, in otherdict.iteritems(): - if len(self) == self._maxsize: - break - dict.__setitem__(self, k, v) - self._lru = dict.keys(self) - assert self._assert_invariants() - return self - - for k in otherdict.iterkeys(): - if dict.has_key(self, k): - self._lru.remove(k) - self._lru.extend(otherdict.keys()) - dict.update(self, otherdict) - - over = len(self) - self._maxsize - if over > 0: - map(dict.__delitem__, [self]*over, self._lru[:over]) - del self._lru[:over] - - assert self._assert_invariants() - return self - - def has_key(self, key): - assert self._assert_invariants() - if dict.has_key(self, key): - assert key in self._lru, "key: %s, self._lru: %s" % tuple(map(hr, (key, self._lru,))) - self._lru.remove(key) - self._lru.append(key) - assert self._assert_invariants() - return True - else: - assert self._assert_invariants() - return False - - def refresh(self, key, strictkey=True): - """ - @param strictkey: raise a KeyError exception if key isn't present - """ - assert self._assert_invariants() - if not dict.has_key(self, key): - if strictkey: - raise KeyError, key - return - self._lru.remove(key) - self._lru.append(key) - - def popitem(self): - if not self._lru: - raise KeyError, 'popitem(): dictionary is empty' - k = self._lru[-1] - obj = self.remove(k) - return (k, obj,) diff --git a/libs/pyutil/randutil.py b/libs/pyutil/randutil.py index a3efb74f..82eb3e18 100644 --- a/libs/pyutil/randutil.py +++ b/libs/pyutil/randutil.py @@ -80,6 +80,5 @@ seed = randobj.seed def randstr(n): return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) -import random as insecurerandom def insecurerandstr(n): - return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n))) + return os.urandom(n) diff --git a/libs/pyutil/randutil.py~ b/libs/pyutil/randutil.py~ deleted file mode 100644 index b0f1c4f9..00000000 --- a/libs/pyutil/randutil.py~ +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn -# This file is part of pyutil; see README.rst for licensing terms. - -import warnings -import os, random - -try: - import hashexpand - class SHA256Random(hashexpand.SHA256Expander, random.Random): - def __init__(self, seed=None, deterministic=True): - warnings.warn("deprecated", DeprecationWarning) - if not deterministic: - raise NotImplementedError, "SHA256Expander is always deterministic. For non-deterministic, try urandomRandom." - - hashexpand.SHA256Expander.__init__(self) - random.Random.__init__(self, seed) - self.seed(seed) - - def seed(self, seed=None): - if seed is None: - import increasing_timer - seed = repr(increasing_timer.time()) - hashexpand.SHA256Expander.seed(self, seed) - - - class SHA256Random(hashexpand.SHA256Expander, random.Random): - def __init__(self, seed=""): - warnings.warn("deprecated", DeprecationWarning) - hashexpand.SHA256Expander.__init__(self) - self.seed(seed) - - def seed(self, seed=None): - if seed is None: - seed = os.urandom(32) - hashexpand.SHA256Expander.seed(self, seed) -except ImportError, le: - class InsecureSHA256Random: - def __init__(self, seed=None): - raise ImportError, le - class SHA256Random: - def __init__(self, seed=""): - raise ImportError, le - -class devrandomRandom(random.Random): - """ The problem with using this one, of course, is that it blocks. This - is, of course, a security flaw. (On Linux and probably on other - systems.) --Zooko 2005-03-04 - - Not repeatable. - """ - def __init__(self): - warnings.warn("deprecated", DeprecationWarning) - self.dr = open("/dev/random", "r") - - def get(self, bytes): - return self.dr.read(bytes) - - -class devurandomRandom(random.Random): - """ The problem with using this one is that it gives answers even when it - has never been properly seeded, e.g. when you are booting from CD and have - just started up and haven't yet gathered enough entropy to actually be - unguessable. (On Linux and probably on other systems.) --Zooko 2005-03-04 - - Not repeatable. - """ - def get(self, bytes): - warnings.warn("deprecated", DeprecationWarning) - return os.urandom(bytes) - - -randobj = devurandomRandom() -get = randobj.get -random = randobj.random -randrange = randobj.randrange -shuffle = randobj.shuffle -choice = randobj.choice -seed = randobj.seed - -def randstr(n): - return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) - -import random as insecurerandom -def insecurerandstr(n): - return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n))) diff --git a/libs/pyutil/scripts/passphrase.py b/libs/pyutil/scripts/passphrase.py new file mode 100644 index 00000000..bed79c13 --- /dev/null +++ b/libs/pyutil/scripts/passphrase.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse, math, random + +from pyutil.mathutil import div_ceil + +from pkg_resources import resource_stream + +def recursive_subset_sum(entropy_needed, wordlists): + # Pick a minimalish set of numbers which sum to at least + # entropy_needed. + + # Okay now what's the smallest number of words which will give us + # at least this much entropy? + entropy_of_biggest_wordlist = wordlists[-1][0] + assert isinstance(entropy_of_biggest_wordlist, float), wordlists[-1] + needed_words = div_ceil(entropy_needed, entropy_of_biggest_wordlist) + # How much entropy do we need from each word? + needed_entropy_per_word = entropy_needed / needed_words + # What's the smallest wordlist that offers at least this much + # entropy per word? + for (wlentropy, wl) in wordlists: + if wlentropy >= needed_entropy_per_word: + break + assert wlentropy >= needed_entropy_per_word, (wlentropy, needed_entropy_per_word) + + result = [(wlentropy, wl)] + # If we need more, recurse... + if wlentropy < entropy_needed: + rest = recursive_subset_sum(entropy_needed - wlentropy, wordlists) + result.extend(rest) + return result + +def gen_passphrase(entropy, allwords): + maxlenwords = [] + i = 2 # The smallest set is words of length 1 or 2. + words = [x for x in allwords if len(x) <= i] + maxlenwords.append((math.log(len(words), 2), words)) + while len(maxlenwords[-1][1]) < len(allwords): + i += 1 + words = [x for x in allwords if len(x) <= i] + maxlenwords.append((math.log(len(words), 2), words)) + + sr = random.SystemRandom() + passphrase = [] + + wordlists_to_use = recursive_subset_sum(entropy, maxlenwords) + + passphraseentropy = 0.0 + for (wle, wl) in wordlists_to_use: + passphrase.append(sr.choice(wl)) + passphraseentropy += wle + + return (u".".join(passphrase), passphraseentropy) + +def main(): + parser = argparse.ArgumentParser(prog="chbs", description="Create a random passphrase by picking a few random words.") + + parser.add_argument('-d', '--dictionary', help="what file to read a list of words from (or omit this option to use chbs's bundled dictionary)", type=argparse.FileType('rU'), metavar="DICT") + parser.add_argument('bits', help="how many bits of entropy minimum", type=float, metavar="BITS") + args = parser.parse_args() + + dicti = args.dictionary + if not dicti: + dicti = resource_stream('pyutil', 'data/wordlist.txt') + allwords = set([x.decode('utf-8').strip().lower() for x in dicti.readlines()]) + + passphrase, bits = gen_passphrase(args.bits, allwords) + + print u"Your new password is: '%s'. It is worth about %s bits." % (passphrase, bits) diff --git a/libs/pyutil/scripts/time_comparisons.py b/libs/pyutil/scripts/time_comparisons.py new file mode 100644 index 00000000..15a38852 --- /dev/null +++ b/libs/pyutil/scripts/time_comparisons.py @@ -0,0 +1,209 @@ +# If you run this file, it will make up a random secret and then crack it +# using timing information from a string comparison function. Maybe--if it +# gets lucky. It takes a long, long time to work. + +# So, the thing I need help with is statistics. The way this thing works is +# extremely stupid. Suppose you want to know which function invocation takes +# longer: comparison(secret, guess1) or comparison(secret, guess2)? + +# If you can correctly determine that one of them takes longer than the +# other, then (a) you can use that to crack the secret, and (b) this is a +# unit test demonstrating that comparison() is not timing-safe. + +# So how does this script do it? Extremely stupidly. First of all, you can't +# reliably measure tiny times, so to measure the time that a function takes, +# we run that function 10,000 times in a row, measure how long that took, and +# divide by 10,000 to estimate how long any one run would have taken. + +# Then, we do that 100 times in a row, and take the fastest of 100 runs. (I +# also experimented with taking the mean of 100 runs instead of the fastest.) + +# Then, we just say whichever comparison took longer (for its fastest run of +# 100 runs of 10,000 executions per run) is the one we think is a closer +# guess to the secret. + +# Now I would *like* to think that there is some kind of statistical analysis +# more sophisticated than "take the slowest of the fastest of 100 runs of +# 10,000 executions". Such improved statistical analysis would hopefully be +# able to answer these two questions: + +# 1. Are these two function calls -- comparison(secret, guess1) and +# comparison(secret, guess2) -- drawing from the same distribution or +# different? If you can answer that question, then you've answered the +# question of whether "comparison" is timing-safe or not. + +# And, this would also allow the cracker to recover from a false step. If it +# incorrectly decides the the prefix of the secret is ABCX, when the real +# secret is ABCD, then after that every next step it takes will be the +# "drawing from the same distribution" kind -- any difference between ABCXQ +# and ABCXR will be just due to noise, since both are equally far from the +# correct answer, which startsw with ABCD. If it could realize that there is +# no real difference between the distributions, then it could back-track and +# recover. + +# 2. Giving the ability to measure, noisily, the time taken by comparison(), +# how can you most efficiently figure out which guess takes the longest? If +# you can do that more efficiently, you can crack secrets more efficiently. + +# The script takes two arguments. The first is how many symbols in the +# secret, and the second is how big the alphabet from which the symbols are +# drawn. To prove that this script can *ever* work, try passing length 5 and +# alphabet size 2. Also try editing the code to let is use sillycomp. That'll +# definitely make it work. If you can improve this script (as per the thing +# above about "needing better statistics") to the degree that it can crack a +# secret with length 32 and alphabet size 256, then that would be awesome. + +# See the result of this commandline: + +# $ python -c 'import time_comparisons ; time_comparisons.print_measurements()' + + +from pyutil import benchutil + +import hashlib, random, os + +from decimal import Decimal +D=Decimal + +p1 = 'a'*32 +p1a = 'a'*32 +p2 = 'a'*31+'b' # close, but no cigar +p3 = 'b'*32 # different in the first byte + +def randstr(n, alphabetsize): + alphabet = [ chr(x) for x in range(alphabetsize) ] + return ''.join([random.choice(alphabet) for i in range(n)]) + +def compare(n, f, a, b): + for i in xrange(n): + f(a, b) + +def eqeqcomp(a, b): + return a == b + +def sillycomp(a, b): + # This exposes a lot of information in its timing about how many leading bytes match. + for i in range(len(a)): + if a[i] != b[i]: + return False + for i in xrange(2**9): + pass + if len(a) == len(b): + return True + else: + return False + +def hashcomp(a, b): + # Brian Warner invented this for Tahoe-LAFS. It seems like it should be very safe agaist timing leakage of any kind, because of the inclusion of a new random randkey every time. Note that exposing the value of the hash (i.e. the output of md5(randkey+secret)) is *not* a security problem. You can post that on your web site and let all attackers have it, no problem. (Provided that the value of "randkey" remains secret.) + + randkey = os.urandom(32) + return hashlib.md5(randkey+ a).digest() == hashlib.md5(randkey+b).digest() + +def xorcomp(a, b): + # This appears to be the most popular timing-insensitive string comparison function. I'm not completely sure it is fully timing-insensitive. (There are all sorts of funny things inside Python, such as caching of integer objects < 100...) + if len(a) != len(b): + return False + result = 0 + for x, y in zip(a, b): + result |= ord(x) ^ ord(y) + return result == 0 + +def print_measurements(): + N=10**4 + REPS=10**2 + + print "all times are in nanoseconds per comparison (in scientific notation)" + print + + for comparator in [eqeqcomp, hashcomp, xorcomp, sillycomp]: + print "using comparator ", comparator + + # for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]: + trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')] + random.shuffle(trials) + for (a, b, desc) in trials: + print "comparing two strings that are %s to each other" % (desc,) + + def f(n): + compare(n, comparator, a, b) + + benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) + + print + +def try_to_crack_secret(cracker, comparator, secretlen, alphabetsize): + secret = randstr(secretlen, alphabetsize) + + def test_guess(x): + return comparator(secret, x) + + print "Giving cracker %s a chance to figure out the secret. Don't tell him, but the secret is %s. Whenever he makes a guess, we'll use comparator %s to decide if his guess is right ..." % (cracker, secret.encode('hex'), comparator,) + + guess = cracker(test_guess, secretlen, alphabetsize) + + print "Cracker %s guessed %r" % (cracker, guess,) + if guess == secret: + print "HE FIGURED IT OUT!? HOW DID HE DO THAT." + else: + print "HAHA. Our secret is safe." + +def byte_at_a_time_cracker(test_guess, secretlen, alphabetsize): + # If we were cleverer, we'd add some backtracking behaviour where, if we can't find any x such that ABCx stands out from the crowd as taking longer than all the other ABCy's, then we start to think that we've taken a wrong step and we go back to trying ABy's. Make sense? But we're not that clever. Once we take a step, we don't backtrack. + + print + + guess=[] + + while len(guess) < secretlen: + best_next_byte = None + best_next_byte_time = None + + # For each possible byte... + for next_byte in range(alphabetsize): + c = chr(next_byte) + + # Construct a guess with our best candidate so far... + candidate_guess = guess[:] + + # Plus that byte... + candidate_guess.append(c) + s = ''.join(candidate_guess) + + # Plus random bytes... + s += os.urandom(32 - len(s)) + + # And see how long it takes the test_guess to consider it... + def f(n): + for i in xrange(n): + test_guess(s) + + times = benchutil.rep_bench(f, 10**7, MAXREPS=10**3, quiet=True) + + fastesttime = times['mean'] + + print "%s..."%(c.encode('hex'),), + if best_next_byte is None or fastesttime > best_next_byte_time: + print "new candidate for slowest next-char: %s, took: %s" % (c.encode('hex'), fastesttime,), + + best_next_byte_time = fastesttime + best_next_byte = c + + # Okay we've tried all possible next bytes. Our guess is this one (the one that took longest to be tested by test_guess): + guess.append(best_next_byte) + print "SLOWEST next-char %s! Current guess at secret: %s" % (best_next_byte.encode('hex'), ''.join(guess).encode('hex'),) + + guess = ''.join(guess) + print "Our guess for the secret: %r" % (guess,) + return guess + +if __name__ == '__main__': + import sys + secretlen = int(sys.argv[1]) + alphabetsize = int(sys.argv[2]) + if alphabetsize > 256: + raise Exception("We assume we can fit one element of the alphabet into a byte.") + + print "secretlen: %d, alphabetsize: %d" % (secretlen, alphabetsize,) + + # try_to_crack_secret(byte_at_a_time_cracker, sillycomp, secretlen, alphabetsize) + try_to_crack_secret(byte_at_a_time_cracker, eqeqcomp, secretlen, alphabetsize) diff --git a/libs/pyutil/test/current/test_mathutil.py b/libs/pyutil/test/current/test_mathutil.py index 7c189dfb..da788758 100644 --- a/libs/pyutil/test/current/test_mathutil.py +++ b/libs/pyutil/test/current/test_mathutil.py @@ -42,6 +42,13 @@ class MathUtilTestCase(unittest.TestCase): self.failUnlessEqual(f(5, 3), 2) self.failUnlessEqual(f(6, 3), 2) self.failUnlessEqual(f(7, 3), 3) + self.failUnless(isinstance(f(0.0, 1), int)) + self.failUnlessEqual(f(7.0, 3.0), 3) + self.failUnlessEqual(f(7, 3.0), 3) + self.failUnlessEqual(f(7.0, 3), 3) + self.failUnlessEqual(f(6.0, 3.0), 2) + self.failUnlessEqual(f(6.0, 3), 2) + self.failUnlessEqual(f(6, 3.0), 2) def test_next_multiple(self): f = mathutil.next_multiple diff --git a/libs/pyutil/test/current/test_mathutil.py~ b/libs/pyutil/test/current/test_mathutil.py~ new file mode 100644 index 00000000..7c189dfb --- /dev/null +++ b/libs/pyutil/test/current/test_mathutil.py~ @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +import unittest + +from pyutil import mathutil +from pyutil.assertutil import _assert + +class MathUtilTestCase(unittest.TestCase): + def _help_test_is_power_of_k(self, k): + for i in range(2, 40): + _assert(mathutil.is_power_of_k(k**i, k), k, i) + + def test_is_power_of_k(self): + for i in range(2, 5): + self._help_test_is_power_of_k(i) + + def test_log_ceil(self): + f = mathutil.log_ceil + self.failUnlessEqual(f(1, 2), 0) + self.failUnlessEqual(f(1, 3), 0) + self.failUnlessEqual(f(2, 2), 1) + self.failUnlessEqual(f(2, 3), 1) + self.failUnlessEqual(f(3, 2), 2) + + def test_log_floor(self): + f = mathutil.log_floor + self.failUnlessEqual(f(1, 2), 0) + self.failUnlessEqual(f(1, 3), 0) + self.failUnlessEqual(f(2, 2), 1) + self.failUnlessEqual(f(2, 3), 0) + self.failUnlessEqual(f(3, 2), 1) + + def test_div_ceil(self): + f = mathutil.div_ceil + self.failUnlessEqual(f(0, 1), 0) + self.failUnlessEqual(f(0, 2), 0) + self.failUnlessEqual(f(0, 3), 0) + self.failUnlessEqual(f(1, 3), 1) + self.failUnlessEqual(f(2, 3), 1) + self.failUnlessEqual(f(3, 3), 1) + self.failUnlessEqual(f(4, 3), 2) + self.failUnlessEqual(f(5, 3), 2) + self.failUnlessEqual(f(6, 3), 2) + self.failUnlessEqual(f(7, 3), 3) + + def test_next_multiple(self): + f = mathutil.next_multiple + self.failUnlessEqual(f(5, 1), 5) + self.failUnlessEqual(f(5, 2), 6) + self.failUnlessEqual(f(5, 3), 6) + self.failUnlessEqual(f(5, 4), 8) + self.failUnlessEqual(f(5, 5), 5) + self.failUnlessEqual(f(5, 6), 6) + self.failUnlessEqual(f(32, 1), 32) + self.failUnlessEqual(f(32, 2), 32) + self.failUnlessEqual(f(32, 3), 33) + self.failUnlessEqual(f(32, 4), 32) + self.failUnlessEqual(f(32, 5), 35) + self.failUnlessEqual(f(32, 6), 36) + self.failUnlessEqual(f(32, 7), 35) + self.failUnlessEqual(f(32, 8), 32) + self.failUnlessEqual(f(32, 9), 36) + self.failUnlessEqual(f(32, 10), 40) + self.failUnlessEqual(f(32, 11), 33) + self.failUnlessEqual(f(32, 12), 36) + self.failUnlessEqual(f(32, 13), 39) + self.failUnlessEqual(f(32, 14), 42) + self.failUnlessEqual(f(32, 15), 45) + self.failUnlessEqual(f(32, 16), 32) + self.failUnlessEqual(f(32, 17), 34) + self.failUnlessEqual(f(32, 18), 36) + self.failUnlessEqual(f(32, 589), 589) + + def test_pad_size(self): + f = mathutil.pad_size + self.failUnlessEqual(f(0, 4), 0) + self.failUnlessEqual(f(1, 4), 3) + self.failUnlessEqual(f(2, 4), 2) + self.failUnlessEqual(f(3, 4), 1) + self.failUnlessEqual(f(4, 4), 0) + self.failUnlessEqual(f(5, 4), 3) + + def test_is_power_of_k_part_2(self): + f = mathutil.is_power_of_k + for i in range(1, 100): + if i in (1, 2, 4, 8, 16, 32, 64): + self.failUnless(f(i, 2), "but %d *is* a power of 2" % i) + else: + self.failIf(f(i, 2), "but %d is *not* a power of 2" % i) + for i in range(1, 100): + if i in (1, 3, 9, 27, 81): + self.failUnless(f(i, 3), "but %d *is* a power of 3" % i) + else: + self.failIf(f(i, 3), "but %d is *not* a power of 3" % i) + + def test_next_power_of_k(self): + f = mathutil.next_power_of_k + self.failUnlessEqual(f(0,2), 1) + self.failUnlessEqual(f(1,2), 1) + self.failUnlessEqual(f(2,2), 2) + self.failUnlessEqual(f(3,2), 4) + self.failUnlessEqual(f(4,2), 4) + for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i) + for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i) + for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i) + for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i) + for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i) + + self.failUnlessEqual(f(0,3), 1) + self.failUnlessEqual(f(1,3), 1) + self.failUnlessEqual(f(2,3), 3) + self.failUnlessEqual(f(3,3), 3) + for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i) + for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i) + for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i) + for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i) + + def test_ave(self): + f = mathutil.ave + self.failUnlessEqual(f([1,2,3]), 2) + self.failUnlessEqual(f([0,0,0,4]), 1) + self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) + + def failUnlessEqualContents(self, a, b): + self.failUnlessEqual(sorted(a), sorted(b)) + + def test_permute(self): + f = mathutil.permute + self.failUnlessEqualContents(f([]), []) + self.failUnlessEqualContents(f([1]), [[1]]) + self.failUnlessEqualContents(f([1,2]), [[1,2], [2,1]]) + self.failUnlessEqualContents(f([1,2,3]), + [[1,2,3], [1,3,2], + [2,1,3], [2,3,1], + [3,1,2], [3,2,1]]) diff --git a/libs/pyutil/time_comparisons.py b/libs/pyutil/time_comparisons.py deleted file mode 100644 index ee1bcfa0..00000000 --- a/libs/pyutil/time_comparisons.py +++ /dev/null @@ -1,44 +0,0 @@ -from pyutil import benchutil - -import hashlib, random, os - -from decimal import Decimal -D=Decimal - -p1 = 'a'*32 -p1a = 'a'*32 -p2 = 'a'*31+'b' # close, but no cigar -p3 = 'b'*32 # different in the first byte - -def compare(n, f, a, b): - for i in xrange(n): - f(a, b) - -def eqeqcomp(a, b): - return a == b - -def hashcomp(a, b): - salt = os.urandom(32) - return hashlib.md5(salt+ a).digest() == hashlib.md5(salt+b).digest() - -N=10**4 -REPS=10**2 - -print "all times are in nanoseconds per comparison (scientific notation)" -print - -for comparator in [eqeqcomp, hashcomp]: - print "using comparator ", comparator - - # for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]: - trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')] - random.shuffle(trials) - for (a, b, desc) in trials: - print "comparing two strings that are %s to each other" % (desc,) - - def f(n): - compare(n, comparator, a, b) - - benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - - print diff --git a/libs/pyutil/time_comparisons.py~ b/libs/pyutil/time_comparisons.py~ deleted file mode 100644 index abf151ad..00000000 --- a/libs/pyutil/time_comparisons.py~ +++ /dev/null @@ -1,72 +0,0 @@ -from pyutil import benchutil - -import hashlib -import os - -from decimal import Decimal -D=Decimal - -p1 = 'a'*32 -p1a = 'a'*32 -p2 = 'a'*31+'b' # close, but no cigar -p3 = 'b'*32 # different in the first byte - -def compare(n, f, a, b): - for i in xrange(n): - f(a, b) - -def eqeq(a, b): - return a == b - -def equalsequals_s(n): - # return compare(n, eqeq, - for i in xrange(n): - p1 == p1a - -def equalsequals_c(n): - for i in xrange(n): - p1 == p2 - -def equalsequals_f(n): - for i in xrange(n): - p1 == p3 - -def hash_s(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p1a).digest() - -def hash_c(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p2).digest() - -def hash_f(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p3).digest() - -N=10**4 -REPS=10**2 - -print "using '=='" - -print "same" -benchutil.rep_bench(equalsequals_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "close" -benchutil.rep_bench(equalsequals_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "far" -benchutil.rep_bench(equalsequals_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "using hash" - -print "same" -benchutil.rep_bench(hash_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "far" -benchutil.rep_bench(hash_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "close" -benchutil.rep_bench(hash_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index 0cbfa7c0..a2cef356 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -549,7 +549,7 @@ class OAuth2Mixin(object): @return_future def authorize_redirect(self, redirect_uri=None, client_id=None, client_secret=None, extra_params=None, - callback=None): + callback=None, scope=None, response_type="code"): """Redirects the user to obtain OAuth authorization for this service. Some providers require that you register a redirect URL with @@ -566,10 +566,13 @@ class OAuth2Mixin(object): """ args = { "redirect_uri": redirect_uri, - "client_id": client_id + "client_id": client_id, + "response_type": response_type } if extra_params: args.update(extra_params) + if scope: + args['scope'] = ' '.join(scope) self.redirect( url_concat(self._OAUTH_AUTHORIZE_URL, args)) callback() @@ -945,6 +948,67 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): return OpenIdMixin.get_authenticated_user(self) +class GoogleOAuth2Mixin(OAuth2Mixin): + """Google authentication using OAuth2.""" + _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth" + _OAUTH_ACCESS_TOKEN_URL = "https://accounts.google.com/o/oauth2/token" + _OAUTH_NO_CALLBACKS = False + _OAUTH_SETTINGS_KEY = 'google_oauth' + + @_auth_return_future + def get_authenticated_user(self, redirect_uri, code, callback): + """Handles the login for the Google user, returning a user object. + + Example usage:: + + class GoogleOAuth2LoginHandler(LoginHandler, tornado.auth.GoogleOAuth2Mixin): + @tornado.web.asynchronous + @tornado.gen.coroutine + def get(self): + if self.get_argument("code", False): + user = yield self.get_authenticated_user( + redirect_uri='http://your.site.com/auth/google', + code=self.get_argument("code")) + # Save the user with e.g. set_secure_cookie + else: + yield self.authorize_redirect( + redirect_uri='http://your.site.com/auth/google', + client_id=self.settings["google_consumer_key"], + scope=['openid', 'email'], + response_type='code', + extra_params={"approval_prompt": "auto"}) + """ + http = self.get_auth_http_client() + body = urllib_parse.urlencode({ + "redirect_uri": redirect_uri, + "code": code, + "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'], + "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'], + "grant_type": "authorization_code", + }) + + http.fetch(self._OAUTH_ACCESS_TOKEN_URL, + self.async_callback(self._on_access_token, callback), + method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body) + + def _on_access_token(self, future, response): + """Callback function for the exchange to the access token.""" + if response.error: + future.set_exception(AuthError('Google auth error: %s' % str(response))) + return + + args = escape.json_decode(response.body) + future.set_result(args) + + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() + + class FacebookMixin(object): """Facebook Connect authentication. diff --git a/libs/tornado/autoreload.py b/libs/tornado/autoreload.py index 05754299..79cccb49 100755 --- a/libs/tornado/autoreload.py +++ b/libs/tornado/autoreload.py @@ -16,11 +16,15 @@ """xAutomatically restart the server when a source file is modified. -Most applications should not access this module directly. Instead, pass the -keyword argument ``debug=True`` to the `tornado.web.Application` constructor. -This will enable autoreload mode as well as checking for changes to templates -and static resources. Note that restarting is a destructive operation -and any requests in progress will be aborted when the process restarts. +Most applications should not access this module directly. Instead, +pass the keyword argument ``autoreload=True`` to the +`tornado.web.Application` constructor (or ``debug=True``, which +enables this setting and several others). This will enable autoreload +mode as well as checking for changes to templates and static +resources. Note that restarting is a destructive operation and any +requests in progress will be aborted when the process restarts. (If +you want to disable autoreload while using other debug-mode features, +pass both ``debug=True`` and ``autoreload=False``). This module can also be used as a command-line wrapper around scripts such as unit test runners. See the `main` method for details. @@ -38,6 +42,7 @@ Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) because it re-executes Python using ``sys.executable`` and ``sys.argv``. Additionally, modifying these variables will cause reloading to behave incorrectly. + """ from __future__ import absolute_import, division, print_function, with_statement diff --git a/libs/tornado/curl_httpclient.py b/libs/tornado/curl_httpclient.py index e0900569..cb97710a 100755 --- a/libs/tornado/curl_httpclient.py +++ b/libs/tornado/curl_httpclient.py @@ -360,6 +360,7 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') + curl.unsetopt(pycurl.PROXYUSERPWD) if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) @@ -382,6 +383,8 @@ def _curl_setup_request(curl, request, buffer, headers): # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) + else: + curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) # Set the request method through curl's irritating interface which makes # up names for almost every single method @@ -404,6 +407,11 @@ def _curl_setup_request(curl, request, buffer, headers): # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): + if request.body is None: + raise AssertionError( + 'Body must not be empty for "%s" request' + % request.method) + request_buffer = BytesIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": @@ -414,6 +422,9 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) + elif request.method == "GET": + if request.body is not None: + raise AssertionError('Body must be empty for GET request') if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') diff --git a/libs/tornado/gen.py b/libs/tornado/gen.py index 92b7458e..217ebdf5 100755 --- a/libs/tornado/gen.py +++ b/libs/tornado/gen.py @@ -38,8 +38,8 @@ since it is both shorter and provides better exception handling):: def get(self): yield gen.Task(AsyncHTTPClient().fetch, "http://example.com") -You can also yield a list of ``Futures`` and/or ``Tasks``, which will be -started at the same time and run in parallel; a list of results will +You can also yield a list or dict of ``Futures`` and/or ``Tasks``, which will be +started at the same time and run in parallel; a list or dict of results will be returned when they are all finished:: @gen.coroutine @@ -47,6 +47,13 @@ be returned when they are all finished:: http_client = AsyncHTTPClient() response1, response2 = yield [http_client.fetch(url1), http_client.fetch(url2)] + response_dict = yield dict(response3=http_client.fetch(url3), + response4=http_client.fetch(url4)) + response3 = response_dict['response3'] + response4 = response_dict['response4'] + +.. versionchanged:: 3.2 + Dict support added. For more complicated interfaces, `Task` can be split into two parts: `Callback` and `Wait`:: @@ -404,6 +411,10 @@ class Multi(YieldPoint): a list of ``YieldPoints``. """ def __init__(self, children): + self.keys = None + if isinstance(children, dict): + self.keys = list(children.keys()) + children = children.values() self.children = [] for i in children: if isinstance(i, Future): @@ -423,7 +434,11 @@ class Multi(YieldPoint): return not self.unfinished_children def get_result(self): - return [i.get_result() for i in self.children] + result = (i.get_result() for i in self.children) + if self.keys is not None: + return dict(zip(self.keys, result)) + else: + return list(result) class _NullYieldPoint(YieldPoint): @@ -523,7 +538,7 @@ class Runner(object): self.finished = True self.yield_point = _null_yield_point raise - if isinstance(yielded, list): + if isinstance(yielded, (list, dict)): yielded = Multi(yielded) elif isinstance(yielded, Future): yielded = YieldFuture(yielded) diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index 67675894..b58a8348 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -282,7 +282,8 @@ class HTTPRequest(object): :arg int max_redirects: Limit for ``follow_redirects`` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server - :arg string network_interface: Network interface to use for request + :arg string network_interface: Network interface to use for request. + ``curl_httpclient`` only; see note below. :arg callable streaming_callback: If set, ``streaming_callback`` will be run with each chunk of data as it is received, and ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in @@ -310,14 +311,26 @@ class HTTPRequest(object): :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, - or None to use defaults. Note that in ``curl_httpclient``, if - any request uses a custom ``ca_certs`` file, they all must (they - don't have to all use the same ``ca_certs``, but it's not possible - to mix requests with ``ca_certs`` and requests that use the defaults. + or None to use defaults. See note below when used with + ``curl_httpclient``. :arg bool allow_ipv6: Use IPv6 when available? Default is false in ``simple_httpclient`` and true in ``curl_httpclient`` - :arg string client_key: Filename for client SSL key, if any - :arg string client_cert: Filename for client SSL certificate, if any + :arg string client_key: Filename for client SSL key, if any. See + note below when used with ``curl_httpclient``. + :arg string client_cert: Filename for client SSL certificate, if any. + See note below when used with ``curl_httpclient``. + + .. note:: + + When using ``curl_httpclient`` certain options may be + inherited by subsequent fetches because ``pycurl`` does + not allow them to be cleanly reset. This applies to the + ``ca_certs``, ``client_key``, ``client_cert``, and + ``network_interface`` arguments. If you use these + options, you should pass them on every request (you don't + have to always use the same values, but it's not possible + to mix requests that specify these options with ones that + use the defaults). .. versionadded:: 3.1 The ``auth_mode`` argument. @@ -372,6 +385,9 @@ class HTTPResponse(object): * headers: `tornado.httputil.HTTPHeaders` object + * effective_url: final location of the resource after following any + redirects + * buffer: ``cStringIO`` object for response body * body: response body as string (created on demand from ``self.buffer``) diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index d005545e..34e7b768 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -29,6 +29,7 @@ from __future__ import absolute_import, division, print_function, with_statement import socket import ssl import time +import copy from tornado.escape import native_str, parse_qs_bytes from tornado import httputil @@ -326,8 +327,8 @@ class HTTPConnection(object): self.request_callback(self._request) except _BadRequestException as e: - gen_log.info("Malformed HTTP request from %s: %s", - self.address[0], e) + gen_log.info("Malformed HTTP request from %r: %s", + self.address, e) self.close() return @@ -336,7 +337,10 @@ class HTTPConnection(object): if self._request.method in ("POST", "PATCH", "PUT"): httputil.parse_body_arguments( self._request.headers.get("Content-Type", ""), data, - self._request.arguments, self._request.files) + self._request.body_arguments, self._request.files) + + for k, v in self._request.body_arguments.items(): + self._request.arguments.setdefault(k, []).extend(v) self.request_callback(self._request) @@ -403,6 +407,20 @@ class HTTPRequest(object): `.RequestHandler.get_argument`, which returns argument values as unicode strings. + .. attribute:: query_arguments + + Same format as ``arguments``, but contains only arguments extracted + from the query string. + + .. versionadded:: 3.2 + + .. attribute:: body_arguments + + Same format as ``arguments``, but contains only arguments extracted + from the request body. + + .. versionadded:: 3.2 + .. attribute:: files File uploads are available in the files property, which maps file @@ -457,6 +475,8 @@ class HTTPRequest(object): self.path, sep, self.query = uri.partition('?') self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) + self.query_arguments = copy.deepcopy(self.arguments) + self.body_arguments = {} def supports_http_1_1(self): """Returns True if this request supports HTTP/1.1 semantics""" diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index 3e7337d9..2575bc56 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -320,7 +320,11 @@ def parse_body_arguments(content_type, body, arguments, files): with the parsed contents. """ if content_type.startswith("application/x-www-form-urlencoded"): - uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + try: + uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + except Exception as e: + gen_log.warning('Invalid x-www-form-urlencoded body: %s', e) + uri_arguments = {} for name, values in uri_arguments.items(): if values: arguments.setdefault(name, []).extend(values) diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 91ee2c5b..a36ab7a5 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -676,8 +676,7 @@ class PollIOLoop(IOLoop): while self._events: fd, events = self._events.popitem() try: - if self._handlers.has_key(fd): - self._handlers[fd](fd, events) + self._handlers[fd](fd, events) except (OSError, IOError) as e: if e.args[0] == errno.EPIPE: # Happens when the client closes the connection diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index 6bdc6397..08430cea 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -774,7 +774,7 @@ class IOStream(BaseIOStream): # Sometimes setsockopt will fail if the socket is closed # at the wrong time. This can happen with HTTPServer # resetting the value to false between requests. - if e.errno != errno.EINVAL: + if e.errno not in (errno.EINVAL, errno.ECONNRESET): raise diff --git a/libs/tornado/log.py b/libs/tornado/log.py index fa11f379..648db5c6 100755 --- a/libs/tornado/log.py +++ b/libs/tornado/log.py @@ -51,7 +51,7 @@ gen_log = logging.getLogger("tornado.general") def _stderr_supports_color(): color = False - if curses and sys.stderr.isatty(): + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): try: curses.setupterm() if curses.tigetnum("colors") > 0: diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index 9dc8506e..21db4755 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -20,7 +20,6 @@ from __future__ import absolute_import, division, print_function, with_statement import errno import os -import re import socket import ssl import stat @@ -30,6 +29,13 @@ from tornado.ioloop import IOLoop from tornado.platform.auto import set_close_exec from tornado.util import Configurable +if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ + ssl_match_hostname = ssl.match_hostname + SSLCertificateError = ssl.CertificateError +else: + import backports.ssl_match_hostname + ssl_match_hostname = backports.ssl_match_hostname.match_hostname + SSLCertificateError = backports.ssl_match_hostname.CertificateError def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): """Creates listening sockets bound to the given port and address. @@ -391,73 +397,3 @@ def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): return context.wrap_socket(socket, **kwargs) else: return ssl.wrap_socket(socket, **dict(context, **kwargs)) - -if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ - ssl_match_hostname = ssl.match_hostname - SSLCertificateError = ssl.CertificateError -else: - # match_hostname was added to the standard library ssl module in python 3.2. - # The following code was backported for older releases and copied from - # https://bitbucket.org/brandon/backports.ssl_match_hostname - class SSLCertificateError(ValueError): - pass - - def _dnsname_to_pat(dn, max_wildcards=1): - pats = [] - for frag in dn.split(r'.'): - if frag.count('*') > max_wildcards: - # Issue #17980: avoid denials of service by refusing more - # than one wildcard per fragment. A survery of established - # policy among SSL implementations showed it to be a - # reasonable choice. - raise SSLCertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) - if frag == '*': - # When '*' is a fragment by itself, it matches a non-empty dotless - # fragment. - pats.append('[^.]+') - else: - # Otherwise, '*' matches any dotless fragment. - frag = re.escape(frag) - pats.append(frag.replace(r'\*', '[^.]*')) - return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - - def ssl_match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules - are mostly followed, but IP addresses are not accepted for *hostname*. - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise SSLCertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise SSLCertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise SSLCertificateError("no appropriate commonName or " - "subjectAltName fields were found") diff --git a/libs/tornado/platform/asyncio.py b/libs/tornado/platform/asyncio.py new file mode 100644 index 00000000..a8f5bad4 --- /dev/null +++ b/libs/tornado/platform/asyncio.py @@ -0,0 +1,134 @@ +"""Bridges between the `asyncio` module and Tornado IOLoop. + +This is a work in progress and interfaces are subject to change. + +To test: +python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOLoop +python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOMainLoop +(the tests log a few warnings with AsyncIOMainLoop because they leave some +unfinished callbacks on the event loop that fail when it resumes) +""" +import asyncio +import datetime +import functools +import os + +from tornado.ioloop import IOLoop +from tornado import stack_context + +class BaseAsyncIOLoop(IOLoop): + def initialize(self, asyncio_loop, close_loop=False): + self.asyncio_loop = asyncio_loop + self.close_loop = close_loop + self.asyncio_loop.call_soon(self.make_current) + # Maps fd to handler function (as in IOLoop.add_handler) + self.handlers = {} + # Set of fds listening for reads/writes + self.readers = set() + self.writers = set() + self.closing = False + + def close(self, all_fds=False): + self.closing = True + for fd in list(self.handlers): + self.remove_handler(fd) + if all_fds: + os.close(fd) + if self.close_loop: + self.asyncio_loop.close() + + def add_handler(self, fd, handler, events): + if fd in self.handlers: + raise ValueError("fd %d added twice" % fd) + self.handlers[fd] = stack_context.wrap(handler) + if events & IOLoop.READ: + self.asyncio_loop.add_reader( + fd, self._handle_events, fd, IOLoop.READ) + self.readers.add(fd) + if events & IOLoop.WRITE: + self.asyncio_loop.add_writer( + fd, self._handle_events, fd, IOLoop.WRITE) + self.writers.add(fd) + + def update_handler(self, fd, events): + if events & IOLoop.READ: + if fd not in self.readers: + self.asyncio_loop.add_reader( + fd, self._handle_events, fd, IOLoop.READ) + self.readers.add(fd) + else: + if fd in self.readers: + self.asyncio_loop.remove_reader(fd) + self.readers.remove(fd) + if events & IOLoop.WRITE: + if fd not in self.writers: + self.asyncio_loop.add_writer( + fd, self._handle_events, fd, IOLoop.WRITE) + self.writers.add(fd) + else: + if fd in self.writers: + self.asyncio_loop.remove_writer(fd) + self.writers.remove(fd) + + def remove_handler(self, fd): + if fd not in self.handlers: + return + if fd in self.readers: + self.asyncio_loop.remove_reader(fd) + self.readers.remove(fd) + if fd in self.writers: + self.asyncio_loop.remove_writer(fd) + self.writers.remove(fd) + del self.handlers[fd] + + def _handle_events(self, fd, events): + self.handlers[fd](fd, events) + + def start(self): + self.asyncio_loop.run_forever() + + def stop(self): + self.asyncio_loop.stop() + + def _run_callback(self, callback, *args, **kwargs): + try: + callback(*args, **kwargs) + except Exception: + self.handle_callback_exception(callback) + + def add_timeout(self, deadline, callback): + if isinstance(deadline, (int, float)): + delay = max(deadline - self.time(), 0) + elif isinstance(deadline, datetime.timedelta): + delay = deadline.total_seconds() + else: + raise TypeError("Unsupported deadline %r", deadline) + return self.asyncio_loop.call_later(delay, self._run_callback, + stack_context.wrap(callback)) + + def remove_timeout(self, timeout): + timeout.cancel() + + def add_callback(self, callback, *args, **kwargs): + if self.closing: + raise RuntimeError("IOLoop is closing") + if kwargs: + self.asyncio_loop.call_soon_threadsafe(functools.partial( + self._run_callback, stack_context.wrap(callback), + *args, **kwargs)) + else: + self.asyncio_loop.call_soon_threadsafe( + self._run_callback, stack_context.wrap(callback), *args) + + add_callback_from_signal = add_callback + + +class AsyncIOMainLoop(BaseAsyncIOLoop): + def initialize(self): + super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), + close_loop=False) + +class AsyncIOLoop(BaseAsyncIOLoop): + def initialize(self): + super(AsyncIOLoop, self).initialize(asyncio.new_event_loop(), + close_loop=True) diff --git a/libs/tornado/process.py b/libs/tornado/process.py index ffd2d29d..942c5c3f 100755 --- a/libs/tornado/process.py +++ b/libs/tornado/process.py @@ -92,7 +92,8 @@ def fork_processes(num_processes, max_restarts=100): between any server code. Note that multiple processes are not compatible with the autoreload - module (or the debug=True option to `tornado.web.Application`). + module (or the ``autoreload=True`` option to `tornado.web.Application` + which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``fork_processes``. diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index d8dbb271..2558ada8 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -72,6 +72,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.max_clients = max_clients self.queue = collections.deque() self.active = {} + self.waiting = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver @@ -89,7 +90,16 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.resolver.close() def fetch_impl(self, request, callback): - self.queue.append((request, callback)) + key = object() + self.queue.append((key, request, callback)) + if not len(self.active) < self.max_clients: + timeout_handle = self.io_loop.add_timeout( + self.io_loop.time() + min(request.connect_timeout, + request.request_timeout), + functools.partial(self._on_timeout, key)) + else: + timeout_handle = None + self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " @@ -99,8 +109,10 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: - request, callback = self.queue.popleft() - key = object() + key, request, callback = self.queue.popleft() + if key not in self.waiting: + continue + self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) @@ -113,6 +125,22 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): del self.active[key] self._process_queue() + def _remove_timeout(self, key): + if key in self.waiting: + request, callback, timeout_handle = self.waiting[key] + if timeout_handle is not None: + self.io_loop.remove_timeout(timeout_handle) + del self.waiting[key] + + def _on_timeout(self, key): + request, callback, timeout_handle = self.waiting[key] + self.queue.remove((key, request, callback)) + timeout_response = HTTPResponse( + request, 599, error=HTTPError(599, "Timeout"), + request_time=self.io_loop.time() - request.start_time) + self.io_loop.add_callback(callback, timeout_response) + del self.waiting[key] + class _HTTPConnection(object): _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) @@ -162,15 +190,18 @@ class _HTTPConnection(object): # so restrict to ipv4 by default. af = socket.AF_INET + timeout = min(self.request.connect_timeout, self.request.request_timeout) + if timeout: + self._timeout = self.io_loop.add_timeout( + self.start_time + timeout, + stack_context.wrap(self._on_timeout)) self.resolver.resolve(host, port, af, callback=self._on_resolve) def _on_resolve(self, addrinfo): + if self.final_callback is None: + # final_callback is cleared if we've hit our timeout + return self.stream = self._create_stream(addrinfo) - timeout = min(self.request.connect_timeout, self.request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - stack_context.wrap(self._on_timeout)) self.stream.set_close_callback(self._on_close) # ipv6 addresses are broken (in self.parsed.hostname) until # 2.7, here is correctly parsed value calculated in __init__ @@ -199,10 +230,10 @@ class _HTTPConnection(object): # the SSL_OP_NO_SSLv2, but that wasn't exposed to python # until 3.2. Python 2.7 adds the ciphers argument, which # can also be used to disable SSLv2. As a last resort - # on python 2.6, we set ssl_version to SSLv3. This is + # on python 2.6, we set ssl_version to TLSv1. This is # more narrow than we'd like since it also breaks - # compatibility with servers configured for TLSv1 only, - # but nearly all servers support SSLv3: + # compatibility with servers configured for SSLv3 only, + # but nearly all servers support both SSLv3 and TLSv1: # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html if sys.version_info >= (2, 7): ssl_options["ciphers"] = "DEFAULT:!SSLv2" @@ -210,7 +241,7 @@ class _HTTPConnection(object): # This is really only necessary for pre-1.0 versions # of openssl, but python 2.6 doesn't expose version # information. - ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 + ssl_options["ssl_version"] = ssl.PROTOCOL_TLSv1 return SSLIOStream(socket.socket(af), io_loop=self.io_loop, @@ -233,6 +264,8 @@ class _HTTPConnection(object): def _on_connect(self): self._remove_timeout() + if self.final_callback is None: + return if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, @@ -269,9 +302,15 @@ class _HTTPConnection(object): self.request.headers["User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: if self.request.method in ("POST", "PATCH", "PUT"): - assert self.request.body is not None + if self.request.body is None: + raise AssertionError( + 'Body must not be empty for "%s" request' + % self.request.method) else: - assert self.request.body is None + if self.request.body is not None: + raise AssertionError( + 'Body must be empty for "%s" request' + % self.request.method) if self.request.body is not None: self.request.headers["Content-Length"] = str(len( self.request.body)) diff --git a/libs/tornado/speedups.c b/libs/tornado/speedups.c new file mode 100644 index 00000000..8a316c58 --- /dev/null +++ b/libs/tornado/speedups.c @@ -0,0 +1,49 @@ +#include <Python.h> + +static PyObject* websocket_mask(PyObject* self, PyObject* args) { + const char* mask; + int mask_len; + const char* data; + int data_len; + int i; + + if (!PyArg_ParseTuple(args, "s#s#", &mask, &mask_len, &data, &data_len)) { + return NULL; + } + + PyObject* result = PyBytes_FromStringAndSize(NULL, data_len); + if (!result) { + return NULL; + } + char* buf = PyBytes_AsString(result); + for (i = 0; i < data_len; i++) { + buf[i] = data[i] ^ mask[i % 4]; + } + + return result; +} + +static PyMethodDef methods[] = { + {"websocket_mask", websocket_mask, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef speedupsmodule = { + PyModuleDef_HEAD_INIT, + "speedups", + NULL, + -1, + methods +}; + +PyMODINIT_FUNC +PyInit_speedups() { + return PyModule_Create(&speedupsmodule); +} +#else // Python 2.x +PyMODINIT_FUNC +initspeedups() { + Py_InitModule("tornado.speedups", methods); +} +#endif diff --git a/libs/tornado/tcpserver.py b/libs/tornado/tcpserver.py index 8473a21a..c0773732 100755 --- a/libs/tornado/tcpserver.py +++ b/libs/tornado/tcpserver.py @@ -180,7 +180,8 @@ class TCPServer(object): between any server code. Note that multiple processes are not compatible with the autoreload - module (or the ``debug=True`` option to `tornado.web.Application`). + module (or the ``autoreload=True`` option to `tornado.web.Application` + which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``TCPServer.start(n)``. """ diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 5f8d6091..b6d7e97e 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -250,7 +250,7 @@ class RequestHandler(object): not self.request.connection.no_keep_alive): conn_header = self.request.headers.get("Connection") if conn_header and (conn_header.lower() == "keep-alive"): - self.set_header("Connection", "Keep-Alive") + self._headers["Connection"] = "Keep-Alive" self._write_buffer = [] self._status_code = 200 self._reason = httputil.responses[200] @@ -348,12 +348,7 @@ class RequestHandler(object): The returned value is always unicode. """ - args = self.get_arguments(name, strip=strip) - if not args: - if default is self._ARG_DEFAULT: - raise MissingArgumentError(name) - return default - return args[-1] + return self._get_argument(name, default, self.request.arguments, strip) def get_arguments(self, name, strip=True): """Returns a list of the arguments with the given name. @@ -362,9 +357,73 @@ class RequestHandler(object): The returned values are always unicode. """ + return self._get_arguments(name, self.request.arguments, strip) + def get_body_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name + from the request body. + + If default is not provided, the argument is considered to be + required, and we raise a `MissingArgumentError` if it is missing. + + If the argument appears in the url more than once, we return the + last value. + + The returned value is always unicode. + + .. versionadded:: 3.2 + """ + return self._get_argument(name, default, self.request.body_arguments, strip) + + def get_body_arguments(self, name, strip=True): + """Returns a list of the body arguments with the given name. + + If the argument is not present, returns an empty list. + + The returned values are always unicode. + + .. versionadded:: 3.2 + """ + return self._get_arguments(name, self.request.body_arguments, strip) + + def get_query_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name + from the request query string. + + If default is not provided, the argument is considered to be + required, and we raise a `MissingArgumentError` if it is missing. + + If the argument appears in the url more than once, we return the + last value. + + The returned value is always unicode. + + .. versionadded:: 3.2 + """ + return self._get_argument(name, default, self.request.query_arguments, strip) + + def get_query_arguments(self, name, strip=True): + """Returns a list of the query arguments with the given name. + + If the argument is not present, returns an empty list. + + The returned values are always unicode. + + .. versionadded:: 3.2 + """ + return self._get_arguments(name, self.request.query_arguments, strip) + + def _get_argument(self, name, default, source, strip=True): + args = self._get_arguments(name, source, strip=strip) + if not args: + if default is self._ARG_DEFAULT: + raise MissingArgumentError(name) + return default + return args[-1] + + def _get_arguments(self, name, source, strip=True): values = [] - for v in self.request.arguments.get(name, []): + for v in source.get(name, []): v = self.decode_argument(v, name=name) if isinstance(v, unicode_type): # Get rid of any weird control chars (unless decoding gave @@ -838,7 +897,7 @@ class RequestHandler(object): else: self.finish(self.get_error_html(status_code, **kwargs)) return - if self.settings.get("debug") and "exc_info" in kwargs: + if self.settings.get("serve_traceback") and "exc_info" in kwargs: # in debug mode, try to send a traceback self.set_header('Content-Type', 'text/plain') for line in traceback.format_exception(*kwargs["exc_info"]): @@ -1318,6 +1377,12 @@ def asynchronous(method): if not self._finished: self.finish() IOLoop.current().add_future(result, future_complete) + # Once we have done this, hide the Future from our + # caller (i.e. RequestHandler._when_complete), which + # would otherwise set up its own callback and + # exception handler (resulting in exceptions being + # logged twice). + return None return result return wrapper @@ -1383,10 +1448,16 @@ class Application(object): or (regexp, request_class) tuples. When we receive requests, we iterate over the list in order and instantiate an instance of the first request class whose regexp matches the request path. + The request class can be specified as either a class object or a + (fully-qualified) name. - Each tuple can contain an optional third element, which should be - a dictionary if it is present. That dictionary is passed as - keyword arguments to the contructor of the handler. This pattern + Each tuple can contain additional elements, which correspond to the + arguments to the `URLSpec` constructor. (Prior to Tornado 3.2, this + only tuples of two or three elements were allowed). + + A dictionary may be passed as the third element of the tuple, + which will be used as keyword arguments to the handler's + constructor and `~RequestHandler.initialize` method. This pattern is used for the `StaticFileHandler` in this example (note that a `StaticFileHandler` can be installed automatically with the static_path setting described below):: @@ -1409,6 +1480,7 @@ class Application(object): and ``/robots.txt`` from the same directory. A custom subclass of `StaticFileHandler` can be specified with the ``static_handler_class`` setting. + """ def __init__(self, handlers=None, default_host="", transforms=None, wsgi=False, **settings): @@ -1447,8 +1519,14 @@ class Application(object): if handlers: self.add_handlers(".*$", handlers) + if self.settings.get('debug'): + self.settings.setdefault('autoreload', True) + self.settings.setdefault('compiled_template_cache', False) + self.settings.setdefault('static_hash_cache', False) + self.settings.setdefault('serve_traceback', True) + # Automatically reload modified modules - if self.settings.get("debug") and not wsgi: + if self.settings.get('autoreload') and not wsgi: from tornado import autoreload autoreload.start() @@ -1493,20 +1571,8 @@ class Application(object): for spec in host_handlers: if isinstance(spec, (tuple, list)): - assert len(spec) in (2, 3) - pattern = spec[0] - handler = spec[1] - - if isinstance(handler, str): - # import the Module and instantiate the class - # Must be a fully qualified name (module.ClassName) - handler = import_object(handler) - - if len(spec) == 3: - kwargs = spec[2] - else: - kwargs = {} - spec = URLSpec(pattern, handler, kwargs) + assert len(spec) in (2, 3, 4) + spec = URLSpec(*spec) handlers.append(spec) if spec.name: if spec.name in self.named_handlers: @@ -1597,14 +1663,23 @@ class Application(object): args = [unquote(s) for s in match.groups()] break if not handler: - handler = ErrorHandler(self, request, status_code=404) + if self.settings.get('default_handler_class'): + handler_class = self.settings['default_handler_class'] + handler_args = self.settings.get( + 'default_handler_args', {}) + else: + handler_class = ErrorHandler + handler_args = dict(status_code=404) + handler = handler_class(self, request, **handler_args) - # In debug mode, re-compile templates and reload static files on every + # If template cache is disabled (usually in the debug mode), + # re-compile templates and reload static files on every # request so you don't need to restart to see changes - if self.settings.get("debug"): + if not self.settings.get("compiled_template_cache", True): with RequestHandler._template_loader_lock: for loader in RequestHandler._template_loaders.values(): loader.reset() + if not self.settings.get('static_hash_cache', True): StaticFileHandler.reset() handler._execute(transforms, *args, **kwargs) @@ -2454,7 +2529,7 @@ class _UIModuleNamespace(object): class URLSpec(object): """Specifies mappings between URLs and handlers.""" - def __init__(self, pattern, handler_class, kwargs=None, name=None): + def __init__(self, pattern, handler, kwargs=None, name=None): """Parameters: * ``pattern``: Regular expression to be matched. Any groups @@ -2475,7 +2550,13 @@ class URLSpec(object): assert len(self.regex.groupindex) in (0, self.regex.groups), \ ("groups in url regexes must either be all named or all " "positional: %r" % self.regex.pattern) - self.handler_class = handler_class + + if isinstance(handler, str): + # import the Module and instantiate the class + # Must be a fully qualified name (module.ClassName) + handler = import_object(handler) + + self.handler_class = handler self.kwargs = kwargs or {} self.name = name self._path, self._group_count = self._find_groups() diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index 676d21bf..8c2f5a64 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -33,7 +33,7 @@ import tornado.web from tornado.concurrent import TracebackFuture from tornado.escape import utf8, native_str -from tornado import httpclient +from tornado import httpclient, httputil from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from tornado.log import gen_log, app_log @@ -52,6 +52,10 @@ class WebSocketError(Exception): class WebSocketClosedError(WebSocketError): + """Raised by operations on a closed connection. + + .. versionadded:: 3.2 + """ pass @@ -163,6 +167,12 @@ class WebSocketHandler(tornado.web.RequestHandler): encoded as json). If the ``binary`` argument is false, the message will be sent as utf8; in binary mode any byte string is allowed. + + If the connection is already closed, raises `WebSocketClosedError`. + + .. versionchanged:: 3.2 + `WebSocketClosedError` was added (previously a closed connection + would raise an `AttributeError`) """ if self.ws_connection is None: raise WebSocketClosedError() @@ -586,7 +596,7 @@ class WebSocketProtocol13(WebSocketProtocol): frame += struct.pack("!BQ", 127 | mask_bit, l) if self.mask_outgoing: mask = os.urandom(4) - data = mask + self._apply_mask(mask, data) + data = mask + _websocket_mask(mask, data) frame += data self.stream.write(frame) @@ -671,21 +681,8 @@ class WebSocketProtocol13(WebSocketProtocol): except StreamClosedError: self._abort() - def _apply_mask(self, mask, data): - mask = array.array("B", mask) - unmasked = array.array("B", data) - for i in xrange(len(data)): - unmasked[i] = unmasked[i] ^ mask[i % 4] - if hasattr(unmasked, 'tobytes'): - # tostring was deprecated in py32. It hasn't been removed, - # but since we turn on deprecation warnings in our tests - # we need to use the right one. - return unmasked.tobytes() - else: - return unmasked.tostring() - def _on_masked_frame_data(self, data): - self._on_frame_data(self._apply_mask(self._frame_mask, data)) + self._on_frame_data(_websocket_mask(self._frame_mask, data)) def _on_frame_data(self, data): if self._frame_opcode_is_control: @@ -771,7 +768,11 @@ class WebSocketProtocol13(WebSocketProtocol): class WebSocketClientConnection(simple_httpclient._HTTPConnection): - """WebSocket client connection.""" + """WebSocket client connection. + + This class should not be instantiated directly; use the + `websocket_connect` function instead. + """ def __init__(self, io_loop, request): self.connect_future = TracebackFuture() self.read_future = None @@ -793,9 +794,19 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.resolver) + def close(self): + """Closes the websocket connection. + + .. versionadded:: 3.2 + """ + if self.protocol is not None: + self.protocol.close() + self.protocol = None + def _on_close(self): self.on_message(None) self.resolver.close() + super(WebSocketClientConnection, self)._on_close() def _on_http_response(self, response): if not self.connect_future.done(): @@ -859,13 +870,54 @@ def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None): Takes a url and returns a Future whose result is a `WebSocketClientConnection`. + + .. versionchanged:: 3.2 + Also accepts ``HTTPRequest`` objects in place of urls. """ if io_loop is None: io_loop = IOLoop.current() - request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) + if isinstance(url, httpclient.HTTPRequest): + assert connect_timeout is None + request = url + # Copy and convert the headers dict/object (see comments in + # AsyncHTTPClient.fetch) + request.headers = httputil.HTTPHeaders(request.headers) + else: + request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) request = httpclient._RequestProxy( request, httpclient.HTTPRequest._DEFAULTS) conn = WebSocketClientConnection(io_loop, request) if callback is not None: io_loop.add_future(conn.connect_future, callback) return conn.connect_future + +def _websocket_mask_python(mask, data): + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length. + Returns a `bytes` object of the same length as `data` with the mask applied + as specified in section 5.3 of RFC 6455. + + This pure-python implementation may be replaced by an optimized version when available. + """ + mask = array.array("B", mask) + unmasked = array.array("B", data) + for i in xrange(len(data)): + unmasked[i] = unmasked[i] ^ mask[i % 4] + if hasattr(unmasked, 'tobytes'): + # tostring was deprecated in py32. It hasn't been removed, + # but since we turn on deprecation warnings in our tests + # we need to use the right one. + return unmasked.tobytes() + else: + return unmasked.tostring() + +if os.environ.get('TORNADO_NO_EXTENSION'): + # This environment variable exists to make it easier to do performance comparisons; + # it's not guaranteed to remain supported in the future. + _websocket_mask = _websocket_mask_python +else: + try: + from tornado.speedups import websocket_mask as _websocket_mask + except ImportError: + _websocket_mask = _websocket_mask_python diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index 5e25a564..8e5ddedb 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -33,6 +33,7 @@ from __future__ import absolute_import, division, print_function, with_statement import sys import time +import copy import tornado from tornado import escape @@ -142,11 +143,14 @@ class HTTPRequest(object): self.path += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", ""))) self.uri = self.path self.arguments = {} + self.query_arguments = {} + self.body_arguments = {} self.query = environ.get("QUERY_STRING", "") if self.query: self.uri += "?" + self.query self.arguments = parse_qs_bytes(native_str(self.query), keep_blank_values=True) + self.query_arguments = copy.deepcopy(self.arguments) self.version = "HTTP/1.1" self.headers = httputil.HTTPHeaders() if environ.get("CONTENT_TYPE"): @@ -171,7 +175,10 @@ class HTTPRequest(object): # Parse request body self.files = {} httputil.parse_body_arguments(self.headers.get("Content-Type", ""), - self.body, self.arguments, self.files) + self.body, self.body_arguments, self.files) + + for k, v in self.body_arguments.items(): + self.arguments.setdefault(k, []).extend(v) self._start_time = time.time() self._finish_time = None
, , and blocks) we @@ -1026,19 +977,19 @@ def getPhases(debug): self.processSpaceCharacters = self.processSpaceCharactersNonPre if (data.startswith("\n") and self.tree.openElements[-1].name in ("pre", "listing", "textarea") - and not self.tree.openElements[-1].hasContent()): + and not self.tree.openElements[-1].hasContent()): data = data[1:] if data: self.tree.reconstructActiveFormattingElements() self.tree.insertText(data) def processCharacters(self, token): - if token["data"] == u"\u0000": - #The tokenizer should always emit null on its own + if token["data"] == "\u0000": + # The tokenizer should always emit null on its own return self.tree.reconstructActiveFormattingElements() self.tree.insertText(token["data"]) - #This must be bad for performance + # This must be bad for performance if (self.parser.framesetOK and any([char not in spaceCharacters for char in token["data"]])): @@ -1054,11 +1005,11 @@ def getPhases(debug): def startTagBody(self, token): self.parser.parseError("unexpected-start-tag", {"name": "body"}) if (len(self.tree.openElements) == 1 - or self.tree.openElements[1].name != "body"): + or self.tree.openElements[1].name != "body"): assert self.parser.innerHTML else: self.parser.framesetOK = False - for attr, value in token["data"].iteritems(): + for attr, value in token["data"].items(): if attr not in self.tree.openElements[1].attributes: self.tree.openElements[1].attributes[attr] = value @@ -1090,7 +1041,7 @@ def getPhases(debug): def startTagForm(self, token): if self.tree.formPointer: - self.parser.parseError(u"unexpected-start-tag", {"name": "form"}) + self.parser.parseError("unexpected-start-tag", {"name": "form"}) else: if self.tree.elementInScope("p", variant="button"): self.endTagP(impliedTagToken("p")) @@ -1100,9 +1051,9 @@ def getPhases(debug): def startTagListItem(self, token): self.parser.framesetOK = False - stopNamesMap = {"li":["li"], - "dt":["dt", "dd"], - "dd":["dt", "dd"]} + stopNamesMap = {"li": ["li"], + "dt": ["dt", "dd"], + "dd": ["dt", "dd"]} stopNames = stopNamesMap[token["name"]] for node in reversed(self.tree.openElements): if node.name in stopNames: @@ -1110,7 +1061,7 @@ def getPhases(debug): impliedTagToken(node.name, "EndTag")) break if (node.nameTuple in specialElements and - node.name not in ("address", "div", "p")): + node.name not in ("address", "div", "p")): break if self.tree.elementInScope("p", variant="button"): @@ -1137,7 +1088,7 @@ def getPhases(debug): afeAElement = self.tree.elementInActiveFormattingElements("a") if afeAElement: self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "a", "endName": "a"}) + {"startName": "a", "endName": "a"}) self.endTagFormatting(impliedTagToken("a")) if afeAElement in self.tree.openElements: self.tree.openElements.remove(afeAElement) @@ -1154,7 +1105,7 @@ def getPhases(debug): self.tree.reconstructActiveFormattingElements() if self.tree.elementInScope("nobr"): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "nobr", "endName": "nobr"}) + {"startName": "nobr", "endName": "nobr"}) self.processEndTag(impliedTagToken("nobr")) # XXX Need tests that trigger the following self.tree.reconstructActiveFormattingElements() @@ -1163,7 +1114,7 @@ def getPhases(debug): def startTagButton(self, token): if self.tree.elementInScope("button"): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "button", "endName": "button"}) + {"startName": "button", "endName": "button"}) self.processEndTag(impliedTagToken("button")) return token else: @@ -1203,8 +1154,8 @@ def getPhases(debug): framesetOK = self.parser.framesetOK self.startTagVoidFormatting(token) if ("type" in token["data"] and - token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): - #input type=hidden doesn't change framesetOK + token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): + # input type=hidden doesn't change framesetOK self.parser.framesetOK = framesetOK def startTagParamSource(self, token): @@ -1223,7 +1174,7 @@ def getPhases(debug): def startTagImage(self, token): # No really... self.parser.parseError("unexpected-start-tag-treated-as", - {"originalName": "image", "newName": "img"}) + {"originalName": "image", "newName": "img"}) self.processStartTag(impliedTagToken("img", "StartTag", attributes=token["data"], selfClosing=token["selfClosing"])) @@ -1243,18 +1194,18 @@ def getPhases(debug): if "prompt" in token["data"]: prompt = token["data"]["prompt"] else: - prompt = u"This is a searchable index. Enter search keywords: " + prompt = "This is a searchable index. Enter search keywords: " self.processCharacters( - {"type":tokenTypes["Characters"], "data":prompt}) + {"type": tokenTypes["Characters"], "data": prompt}) attributes = token["data"].copy() if "action" in attributes: del attributes["action"] if "prompt" in attributes: del attributes["prompt"] attributes["name"] = "isindex" - self.processStartTag(impliedTagToken("input", "StartTag", - attributes = attributes, - selfClosing = + self.processStartTag(impliedTagToken("input", "StartTag", + attributes=attributes, + selfClosing= token["selfClosing"])) self.processEndTag(impliedTagToken("label")) self.processStartTag(impliedTagToken("hr", "StartTag")) @@ -1287,7 +1238,7 @@ def getPhases(debug): if self.parser.phase in (self.parser.phases["inTable"], self.parser.phases["inCaption"], self.parser.phases["inColumnGroup"], - self.parser.phases["inTableBody"], + self.parser.phases["inTableBody"], self.parser.phases["inRow"], self.parser.phases["inCell"]): self.parser.phase = self.parser.phases["inSelectInTable"] @@ -1307,8 +1258,8 @@ def getPhases(debug): self.parser.adjustForeignAttributes(token) token["namespace"] = namespaces["mathml"] self.tree.insertElement(token) - #Need to get the parse error right for the case where the token - #has a namespace not equal to the xmlns attribute + # Need to get the parse error right for the case where the token + # has a namespace not equal to the xmlns attribute if token["selfClosing"]: self.tree.openElements.pop() token["selfClosingAcknowledged"] = True @@ -1319,8 +1270,8 @@ def getPhases(debug): self.parser.adjustForeignAttributes(token) token["namespace"] = namespaces["svg"] self.tree.insertElement(token) - #Need to get the parse error right for the case where the token - #has a namespace not equal to the xmlns attribute + # Need to get the parse error right for the case where the token + # has a namespace not equal to the xmlns attribute if token["selfClosing"]: self.tree.openElements.pop() token["selfClosingAcknowledged"] = True @@ -1362,7 +1313,7 @@ def getPhases(debug): "tbody", "td", "tfoot", "th", "thead", "tr", "body", "html")): - #Not sure this is the correct name for the parse error + # Not sure this is the correct name for the parse error self.parser.parseError( "expected-one-end-tag-but-got-another", {"expectedName": "body", "gotName": node.name}) @@ -1370,20 +1321,20 @@ def getPhases(debug): self.parser.phase = self.parser.phases["afterBody"] def endTagHtml(self, token): - #We repeat the test for the body end tag token being ignored here + # We repeat the test for the body end tag token being ignored here if self.tree.elementInScope("body"): self.endTagBody(impliedTagToken("body")) return token def endTagBlock(self, token): - #Put us back in the right whitespace handling mode + # Put us back in the right whitespace handling mode if token["name"] == "pre": self.processSpaceCharacters = self.processSpaceCharactersNonPre inScope = self.tree.elementInScope(token["name"]) if inScope: self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != token["name"]: - self.parser.parseError("end-tag-too-early", {"name": token["name"]}) + self.parser.parseError("end-tag-too-early", {"name": token["name"]}) if inScope: node = self.tree.openElements.pop() while node.name != token["name"]: @@ -1394,7 +1345,7 @@ def getPhases(debug): self.tree.formPointer = None if node is None or not self.tree.elementInScope(node): self.parser.parseError("unexpected-end-tag", - {"name":"form"}) + {"name": "form"}) else: self.tree.generateImpliedEndTags() if self.tree.openElements[-1] != node: @@ -1410,7 +1361,7 @@ def getPhases(debug): if not self.tree.elementInScope(token["name"], variant=variant): self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) else: - self.tree.generateImpliedEndTags(exclude = token["name"]) + self.tree.generateImpliedEndTags(exclude=token["name"]) if self.tree.openElements[-1].name != token["name"]: self.parser.parseError( "end-tag-too-early", @@ -1436,65 +1387,105 @@ def getPhases(debug): def endTagFormatting(self, token): """The much-feared adoption agency algorithm""" - # http://www.whatwg.org/specs/web-apps/current-work/#adoptionAgency + # http://svn.whatwg.org/webapps/complete.html#adoptionAgency revision 7867 # XXX Better parseError messages appreciated. - name = token["name"] + # Step 1 outerLoopCounter = 0 + + # Step 2 while outerLoopCounter < 8: + + # Step 3 outerLoopCounter += 1 - # Step 1 paragraph 1 + # Step 4: + + # Let the formatting element be the last element in + # the list of active formatting elements that: + # - is between the end of the list and the last scope + # marker in the list, if any, or the start of the list + # otherwise, and + # - has the same tag name as the token. formattingElement = self.tree.elementInActiveFormattingElements( token["name"]) - if (not formattingElement or + if (not formattingElement or (formattingElement in self.tree.openElements and not self.tree.elementInScope(formattingElement.name))): - self.parser.parseError("adoption-agency-1.1", {"name": token["name"]}) + # If there is no such node, then abort these steps + # and instead act as described in the "any other + # end tag" entry below. + self.endTagOther(token) return - # Step 1 paragraph 2 + # Otherwise, if there is such a node, but that node is + # not in the stack of open elements, then this is a + # parse error; remove the element from the list, and + # abort these steps. elif formattingElement not in self.tree.openElements: self.parser.parseError("adoption-agency-1.2", {"name": token["name"]}) self.tree.activeFormattingElements.remove(formattingElement) return - # Step 1 paragraph 3 - if formattingElement != self.tree.openElements[-1]: - self.parser.parseError("adoption-agency-1.3", {"name": token["name"]}) + # Otherwise, if there is such a node, and that node is + # also in the stack of open elements, but the element + # is not in scope, then this is a parse error; ignore + # the token, and abort these steps. + elif not self.tree.elementInScope(formattingElement.name): + self.parser.parseError("adoption-agency-4.4", {"name": token["name"]}) + return - # Step 2 - # Start of the adoption agency algorithm proper + # Otherwise, there is a formatting element and that + # element is in the stack and is in scope. If the + # element is not the current node, this is a parse + # error. In any case, proceed with the algorithm as + # written in the following steps. + else: + if formattingElement != self.tree.openElements[-1]: + self.parser.parseError("adoption-agency-1.3", {"name": token["name"]}) + + # Step 5: + + # Let the furthest block be the topmost node in the + # stack of open elements that is lower in the stack + # than the formatting element, and is an element in + # the special category. There might not be one. afeIndex = self.tree.openElements.index(formattingElement) furthestBlock = None for element in self.tree.openElements[afeIndex:]: if element.nameTuple in specialElements: furthestBlock = element break - # Step 3 + + # Step 6: + + # If there is no furthest block, then the UA must + # first pop all the nodes from the bottom of the stack + # of open elements, from the current node up to and + # including the formatting element, then remove the + # formatting element from the list of active + # formatting elements, and finally abort these steps. if furthestBlock is None: element = self.tree.openElements.pop() while element != formattingElement: element = self.tree.openElements.pop() self.tree.activeFormattingElements.remove(element) return - commonAncestor = self.tree.openElements[afeIndex-1] - # Step 5 - #if furthestBlock.parent: - # furthestBlock.parent.removeChild(furthestBlock) + # Step 7 + commonAncestor = self.tree.openElements[afeIndex - 1] - # Step 5 + # Step 8: # The bookmark is supposed to help us identify where to reinsert - # nodes in step 12. We have to ensure that we reinsert nodes after + # nodes in step 15. We have to ensure that we reinsert nodes after # the node before the active formatting element. Note the bookmark - # can move in step 7.4 + # can move in step 9.7 bookmark = self.tree.activeFormattingElements.index(formattingElement) - # Step 6 + # Step 9 lastNode = node = furthestBlock innerLoopCounter = 0 - + index = self.tree.openElements.index(node) while innerLoopCounter < 3: innerLoopCounter += 1 @@ -1504,15 +1495,13 @@ def getPhases(debug): if node not in self.tree.activeFormattingElements: self.tree.openElements.remove(node) continue - # Step 6.3 + # Step 9.6 if node == formattingElement: break - # Step 6.4 + # Step 9.7 if lastNode == furthestBlock: - bookmark = (self.tree.activeFormattingElements.index(node) - + 1) - # Step 6.5 - #cite = node.parent + bookmark = self.tree.activeFormattingElements.index(node) + 1 + # Step 9.8 clone = node.cloneNode() # Replace node with clone self.tree.activeFormattingElements[ @@ -1520,20 +1509,18 @@ def getPhases(debug): self.tree.openElements[ self.tree.openElements.index(node)] = clone node = clone - - # Step 6.6 + # Step 9.9 # Remove lastNode from its parents, if any if lastNode.parent: lastNode.parent.removeChild(lastNode) node.appendChild(lastNode) - # Step 7.7 + # Step 9.10 lastNode = node - # End of inner loop - # Step 7 + # Step 10 # Foster parent lastNode if commonAncestor is a - # table, tbody, tfoot, thead, or tr we need to foster parent the - # lastNode + # table, tbody, tfoot, thead, or tr we need to foster + # parent the lastNode if lastNode.parent: lastNode.parent.removeChild(lastNode) @@ -1543,23 +1530,23 @@ def getPhases(debug): else: commonAncestor.appendChild(lastNode) - # Step 8 + # Step 11 clone = formattingElement.cloneNode() - # Step 9 + # Step 12 furthestBlock.reparentChildren(clone) - # Step 10 + # Step 13 furthestBlock.appendChild(clone) - # Step 11 + # Step 14 self.tree.activeFormattingElements.remove(formattingElement) self.tree.activeFormattingElements.insert(bookmark, clone) - # Step 12 + # Step 15 self.tree.openElements.remove(formattingElement) self.tree.openElements.insert( - self.tree.openElements.index(furthestBlock) + 1, clone) + self.tree.openElements.index(furthestBlock) + 1, clone) def endTagAppletMarqueeObject(self, token): if self.tree.elementInScope(token["name"]): @@ -1575,7 +1562,7 @@ def getPhases(debug): def endTagBr(self, token): self.parser.parseError("unexpected-end-tag-treated-as", - {"originalName": "br", "newName": "br element"}) + {"originalName": "br", "newName": "br element"}) self.tree.reconstructActiveFormattingElements() self.tree.insertElement(impliedTagToken("br", "StartTag")) self.tree.openElements.pop() @@ -1600,31 +1587,31 @@ def getPhases(debug): self.startTagHandler = utils.MethodDispatcher([]) self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([ - ("script", self.endTagScript)]) + ("script", self.endTagScript)]) self.endTagHandler.default = self.endTagOther def processCharacters(self, token): self.tree.insertText(token["data"]) def processEOF(self): - self.parser.parseError("expected-named-closing-tag-but-got-eof", - self.tree.openElements[-1].name) + self.parser.parseError("expected-named-closing-tag-but-got-eof", + {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() self.parser.phase = self.parser.originalPhase return True def startTagOther(self, token): - assert False, "Tried to process start tag %s in RCDATA/RAWTEXT mode"%token['name'] + assert False, "Tried to process start tag %s in RCDATA/RAWTEXT mode" % token['name'] def endTagScript(self, token): node = self.tree.openElements.pop() assert node.name == "script" self.parser.phase = self.parser.originalPhase - #The rest of this method is all stuff that only happens if - #document.write works + # The rest of this method is all stuff that only happens if + # document.write works def endTagOther(self, token): - node = self.tree.openElements.pop() + self.tree.openElements.pop() self.parser.phase = self.parser.originalPhase class InTablePhase(Phase): @@ -1656,7 +1643,7 @@ def getPhases(debug): def clearStackToTableContext(self): # "clear the stack back to a table context" while self.tree.openElements[-1].name not in ("table", "html"): - #self.parser.parseError("unexpected-implied-end-tag-in-table", + # self.parser.parseError("unexpected-implied-end-tag-in-table", # {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() # When the current node is <html> it's an innerHTML case @@ -1667,7 +1654,7 @@ def getPhases(debug): self.parser.parseError("eof-in-table") else: assert self.parser.innerHTML - #Stop parsing + # Stop parsing def processSpaceCharacters(self, token): originalPhase = self.parser.phase @@ -1682,7 +1669,7 @@ def getPhases(debug): self.parser.phase.processCharacters(token) def insertText(self, token): - #If we get here there must be at least one non-whitespace character + # If we get here there must be at least one non-whitespace character # Do the table magic! self.tree.insertFromTable = True self.parser.phases["inBody"].processCharacters(token) @@ -1714,7 +1701,7 @@ def getPhases(debug): def startTagTable(self, token): self.parser.parseError("unexpected-start-tag-implies-end-tag", - {"startName": "table", "endName": "table"}) + {"startName": "table", "endName": "table"}) self.parser.phase.processEndTag(impliedTagToken("table")) if not self.parser.innerHTML: return token @@ -1723,8 +1710,8 @@ def getPhases(debug): return self.parser.phases["inHead"].processStartTag(token) def startTagInput(self, token): - if ("type" in token["data"] and - token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): + if ("type" in token["data"] and + token["data"]["type"].translate(asciiUpper2Lower) == "hidden"): self.parser.parseError("unexpected-hidden-input-in-table") self.tree.insertElement(token) # XXX associate with form @@ -1751,8 +1738,8 @@ def getPhases(debug): self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != "table": self.parser.parseError("end-tag-too-early-named", - {"gotName": "table", - "expectedName": self.tree.openElements[-1].name}) + {"gotName": "table", + "expectedName": self.tree.openElements[-1].name}) while self.tree.openElements[-1].name != "table": self.tree.openElements.pop() self.tree.openElements.pop() @@ -1781,7 +1768,7 @@ def getPhases(debug): def flushCharacters(self): data = "".join([item["data"] for item in self.characterTokens]) if any([item not in spaceCharacters for item in data]): - token = {"type":tokenTypes["Characters"], "data":data} + token = {"type": tokenTypes["Characters"], "data": data} self.parser.phases["inTable"].insertText(token) elif data: self.tree.insertText(data) @@ -1798,12 +1785,12 @@ def getPhases(debug): return True def processCharacters(self, token): - if token["data"] == u"\u0000": + if token["data"] == "\u0000": return self.characterTokens.append(token) def processSpaceCharacters(self, token): - #pretty sure we should never reach here + # pretty sure we should never reach here self.characterTokens.append(token) # assert False @@ -1817,7 +1804,6 @@ def getPhases(debug): self.parser.phase = self.originalPhase return token - class InCaptionPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-caption def __init__(self, parser, tree): @@ -1849,7 +1835,7 @@ def getPhases(debug): def startTagTableElement(self, token): self.parser.parseError() - #XXX Have to duplicate logic here to find out if the tag is ignored + # XXX Have to duplicate logic here to find out if the tag is ignored ignoreEndTag = self.ignoreEndTagCaption() self.parser.phase.processEndTag(impliedTagToken("caption")) if not ignoreEndTag: @@ -1864,8 +1850,8 @@ def getPhases(debug): self.tree.generateImpliedEndTags() if self.tree.openElements[-1].name != "caption": self.parser.parseError("expected-one-end-tag-but-got-another", - {"gotName": "caption", - "expectedName": self.tree.openElements[-1].name}) + {"gotName": "caption", + "expectedName": self.tree.openElements[-1].name}) while self.tree.openElements[-1].name != "caption": self.tree.openElements.pop() self.tree.openElements.pop() @@ -1889,7 +1875,6 @@ def getPhases(debug): def endTagOther(self, token): return self.parser.phases["inBody"].processEndTag(token) - class InColumnGroupPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-column @@ -1955,7 +1940,6 @@ def getPhases(debug): if not ignoreEndTag: return token - class InTableBodyPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-table0 def __init__(self, parser, tree): @@ -1980,8 +1964,8 @@ def getPhases(debug): # helper methods def clearStackToTableBodyContext(self): while self.tree.openElements[-1].name not in ("tbody", "tfoot", - "thead", "html"): - #self.parser.parseError("unexpected-implied-end-tag-in-table", + "thead", "html"): + # self.parser.parseError("unexpected-implied-end-tag-in-table", # {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() if self.tree.openElements[-1].name == "html": @@ -2003,7 +1987,7 @@ def getPhases(debug): self.parser.phase = self.parser.phases["inRow"] def startTagTableCell(self, token): - self.parser.parseError("unexpected-cell-in-table-body", + self.parser.parseError("unexpected-cell-in-table-body", {"name": token["name"]}) self.startTagTr(impliedTagToken("tr", "StartTag")) return token @@ -2012,7 +1996,7 @@ def getPhases(debug): # XXX AT Any ideas on how to share this with endTagTable? if (self.tree.elementInScope("tbody", variant="table") or self.tree.elementInScope("thead", variant="table") or - self.tree.elementInScope("tfoot", variant="table")): + self.tree.elementInScope("tfoot", variant="table")): self.clearStackToTableBodyContext() self.endTagTableRowGroup( impliedTagToken(self.tree.openElements[-1].name)) @@ -2032,12 +2016,12 @@ def getPhases(debug): self.parser.phase = self.parser.phases["inTable"] else: self.parser.parseError("unexpected-end-tag-in-table-body", - {"name": token["name"]}) + {"name": token["name"]}) def endTagTable(self, token): if (self.tree.elementInScope("tbody", variant="table") or self.tree.elementInScope("thead", variant="table") or - self.tree.elementInScope("tfoot", variant="table")): + self.tree.elementInScope("tfoot", variant="table")): self.clearStackToTableBodyContext() self.endTagTableRowGroup( impliedTagToken(self.tree.openElements[-1].name)) @@ -2049,12 +2033,11 @@ def getPhases(debug): def endTagIgnore(self, token): self.parser.parseError("unexpected-end-tag-in-table-body", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOther(self, token): return self.parser.phases["inTable"].processEndTag(token) - class InRowPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#in-row def __init__(self, parser, tree): @@ -2072,7 +2055,7 @@ def getPhases(debug): ("table", self.endTagTable), (("tbody", "tfoot", "thead"), self.endTagTableRowGroup), (("body", "caption", "col", "colgroup", "html", "td", "th"), - self.endTagIgnore) + self.endTagIgnore) ]) self.endTagHandler.default = self.endTagOther @@ -2080,7 +2063,7 @@ def getPhases(debug): def clearStackToTableRowContext(self): while self.tree.openElements[-1].name not in ("tr", "html"): self.parser.parseError("unexpected-implied-end-tag-in-table-row", - {"name": self.tree.openElements[-1].name}) + {"name": self.tree.openElements[-1].name}) self.tree.openElements.pop() def ignoreEndTagTr(self): @@ -2091,7 +2074,7 @@ def getPhases(debug): self.parser.phases["inTable"].processEOF() def processSpaceCharacters(self, token): - return self.parser.phases["inTable"].processSpaceCharacters(token) + return self.parser.phases["inTable"].processSpaceCharacters(token) def processCharacters(self, token): return self.parser.phases["inTable"].processCharacters(token) @@ -2139,7 +2122,7 @@ def getPhases(debug): def endTagIgnore(self, token): self.parser.parseError("unexpected-end-tag-in-table-row", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOther(self, token): return self.parser.phases["inTable"].processEndTag(token) @@ -2178,7 +2161,7 @@ def getPhases(debug): def startTagTableOther(self, token): if (self.tree.elementInScope("td", variant="table") or - self.tree.elementInScope("th", variant="table")): + self.tree.elementInScope("th", variant="table")): self.closeCell() return token else: @@ -2194,7 +2177,7 @@ def getPhases(debug): self.tree.generateImpliedEndTags(token["name"]) if self.tree.openElements[-1].name != token["name"]: self.parser.parseError("unexpected-cell-end-tag", - {"name": token["name"]}) + {"name": token["name"]}) while True: node = self.tree.openElements.pop() if node.name == token["name"]: @@ -2249,7 +2232,7 @@ def getPhases(debug): assert self.parser.innerHTML def processCharacters(self, token): - if token["data"] == u"\u0000": + if token["data"] == "\u0000": return self.tree.insertText(token["data"]) @@ -2283,19 +2266,19 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-in-select", - {"name": token["name"]}) + {"name": token["name"]}) def endTagOption(self, token): if self.tree.openElements[-1].name == "option": self.tree.openElements.pop() else: self.parser.parseError("unexpected-end-tag-in-select", - {"name": "option"}) + {"name": "option"}) def endTagOptgroup(self, token): # </optgroup> implicitly closes <option> if (self.tree.openElements[-1].name == "option" and - self.tree.openElements[-2].name == "optgroup"): + self.tree.openElements[-2].name == "optgroup"): self.tree.openElements.pop() # It also closes </optgroup> if self.tree.openElements[-1].name == "optgroup": @@ -2303,7 +2286,7 @@ def getPhases(debug): # But nothing else else: self.parser.parseError("unexpected-end-tag-in-select", - {"name": "optgroup"}) + {"name": "optgroup"}) def endTagSelect(self, token): if self.tree.elementInScope("select", variant="select"): @@ -2318,8 +2301,7 @@ def getPhases(debug): def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-in-select", - {"name": token["name"]}) - + {"name": token["name"]}) class InSelectInTablePhase(Phase): def __init__(self, parser, tree): @@ -2360,64 +2342,64 @@ def getPhases(debug): def endTagOther(self, token): return self.parser.phases["inSelect"].processEndTag(token) - class InForeignContentPhase(Phase): - breakoutElements = frozenset(["b", "big", "blockquote", "body", "br", + breakoutElements = frozenset(["b", "big", "blockquote", "body", "br", "center", "code", "dd", "div", "dl", "dt", - "em", "embed", "h1", "h2", "h3", + "em", "embed", "h1", "h2", "h3", "h4", "h5", "h6", "head", "hr", "i", "img", - "li", "listing", "menu", "meta", "nobr", - "ol", "p", "pre", "ruby", "s", "small", - "span", "strong", "strike", "sub", "sup", + "li", "listing", "menu", "meta", "nobr", + "ol", "p", "pre", "ruby", "s", "small", + "span", "strong", "strike", "sub", "sup", "table", "tt", "u", "ul", "var"]) + def __init__(self, parser, tree): Phase.__init__(self, parser, tree) def adjustSVGTagNames(self, token): - replacements = {u"altglyph":u"altGlyph", - u"altglyphdef":u"altGlyphDef", - u"altglyphitem":u"altGlyphItem", - u"animatecolor":u"animateColor", - u"animatemotion":u"animateMotion", - u"animatetransform":u"animateTransform", - u"clippath":u"clipPath", - u"feblend":u"feBlend", - u"fecolormatrix":u"feColorMatrix", - u"fecomponenttransfer":u"feComponentTransfer", - u"fecomposite":u"feComposite", - u"feconvolvematrix":u"feConvolveMatrix", - u"fediffuselighting":u"feDiffuseLighting", - u"fedisplacementmap":u"feDisplacementMap", - u"fedistantlight":u"feDistantLight", - u"feflood":u"feFlood", - u"fefunca":u"feFuncA", - u"fefuncb":u"feFuncB", - u"fefuncg":u"feFuncG", - u"fefuncr":u"feFuncR", - u"fegaussianblur":u"feGaussianBlur", - u"feimage":u"feImage", - u"femerge":u"feMerge", - u"femergenode":u"feMergeNode", - u"femorphology":u"feMorphology", - u"feoffset":u"feOffset", - u"fepointlight":u"fePointLight", - u"fespecularlighting":u"feSpecularLighting", - u"fespotlight":u"feSpotLight", - u"fetile":u"feTile", - u"feturbulence":u"feTurbulence", - u"foreignobject":u"foreignObject", - u"glyphref":u"glyphRef", - u"lineargradient":u"linearGradient", - u"radialgradient":u"radialGradient", - u"textpath":u"textPath"} + replacements = {"altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath"} if token["name"] in replacements: token["name"] = replacements[token["name"]] def processCharacters(self, token): - if token["data"] == u"\u0000": - token["data"] = u"\uFFFD" - elif (self.parser.framesetOK and + if token["data"] == "\u0000": + token["data"] = "\uFFFD" + elif (self.parser.framesetOK and any(char not in spaceCharacters for char in token["data"])): self.parser.framesetOK = False Phase.processCharacters(self, token) @@ -2428,9 +2410,9 @@ def getPhases(debug): (token["name"] == "font" and set(token["data"].keys()) & set(["color", "face", "size"]))): self.parser.parseError("unexpected-html-element-in-foreign-content", - token["name"]) + {"name": token["name"]}) while (self.tree.openElements[-1].namespace != - self.tree.defaultNamespace and + self.tree.defaultNamespace and not self.parser.isHTMLIntegrationPoint(self.tree.openElements[-1]) and not self.parser.isMathMLTextIntegrationPoint(self.tree.openElements[-1])): self.tree.openElements.pop() @@ -2453,11 +2435,11 @@ def getPhases(debug): nodeIndex = len(self.tree.openElements) - 1 node = self.tree.openElements[-1] if node.name != token["name"]: - self.parser.parseError("unexpected-end-tag", token["name"]) + self.parser.parseError("unexpected-end-tag", {"name": token["name"]}) while True: if node.name.translate(asciiUpper2Lower) == token["name"]: - #XXX this isn't in the spec but it seems necessary + # XXX this isn't in the spec but it seems necessary if self.parser.phase == self.parser.phases["inTableText"]: self.parser.phase.flushCharacters() self.parser.phase = self.parser.phase.originalPhase @@ -2475,21 +2457,20 @@ def getPhases(debug): break return new_token - class AfterBodyPhase(Phase): def __init__(self, parser, tree): Phase.__init__(self, parser, tree) self.startTagHandler = utils.MethodDispatcher([ - ("html", self.startTagHtml) - ]) + ("html", self.startTagHtml) + ]) self.startTagHandler.default = self.startTagOther self.endTagHandler = utils.MethodDispatcher([("html", self.endTagHtml)]) self.endTagHandler.default = self.endTagOther def processEOF(self): - #Stop parsing + # Stop parsing pass def processComment(self, token): @@ -2507,11 +2488,11 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-after-body", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token - def endTagHtml(self,name): + def endTagHtml(self, name): if self.parser.innerHTML: self.parser.parseError("unexpected-end-tag-after-body-innerhtml") else: @@ -2519,7 +2500,7 @@ def getPhases(debug): def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-after-body", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token @@ -2562,7 +2543,7 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-in-frameset", - {"name": token["name"]}) + {"name": token["name"]}) def endTagFrameset(self, token): if self.tree.openElements[-1].name == "html": @@ -2571,15 +2552,14 @@ def getPhases(debug): else: self.tree.openElements.pop() if (not self.parser.innerHTML and - self.tree.openElements[-1].name != "frameset"): + self.tree.openElements[-1].name != "frameset"): # If we're not in innerHTML mode and the the current node is not a # "frameset" element (anymore) then switch. self.parser.phase = self.parser.phases["afterFrameset"] def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-in-frameset", - {"name": token["name"]}) - + {"name": token["name"]}) class AfterFramesetPhase(Phase): # http://www.whatwg.org/specs/web-apps/current-work/#after3 @@ -2598,7 +2578,7 @@ def getPhases(debug): self.endTagHandler.default = self.endTagOther def processEOF(self): - #Stop parsing + # Stop parsing pass def processCharacters(self, token): @@ -2609,15 +2589,14 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("unexpected-start-tag-after-frameset", - {"name": token["name"]}) + {"name": token["name"]}) def endTagHtml(self, token): self.parser.phase = self.parser.phases["afterAfterFrameset"] def endTagOther(self, token): self.parser.parseError("unexpected-end-tag-after-frameset", - {"name": token["name"]}) - + {"name": token["name"]}) class AfterAfterBodyPhase(Phase): def __init__(self, parser, tree): @@ -2647,13 +2626,13 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("expected-eof-but-got-start-tag", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token def processEndTag(self, token): self.parser.parseError("expected-eof-but-got-end-tag", - {"name": token["name"]}) + {"name": token["name"]}) self.parser.phase = self.parser.phases["inBody"] return token @@ -2687,12 +2666,11 @@ def getPhases(debug): def startTagOther(self, token): self.parser.parseError("expected-eof-but-got-start-tag", - {"name": token["name"]}) + {"name": token["name"]}) def processEndTag(self, token): self.parser.parseError("expected-eof-but-got-end-tag", - {"name": token["name"]}) - + {"name": token["name"]}) return { "initial": InitialPhase, @@ -2719,14 +2697,16 @@ def getPhases(debug): "afterAfterBody": AfterAfterBodyPhase, "afterAfterFrameset": AfterAfterFramesetPhase, # XXX after after frameset - } + } -def impliedTagToken(name, type="EndTag", attributes = None, - selfClosing = False): + +def impliedTagToken(name, type="EndTag", attributes=None, + selfClosing=False): if attributes is None: attributes = {} - return {"type":tokenTypes[type], "name":unicode(name), "data":attributes, - "selfClosing":selfClosing} + return {"type": tokenTypes[type], "name": name, "data": attributes, + "selfClosing": selfClosing} + class ParseError(Exception): """Error in parsed document""" diff --git a/libs/html5lib/ihatexml.py b/libs/html5lib/ihatexml.py index dd785639..0fc79308 100644 --- a/libs/html5lib/ihatexml.py +++ b/libs/html5lib/ihatexml.py @@ -1,25 +1,105 @@ -import re +from __future__ import absolute_import, division, unicode_literals -baseChar = """[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | [#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | [#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | [#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | [#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | [#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | [#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | [#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | [#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | [#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | [#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | [#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | [#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | [#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | [#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | [#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | [#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | [#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | [#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | [#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | [#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | [#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | [#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | [#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | [#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | [#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | [#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | [#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | [#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | #x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | #x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | #x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | [#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | [#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | #x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | [#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | [#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | [#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | [#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | [#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | #x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | [#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | [#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | [#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | [#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]""" +import re +import warnings + +from .constants import DataLossWarning + +baseChar = """ +[#x0041-#x005A] | [#x0061-#x007A] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | +[#x00F8-#x00FF] | [#x0100-#x0131] | [#x0134-#x013E] | [#x0141-#x0148] | +[#x014A-#x017E] | [#x0180-#x01C3] | [#x01CD-#x01F0] | [#x01F4-#x01F5] | +[#x01FA-#x0217] | [#x0250-#x02A8] | [#x02BB-#x02C1] | #x0386 | +[#x0388-#x038A] | #x038C | [#x038E-#x03A1] | [#x03A3-#x03CE] | +[#x03D0-#x03D6] | #x03DA | #x03DC | #x03DE | #x03E0 | [#x03E2-#x03F3] | +[#x0401-#x040C] | [#x040E-#x044F] | [#x0451-#x045C] | [#x045E-#x0481] | +[#x0490-#x04C4] | [#x04C7-#x04C8] | [#x04CB-#x04CC] | [#x04D0-#x04EB] | +[#x04EE-#x04F5] | [#x04F8-#x04F9] | [#x0531-#x0556] | #x0559 | +[#x0561-#x0586] | [#x05D0-#x05EA] | [#x05F0-#x05F2] | [#x0621-#x063A] | +[#x0641-#x064A] | [#x0671-#x06B7] | [#x06BA-#x06BE] | [#x06C0-#x06CE] | +[#x06D0-#x06D3] | #x06D5 | [#x06E5-#x06E6] | [#x0905-#x0939] | #x093D | +[#x0958-#x0961] | [#x0985-#x098C] | [#x098F-#x0990] | [#x0993-#x09A8] | +[#x09AA-#x09B0] | #x09B2 | [#x09B6-#x09B9] | [#x09DC-#x09DD] | +[#x09DF-#x09E1] | [#x09F0-#x09F1] | [#x0A05-#x0A0A] | [#x0A0F-#x0A10] | +[#x0A13-#x0A28] | [#x0A2A-#x0A30] | [#x0A32-#x0A33] | [#x0A35-#x0A36] | +[#x0A38-#x0A39] | [#x0A59-#x0A5C] | #x0A5E | [#x0A72-#x0A74] | +[#x0A85-#x0A8B] | #x0A8D | [#x0A8F-#x0A91] | [#x0A93-#x0AA8] | +[#x0AAA-#x0AB0] | [#x0AB2-#x0AB3] | [#x0AB5-#x0AB9] | #x0ABD | #x0AE0 | +[#x0B05-#x0B0C] | [#x0B0F-#x0B10] | [#x0B13-#x0B28] | [#x0B2A-#x0B30] | +[#x0B32-#x0B33] | [#x0B36-#x0B39] | #x0B3D | [#x0B5C-#x0B5D] | +[#x0B5F-#x0B61] | [#x0B85-#x0B8A] | [#x0B8E-#x0B90] | [#x0B92-#x0B95] | +[#x0B99-#x0B9A] | #x0B9C | [#x0B9E-#x0B9F] | [#x0BA3-#x0BA4] | +[#x0BA8-#x0BAA] | [#x0BAE-#x0BB5] | [#x0BB7-#x0BB9] | [#x0C05-#x0C0C] | +[#x0C0E-#x0C10] | [#x0C12-#x0C28] | [#x0C2A-#x0C33] | [#x0C35-#x0C39] | +[#x0C60-#x0C61] | [#x0C85-#x0C8C] | [#x0C8E-#x0C90] | [#x0C92-#x0CA8] | +[#x0CAA-#x0CB3] | [#x0CB5-#x0CB9] | #x0CDE | [#x0CE0-#x0CE1] | +[#x0D05-#x0D0C] | [#x0D0E-#x0D10] | [#x0D12-#x0D28] | [#x0D2A-#x0D39] | +[#x0D60-#x0D61] | [#x0E01-#x0E2E] | #x0E30 | [#x0E32-#x0E33] | +[#x0E40-#x0E45] | [#x0E81-#x0E82] | #x0E84 | [#x0E87-#x0E88] | #x0E8A | +#x0E8D | [#x0E94-#x0E97] | [#x0E99-#x0E9F] | [#x0EA1-#x0EA3] | #x0EA5 | +#x0EA7 | [#x0EAA-#x0EAB] | [#x0EAD-#x0EAE] | #x0EB0 | [#x0EB2-#x0EB3] | +#x0EBD | [#x0EC0-#x0EC4] | [#x0F40-#x0F47] | [#x0F49-#x0F69] | +[#x10A0-#x10C5] | [#x10D0-#x10F6] | #x1100 | [#x1102-#x1103] | +[#x1105-#x1107] | #x1109 | [#x110B-#x110C] | [#x110E-#x1112] | #x113C | +#x113E | #x1140 | #x114C | #x114E | #x1150 | [#x1154-#x1155] | #x1159 | +[#x115F-#x1161] | #x1163 | #x1165 | #x1167 | #x1169 | [#x116D-#x116E] | +[#x1172-#x1173] | #x1175 | #x119E | #x11A8 | #x11AB | [#x11AE-#x11AF] | +[#x11B7-#x11B8] | #x11BA | [#x11BC-#x11C2] | #x11EB | #x11F0 | #x11F9 | +[#x1E00-#x1E9B] | [#x1EA0-#x1EF9] | [#x1F00-#x1F15] | [#x1F18-#x1F1D] | +[#x1F20-#x1F45] | [#x1F48-#x1F4D] | [#x1F50-#x1F57] | #x1F59 | #x1F5B | +#x1F5D | [#x1F5F-#x1F7D] | [#x1F80-#x1FB4] | [#x1FB6-#x1FBC] | #x1FBE | +[#x1FC2-#x1FC4] | [#x1FC6-#x1FCC] | [#x1FD0-#x1FD3] | [#x1FD6-#x1FDB] | +[#x1FE0-#x1FEC] | [#x1FF2-#x1FF4] | [#x1FF6-#x1FFC] | #x2126 | +[#x212A-#x212B] | #x212E | [#x2180-#x2182] | [#x3041-#x3094] | +[#x30A1-#x30FA] | [#x3105-#x312C] | [#xAC00-#xD7A3]""" ideographic = """[#x4E00-#x9FA5] | #x3007 | [#x3021-#x3029]""" -combiningCharacter = """[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | [#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | [#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | [#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | #x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | [#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | [#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | #x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | [#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | [#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | #x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | [#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | [#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | [#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | [#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | [#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | #x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | [#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | #x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | [#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | [#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | #x3099 | #x309A""" +combiningCharacter = """ +[#x0300-#x0345] | [#x0360-#x0361] | [#x0483-#x0486] | [#x0591-#x05A1] | +[#x05A3-#x05B9] | [#x05BB-#x05BD] | #x05BF | [#x05C1-#x05C2] | #x05C4 | +[#x064B-#x0652] | #x0670 | [#x06D6-#x06DC] | [#x06DD-#x06DF] | +[#x06E0-#x06E4] | [#x06E7-#x06E8] | [#x06EA-#x06ED] | [#x0901-#x0903] | +#x093C | [#x093E-#x094C] | #x094D | [#x0951-#x0954] | [#x0962-#x0963] | +[#x0981-#x0983] | #x09BC | #x09BE | #x09BF | [#x09C0-#x09C4] | +[#x09C7-#x09C8] | [#x09CB-#x09CD] | #x09D7 | [#x09E2-#x09E3] | #x0A02 | +#x0A3C | #x0A3E | #x0A3F | [#x0A40-#x0A42] | [#x0A47-#x0A48] | +[#x0A4B-#x0A4D] | [#x0A70-#x0A71] | [#x0A81-#x0A83] | #x0ABC | +[#x0ABE-#x0AC5] | [#x0AC7-#x0AC9] | [#x0ACB-#x0ACD] | [#x0B01-#x0B03] | +#x0B3C | [#x0B3E-#x0B43] | [#x0B47-#x0B48] | [#x0B4B-#x0B4D] | +[#x0B56-#x0B57] | [#x0B82-#x0B83] | [#x0BBE-#x0BC2] | [#x0BC6-#x0BC8] | +[#x0BCA-#x0BCD] | #x0BD7 | [#x0C01-#x0C03] | [#x0C3E-#x0C44] | +[#x0C46-#x0C48] | [#x0C4A-#x0C4D] | [#x0C55-#x0C56] | [#x0C82-#x0C83] | +[#x0CBE-#x0CC4] | [#x0CC6-#x0CC8] | [#x0CCA-#x0CCD] | [#x0CD5-#x0CD6] | +[#x0D02-#x0D03] | [#x0D3E-#x0D43] | [#x0D46-#x0D48] | [#x0D4A-#x0D4D] | +#x0D57 | #x0E31 | [#x0E34-#x0E3A] | [#x0E47-#x0E4E] | #x0EB1 | +[#x0EB4-#x0EB9] | [#x0EBB-#x0EBC] | [#x0EC8-#x0ECD] | [#x0F18-#x0F19] | +#x0F35 | #x0F37 | #x0F39 | #x0F3E | #x0F3F | [#x0F71-#x0F84] | +[#x0F86-#x0F8B] | [#x0F90-#x0F95] | #x0F97 | [#x0F99-#x0FAD] | +[#x0FB1-#x0FB7] | #x0FB9 | [#x20D0-#x20DC] | #x20E1 | [#x302A-#x302F] | +#x3099 | #x309A""" -digit = """[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | [#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | [#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | [#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]""" +digit = """ +[#x0030-#x0039] | [#x0660-#x0669] | [#x06F0-#x06F9] | [#x0966-#x096F] | +[#x09E6-#x09EF] | [#x0A66-#x0A6F] | [#x0AE6-#x0AEF] | [#x0B66-#x0B6F] | +[#x0BE7-#x0BEF] | [#x0C66-#x0C6F] | [#x0CE6-#x0CEF] | [#x0D66-#x0D6F] | +[#x0E50-#x0E59] | [#x0ED0-#x0ED9] | [#x0F20-#x0F29]""" -extender = """#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | [#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]""" +extender = """ +#x00B7 | #x02D0 | #x02D1 | #x0387 | #x0640 | #x0E46 | #x0EC6 | #x3005 | +#[#x3031-#x3035] | [#x309D-#x309E] | [#x30FC-#x30FE]""" letter = " | ".join([baseChar, ideographic]) -#Without the -name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, - extender]) +# Without the +name = " | ".join([letter, digit, ".", "-", "_", combiningCharacter, + extender]) nameFirst = " | ".join([letter, "_"]) reChar = re.compile(r"#x([\d|A-F]{4,4})") reCharRange = re.compile(r"\[#x([\d|A-F]{4,4})-#x([\d|A-F]{4,4})\]") + def charStringToList(chars): charRanges = [item.strip() for item in chars.split(" | ")] rv = [] @@ -30,16 +110,17 @@ def charStringToList(chars): if match is not None: rv.append([hexToInt(item) for item in match.groups()]) if len(rv[-1]) == 1: - rv[-1] = rv[-1]*2 + rv[-1] = rv[-1] * 2 foundMatch = True break if not foundMatch: assert len(item) == 1 - + rv.append([ord(item)] * 2) rv = normaliseCharList(rv) return rv + def normaliseCharList(charList): charList = sorted(charList) for item in charList: @@ -49,61 +130,69 @@ def normaliseCharList(charList): while i < len(charList): j = 1 rv.append(charList[i]) - while i + j < len(charList) and charList[i+j][0] <= rv[-1][1] + 1: - rv[-1][1] = charList[i+j][1] + while i + j < len(charList) and charList[i + j][0] <= rv[-1][1] + 1: + rv[-1][1] = charList[i + j][1] j += 1 i += j return rv -#We don't really support characters above the BMP :( +# We don't really support characters above the BMP :( max_unicode = int("FFFF", 16) - + + def missingRanges(charList): rv = [] if charList[0] != 0: rv.append([0, charList[0][0] - 1]) for i, item in enumerate(charList[:-1]): - rv.append([item[1]+1, charList[i+1][0] - 1]) + rv.append([item[1] + 1, charList[i + 1][0] - 1]) if charList[-1][1] != max_unicode: rv.append([charList[-1][1] + 1, max_unicode]) return rv + def listToRegexpStr(charList): rv = [] for item in charList: if item[0] == item[1]: - rv.append(escapeRegexp(unichr(item[0]))) + rv.append(escapeRegexp(chr(item[0]))) else: - rv.append(escapeRegexp(unichr(item[0])) + "-" + - escapeRegexp(unichr(item[1]))) - return "[%s]"%"".join(rv) + rv.append(escapeRegexp(chr(item[0])) + "-" + + escapeRegexp(chr(item[1]))) + return "[%s]" % "".join(rv) + def hexToInt(hex_str): return int(hex_str, 16) + def escapeRegexp(string): specialCharacters = (".", "^", "$", "*", "+", "?", "{", "}", - "[", "]", "|", "(", ")", "-") + "[", "]", "|", "(", ")", "-") for char in specialCharacters: string = string.replace(char, "\\" + char) - if char in string: - print string return string -#output from the above -nonXmlNameBMPRegexp = re.compile(u'[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') +# output from the above +nonXmlNameBMPRegexp = re.compile('[\x00-,/:-@\\[-\\^`\\{-\xb6\xb8-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u02cf\u02d2-\u02ff\u0346-\u035f\u0362-\u0385\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482\u0487-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u0590\u05a2\u05ba\u05be\u05c0\u05c3\u05c5-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u063f\u0653-\u065f\u066a-\u066f\u06b8-\u06b9\u06bf\u06cf\u06d4\u06e9\u06ee-\u06ef\u06fa-\u0900\u0904\u093a-\u093b\u094e-\u0950\u0955-\u0957\u0964-\u0965\u0970-\u0980\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09bb\u09bd\u09c5-\u09c6\u09c9-\u09ca\u09ce-\u09d6\u09d8-\u09db\u09de\u09e4-\u09e5\u09f2-\u0a01\u0a03-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a3b\u0a3d\u0a43-\u0a46\u0a49-\u0a4a\u0a4e-\u0a58\u0a5d\u0a5f-\u0a65\u0a75-\u0a80\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abb\u0ac6\u0aca\u0ace-\u0adf\u0ae1-\u0ae5\u0af0-\u0b00\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3b\u0b44-\u0b46\u0b49-\u0b4a\u0b4e-\u0b55\u0b58-\u0b5b\u0b5e\u0b62-\u0b65\u0b70-\u0b81\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0bbd\u0bc3-\u0bc5\u0bc9\u0bce-\u0bd6\u0bd8-\u0be6\u0bf0-\u0c00\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c3d\u0c45\u0c49\u0c4e-\u0c54\u0c57-\u0c5f\u0c62-\u0c65\u0c70-\u0c81\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cbd\u0cc5\u0cc9\u0cce-\u0cd4\u0cd7-\u0cdd\u0cdf\u0ce2-\u0ce5\u0cf0-\u0d01\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d3d\u0d44-\u0d45\u0d49\u0d4e-\u0d56\u0d58-\u0d5f\u0d62-\u0d65\u0d70-\u0e00\u0e2f\u0e3b-\u0e3f\u0e4f\u0e5a-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eba\u0ebe-\u0ebf\u0ec5\u0ec7\u0ece-\u0ecf\u0eda-\u0f17\u0f1a-\u0f1f\u0f2a-\u0f34\u0f36\u0f38\u0f3a-\u0f3d\u0f48\u0f6a-\u0f70\u0f85\u0f8c-\u0f8f\u0f96\u0f98\u0fae-\u0fb0\u0fb8\u0fba-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u20cf\u20dd-\u20e0\u20e2-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3004\u3006\u3008-\u3020\u3030\u3036-\u3040\u3095-\u3098\u309b-\u309c\u309f-\u30a0\u30fb\u30ff-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') + +nonXmlNameFirstBMPRegexp = re.compile('[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') + +# Simpler things +nonPubidCharRegexp = re.compile("[^\x20\x0D\x0Aa-zA-Z0-9\-\'()+,./:=?;!*#@$_%]") -nonXmlNameFirstBMPRegexp = re.compile(u'[\x00-@\\[-\\^`\\{-\xbf\xd7\xf7\u0132-\u0133\u013f-\u0140\u0149\u017f\u01c4-\u01cc\u01f1-\u01f3\u01f6-\u01f9\u0218-\u024f\u02a9-\u02ba\u02c2-\u0385\u0387\u038b\u038d\u03a2\u03cf\u03d7-\u03d9\u03db\u03dd\u03df\u03e1\u03f4-\u0400\u040d\u0450\u045d\u0482-\u048f\u04c5-\u04c6\u04c9-\u04ca\u04cd-\u04cf\u04ec-\u04ed\u04f6-\u04f7\u04fa-\u0530\u0557-\u0558\u055a-\u0560\u0587-\u05cf\u05eb-\u05ef\u05f3-\u0620\u063b-\u0640\u064b-\u0670\u06b8-\u06b9\u06bf\u06cf\u06d4\u06d6-\u06e4\u06e7-\u0904\u093a-\u093c\u093e-\u0957\u0962-\u0984\u098d-\u098e\u0991-\u0992\u09a9\u09b1\u09b3-\u09b5\u09ba-\u09db\u09de\u09e2-\u09ef\u09f2-\u0a04\u0a0b-\u0a0e\u0a11-\u0a12\u0a29\u0a31\u0a34\u0a37\u0a3a-\u0a58\u0a5d\u0a5f-\u0a71\u0a75-\u0a84\u0a8c\u0a8e\u0a92\u0aa9\u0ab1\u0ab4\u0aba-\u0abc\u0abe-\u0adf\u0ae1-\u0b04\u0b0d-\u0b0e\u0b11-\u0b12\u0b29\u0b31\u0b34-\u0b35\u0b3a-\u0b3c\u0b3e-\u0b5b\u0b5e\u0b62-\u0b84\u0b8b-\u0b8d\u0b91\u0b96-\u0b98\u0b9b\u0b9d\u0ba0-\u0ba2\u0ba5-\u0ba7\u0bab-\u0bad\u0bb6\u0bba-\u0c04\u0c0d\u0c11\u0c29\u0c34\u0c3a-\u0c5f\u0c62-\u0c84\u0c8d\u0c91\u0ca9\u0cb4\u0cba-\u0cdd\u0cdf\u0ce2-\u0d04\u0d0d\u0d11\u0d29\u0d3a-\u0d5f\u0d62-\u0e00\u0e2f\u0e31\u0e34-\u0e3f\u0e46-\u0e80\u0e83\u0e85-\u0e86\u0e89\u0e8b-\u0e8c\u0e8e-\u0e93\u0e98\u0ea0\u0ea4\u0ea6\u0ea8-\u0ea9\u0eac\u0eaf\u0eb1\u0eb4-\u0ebc\u0ebe-\u0ebf\u0ec5-\u0f3f\u0f48\u0f6a-\u109f\u10c6-\u10cf\u10f7-\u10ff\u1101\u1104\u1108\u110a\u110d\u1113-\u113b\u113d\u113f\u1141-\u114b\u114d\u114f\u1151-\u1153\u1156-\u1158\u115a-\u115e\u1162\u1164\u1166\u1168\u116a-\u116c\u116f-\u1171\u1174\u1176-\u119d\u119f-\u11a7\u11a9-\u11aa\u11ac-\u11ad\u11b0-\u11b6\u11b9\u11bb\u11c3-\u11ea\u11ec-\u11ef\u11f1-\u11f8\u11fa-\u1dff\u1e9c-\u1e9f\u1efa-\u1eff\u1f16-\u1f17\u1f1e-\u1f1f\u1f46-\u1f47\u1f4e-\u1f4f\u1f58\u1f5a\u1f5c\u1f5e\u1f7e-\u1f7f\u1fb5\u1fbd\u1fbf-\u1fc1\u1fc5\u1fcd-\u1fcf\u1fd4-\u1fd5\u1fdc-\u1fdf\u1fed-\u1ff1\u1ff5\u1ffd-\u2125\u2127-\u2129\u212c-\u212d\u212f-\u217f\u2183-\u3006\u3008-\u3020\u302a-\u3040\u3095-\u30a0\u30fb-\u3104\u312d-\u4dff\u9fa6-\uabff\ud7a4-\uffff]') class InfosetFilter(object): replacementRegexp = re.compile(r"U[\dA-F]{5,5}") - def __init__(self, replaceChars = None, - dropXmlnsLocalName = False, - dropXmlnsAttrNs = False, - preventDoubleDashComments = False, - preventDashAtCommentEnd = False, - replaceFormFeedCharacters = True): + + def __init__(self, replaceChars=None, + dropXmlnsLocalName=False, + dropXmlnsAttrNs=False, + preventDoubleDashComments=False, + preventDashAtCommentEnd=False, + replaceFormFeedCharacters=True, + preventSingleQuotePubid=False): self.dropXmlnsLocalName = dropXmlnsLocalName self.dropXmlnsAttrNs = dropXmlnsAttrNs @@ -113,14 +202,17 @@ class InfosetFilter(object): self.replaceFormFeedCharacters = replaceFormFeedCharacters + self.preventSingleQuotePubid = preventSingleQuotePubid + self.replaceCache = {} def coerceAttribute(self, name, namespace=None): if self.dropXmlnsLocalName and name.startswith("xmlns:"): - #Need a datalosswarning here + warnings.warn("Attributes cannot begin with xmlns", DataLossWarning) return None - elif (self.dropXmlnsAttrNs and + elif (self.dropXmlnsAttrNs and namespace == "http://www.w3.org/2000/xmlns/"): + warnings.warn("Attributes cannot be in the xml namespace", DataLossWarning) return None else: return self.toXmlName(name) @@ -131,20 +223,35 @@ class InfosetFilter(object): def coerceComment(self, data): if self.preventDoubleDashComments: while "--" in data: + warnings.warn("Comments cannot contain adjacent dashes", DataLossWarning) data = data.replace("--", "- -") return data - + def coerceCharacters(self, data): if self.replaceFormFeedCharacters: + for i in range(data.count("\x0C")): + warnings.warn("Text cannot contain U+000C", DataLossWarning) data = data.replace("\x0C", " ") - #Other non-xml characters + # Other non-xml characters return data + def coercePubid(self, data): + dataOutput = data + for char in nonPubidCharRegexp.findall(data): + warnings.warn("Coercing non-XML pubid", DataLossWarning) + replacement = self.getReplacementCharacter(char) + dataOutput = dataOutput.replace(char, replacement) + if self.preventSingleQuotePubid and dataOutput.find("'") >= 0: + warnings.warn("Pubid cannot contain single quote", DataLossWarning) + dataOutput = dataOutput.replace("'", self.getReplacementCharacter("'")) + return dataOutput + def toXmlName(self, name): nameFirst = name[0] nameRest = name[1:] m = nonXmlNameFirstBMPRegexp.match(nameFirst) if m: + warnings.warn("Coercing non-XML name", DataLossWarning) nameFirstOutput = self.getReplacementCharacter(nameFirst) else: nameFirstOutput = nameFirst @@ -152,10 +259,11 @@ class InfosetFilter(object): nameRestOutput = nameRest replaceChars = set(nonXmlNameBMPRegexp.findall(nameRest)) for char in replaceChars: + warnings.warn("Coercing non-XML name", DataLossWarning) replacement = self.getReplacementCharacter(char) nameRestOutput = nameRestOutput.replace(char, replacement) return nameFirstOutput + nameRestOutput - + def getReplacementCharacter(self, char): if char in self.replaceCache: replacement = self.replaceCache[char] @@ -169,9 +277,9 @@ class InfosetFilter(object): return name def escapeChar(self, char): - replacement = "U" + hex(ord(char))[2:].upper().rjust(5, "0") + replacement = "U%05X" % ord(char) self.replaceCache[char] = replacement return replacement def unescapeChar(self, charcode): - return unichr(int(charcode[1:], 16)) + return chr(int(charcode[1:], 16)) diff --git a/libs/html5lib/inputstream.py b/libs/html5lib/inputstream.py index edec1329..004bdd4a 100644 --- a/libs/html5lib/inputstream.py +++ b/libs/html5lib/inputstream.py @@ -1,19 +1,33 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type + import codecs import re -import types -import sys -from constants import EOF, spaceCharacters, asciiLetters, asciiUppercase -from constants import encodings, ReparseException -import utils +from .constants import EOF, spaceCharacters, asciiLetters, asciiUppercase +from .constants import encodings, ReparseException +from . import utils -#Non-unicode versions of constants for use in the pre-parser -spaceCharactersBytes = frozenset([str(item) for item in spaceCharacters]) -asciiLettersBytes = frozenset([str(item) for item in asciiLetters]) -asciiUppercaseBytes = frozenset([str(item) for item in asciiUppercase]) -spacesAngleBrackets = spaceCharactersBytes | frozenset([">", "<"]) +from io import StringIO -invalid_unicode_re = re.compile(u"[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]") +try: + from io import BytesIO +except ImportError: + BytesIO = StringIO + +try: + from io import BufferedIOBase +except ImportError: + class BufferedIOBase(object): + pass + +# Non-unicode versions of constants for use in the pre-parser +spaceCharactersBytes = frozenset([item.encode("ascii") for item in spaceCharacters]) +asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters]) +asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase]) +spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"]) + +invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]") non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, @@ -23,22 +37,23 @@ non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF]) -ascii_punctuation_re = re.compile(ur"[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]") +ascii_punctuation_re = re.compile("[\u0009-\u000D\u0020-\u002F\u003A-\u0040\u005B-\u0060\u007B-\u007E]") # Cache for charsUntil() charsUntilRegEx = {} - -class BufferedStream: + + +class BufferedStream(object): """Buffering for streams that do not have buffering of their own - The buffer is implemented as a list of chunks on the assumption that + The buffer is implemented as a list of chunks on the assumption that joining many strings will be slow since it is O(n**2) """ - + def __init__(self, stream): self.stream = stream self.buffer = [] - self.position = [-1,0] #chunk number, offset + self.position = [-1, 0] # chunk number, offset def tell(self): pos = 0 @@ -48,11 +63,11 @@ class BufferedStream: return pos def seek(self, pos): - assert pos < self._bufferedBytes() + assert pos <= self._bufferedBytes() offset = pos i = 0 while len(self.buffer[i]) < offset: - offset -= pos + offset -= len(self.buffer[i]) i += 1 self.position = [i, offset] @@ -64,7 +79,7 @@ class BufferedStream: return self._readStream(bytes) else: return self._readFromBuffer(bytes) - + def _bufferedBytes(self): return sum([len(item) for item in self.buffer]) @@ -83,7 +98,7 @@ class BufferedStream: while bufferIndex < len(self.buffer) and remainingBytes != 0: assert remainingBytes > 0 bufferedData = self.buffer[bufferIndex] - + if remainingBytes <= len(bufferedData) - bufferOffset: bytesToRead = remainingBytes self.position = [bufferIndex, bufferOffset + bytesToRead] @@ -91,20 +106,33 @@ class BufferedStream: bytesToRead = len(bufferedData) - bufferOffset self.position = [bufferIndex, len(bufferedData)] bufferIndex += 1 - data = rv.append(bufferedData[bufferOffset: - bufferOffset + bytesToRead]) + rv.append(bufferedData[bufferOffset:bufferOffset + bytesToRead]) remainingBytes -= bytesToRead bufferOffset = 0 if remainingBytes: rv.append(self._readStream(remainingBytes)) - - return "".join(rv) - + + return b"".join(rv) -class HTMLInputStream: +def HTMLInputStream(source, encoding=None, parseMeta=True, chardet=True): + if hasattr(source, "read"): + isUnicode = isinstance(source.read(0), text_type) + else: + isUnicode = isinstance(source, text_type) + + if isUnicode: + if encoding is not None: + raise TypeError("Cannot explicitly set an encoding with a unicode string") + + return HTMLUnicodeInputStream(source) + else: + return HTMLBinaryInputStream(source, encoding, parseMeta, chardet) + + +class HTMLUnicodeInputStream(object): """Provides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing @@ -114,7 +142,7 @@ class HTMLInputStream: _defaultChunkSize = 10240 - def __init__(self, source, encoding=None, parseMeta=True, chardet=True): + def __init__(self, source): """Initialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source @@ -126,49 +154,29 @@ class HTMLInputStream: the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) - + parseMeta - Look for a <meta> element containing encoding information """ - #Craziness - if len(u"\U0010FFFF") == 1: + # Craziness + if len("\U0010FFFF") == 1: self.reportCharacterErrors = self.characterErrorsUCS4 - self.replaceCharactersRegexp = re.compile(u"[\uD800-\uDFFF]") + self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]") else: self.reportCharacterErrors = self.characterErrorsUCS2 - self.replaceCharactersRegexp = re.compile(u"([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])") + self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])") # List of where new lines occur self.newLines = [0] - self.charEncoding = (codecName(encoding), "certain") - - # Raw Stream - for unicode objects this will encode to utf-8 and set - # self.charEncoding as appropriate - self.rawStream = self.openStream(source) - - # Encoding Information - #Number of bytes to use when looking for a meta element with - #encoding information - self.numBytesMeta = 512 - #Number of bytes to use when using detecting encoding using chardet - self.numBytesChardet = 100 - #Encoding to use if no other information can be found - self.defaultEncoding = "windows-1252" - - #Detect encoding iff no explicit "transport level" encoding is supplied - if (self.charEncoding[0] is None): - self.charEncoding = self.detectEncoding(parseMeta, chardet) - + self.charEncoding = ("utf-8", "certain") + self.dataStream = self.openStream(source) self.reset() def reset(self): - self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream, - 'replace') - - self.chunk = u"" + self.chunk = "" self.chunkSize = 0 self.chunkOffset = 0 self.errors = [] @@ -177,8 +185,8 @@ class HTMLInputStream: self.prevNumLines = 0 # number of columns in the last line of the previous chunk self.prevNumCols = 0 - - #Deal with CR LF and surrogates split over chunk boundaries + + # Deal with CR LF and surrogates split over chunk boundaries self._bufferedCharacter = None def openStream(self, source): @@ -191,126 +199,15 @@ class HTMLInputStream: if hasattr(source, 'read'): stream = source else: - # Otherwise treat source as a string and convert to a file object - if isinstance(source, unicode): - source = source.encode('utf-8') - self.charEncoding = ("utf-8", "certain") - try: - from io import BytesIO - except: - # 2to3 converts this line to: from io import StringIO - from cStringIO import StringIO as BytesIO - stream = BytesIO(source) - - if (not(hasattr(stream, "tell") and hasattr(stream, "seek")) or - stream is sys.stdin): - stream = BufferedStream(stream) + stream = StringIO(source) return stream - def detectEncoding(self, parseMeta=True, chardet=True): - #First look for a BOM - #This will also read past the BOM if present - encoding = self.detectBOM() - confidence = "certain" - #If there is no BOM need to look for meta elements with encoding - #information - if encoding is None and parseMeta: - encoding = self.detectEncodingMeta() - confidence = "tentative" - #Guess with chardet, if avaliable - if encoding is None and chardet: - confidence = "tentative" - try: - from chardet.universaldetector import UniversalDetector - buffers = [] - detector = UniversalDetector() - while not detector.done: - buffer = self.rawStream.read(self.numBytesChardet) - if not buffer: - break - buffers.append(buffer) - detector.feed(buffer) - detector.close() - encoding = detector.result['encoding'] - self.rawStream.seek(0) - except ImportError: - pass - # If all else fails use the default encoding - if encoding is None: - confidence="tentative" - encoding = self.defaultEncoding - - #Substitute for equivalent encodings: - encodingSub = {"iso-8859-1":"windows-1252"} - - if encoding.lower() in encodingSub: - encoding = encodingSub[encoding.lower()] - - return encoding, confidence - - def changeEncoding(self, newEncoding): - newEncoding = codecName(newEncoding) - if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"): - newEncoding = "utf-8" - if newEncoding is None: - return - elif newEncoding == self.charEncoding[0]: - self.charEncoding = (self.charEncoding[0], "certain") - else: - self.rawStream.seek(0) - self.reset() - self.charEncoding = (newEncoding, "certain") - raise ReparseException, "Encoding changed from %s to %s"%(self.charEncoding[0], newEncoding) - - def detectBOM(self): - """Attempts to detect at BOM at the start of the stream. If - an encoding can be determined from the BOM return the name of the - encoding otherwise return None""" - bomDict = { - codecs.BOM_UTF8: 'utf-8', - codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be', - codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be' - } - - # Go to beginning of file and read in 4 bytes - string = self.rawStream.read(4) - - # Try detecting the BOM using bytes from the string - encoding = bomDict.get(string[:3]) # UTF-8 - seek = 3 - if not encoding: - # Need to detect UTF-32 before UTF-16 - encoding = bomDict.get(string) # UTF-32 - seek = 4 - if not encoding: - encoding = bomDict.get(string[:2]) # UTF-16 - seek = 2 - - # Set the read position past the BOM if one was found, otherwise - # set it to the start of the stream - self.rawStream.seek(encoding and seek or 0) - - return encoding - - def detectEncodingMeta(self): - """Report the encoding declared by the meta element - """ - buffer = self.rawStream.read(self.numBytesMeta) - parser = EncodingParser(buffer) - self.rawStream.seek(0) - encoding = parser.getEncoding() - - if encoding in ("utf-16", "utf-16-be", "utf-16-le"): - encoding = "utf-8" - - return encoding - def _position(self, offset): chunk = self.chunk - nLines = chunk.count(u'\n', 0, offset) + nLines = chunk.count('\n', 0, offset) positionLine = self.prevNumLines + nLines - lastLinePos = chunk.rfind(u'\n', 0, offset) + lastLinePos = chunk.rfind('\n', 0, offset) if lastLinePos == -1: positionColumn = self.prevNumCols + offset else: @@ -320,7 +217,7 @@ class HTMLInputStream: def position(self): """Returns (line, col) of the current position in the stream.""" line, col = self._position(self.chunkOffset) - return (line+1, col) + return (line + 1, col) def char(self): """ Read one character from the stream or queue if available. Return @@ -343,34 +240,34 @@ class HTMLInputStream: self.prevNumLines, self.prevNumCols = self._position(self.chunkSize) - self.chunk = u"" + self.chunk = "" self.chunkSize = 0 self.chunkOffset = 0 data = self.dataStream.read(chunkSize) - - #Deal with CR LF and surrogates broken across chunks + + # Deal with CR LF and surrogates broken across chunks if self._bufferedCharacter: data = self._bufferedCharacter + data self._bufferedCharacter = None elif not data: # We have no more data, bye-bye stream return False - + if len(data) > 1: lastv = ord(data[-1]) if lastv == 0x0D or 0xD800 <= lastv <= 0xDBFF: self._bufferedCharacter = data[-1] data = data[:-1] - + self.reportCharacterErrors(data) - + # Replace invalid characters # Note U+0000 is dealt with in the tokenizer - data = self.replaceCharactersRegexp.sub(u"\ufffd", data) - - data = data.replace(u"\r\n", u"\n") - data = data.replace(u"\r", u"\n") + data = self.replaceCharactersRegexp.sub("\ufffd", data) + + data = data.replace("\r\n", "\n") + data = data.replace("\r", "\n") self.chunk = data self.chunkSize = len(data) @@ -378,23 +275,22 @@ class HTMLInputStream: return True def characterErrorsUCS4(self, data): - for i in xrange(len(invalid_unicode_re.findall(data))): + for i in range(len(invalid_unicode_re.findall(data))): self.errors.append("invalid-codepoint") def characterErrorsUCS2(self, data): - #Someone picked the wrong compile option - #You lose + # Someone picked the wrong compile option + # You lose skip = False - import sys for match in invalid_unicode_re.finditer(data): if skip: continue codepoint = ord(match.group()) pos = match.start() - #Pretty sure there should be endianness issues here - if utils.isSurrogatePair(data[pos:pos+2]): - #We have a surrogate pair! - char_val = utils.surrogatePairToCodepoint(data[pos:pos+2]) + # Pretty sure there should be endianness issues here + if utils.isSurrogatePair(data[pos:pos + 2]): + # We have a surrogate pair! + char_val = utils.surrogatePairToCodepoint(data[pos:pos + 2]) if char_val in non_bmp_invalid_codepoints: self.errors.append("invalid-codepoint") skip = True @@ -405,7 +301,7 @@ class HTMLInputStream: skip = False self.errors.append("invalid-codepoint") - def charsUntil(self, characters, opposite = False): + def charsUntil(self, characters, opposite=False): """ Returns a string of characters from the stream up to but not including any character in 'characters' or EOF. 'characters' must be a container that supports the 'in' method and iteration over its @@ -417,12 +313,12 @@ class HTMLInputStream: chars = charsUntilRegEx[(characters, opposite)] except KeyError: if __debug__: - for c in characters: + for c in characters: assert(ord(c) < 128) - regex = u"".join([u"\\x%02x" % ord(c) for c in characters]) + regex = "".join(["\\x%02x" % ord(c) for c in characters]) if not opposite: - regex = u"^%s" % regex - chars = charsUntilRegEx[(characters, opposite)] = re.compile(u"[%s]+" % regex) + regex = "^%s" % regex + chars = charsUntilRegEx[(characters, opposite)] = re.compile("[%s]+" % regex) rv = [] @@ -449,7 +345,7 @@ class HTMLInputStream: # Reached EOF break - r = u"".join(rv) + r = "".join(rv) return r def unget(self, char): @@ -468,26 +364,210 @@ class HTMLInputStream: self.chunkOffset -= 1 assert self.chunk[self.chunkOffset] == char -class EncodingBytes(str): + +class HTMLBinaryInputStream(HTMLUnicodeInputStream): + """Provides a unicode stream of characters to the HTMLTokenizer. + + This class takes care of character encoding and removing or replacing + incorrect byte-sequences and also provides column and line tracking. + + """ + + def __init__(self, source, encoding=None, parseMeta=True, chardet=True): + """Initialises the HTMLInputStream. + + HTMLInputStream(source, [encoding]) -> Normalized stream from source + for use by html5lib. + + source can be either a file-object, local filename or a string. + + The optional encoding parameter must be a string that indicates + the encoding. If specified, that encoding will be used, + regardless of any BOM or later declaration (such as in a meta + element) + + parseMeta - Look for a <meta> element containing encoding information + + """ + # Raw Stream - for unicode objects this will encode to utf-8 and set + # self.charEncoding as appropriate + self.rawStream = self.openStream(source) + + HTMLUnicodeInputStream.__init__(self, self.rawStream) + + self.charEncoding = (codecName(encoding), "certain") + + # Encoding Information + # Number of bytes to use when looking for a meta element with + # encoding information + self.numBytesMeta = 512 + # Number of bytes to use when using detecting encoding using chardet + self.numBytesChardet = 100 + # Encoding to use if no other information can be found + self.defaultEncoding = "windows-1252" + + # Detect encoding iff no explicit "transport level" encoding is supplied + if (self.charEncoding[0] is None): + self.charEncoding = self.detectEncoding(parseMeta, chardet) + + # Call superclass + self.reset() + + def reset(self): + self.dataStream = codecs.getreader(self.charEncoding[0])(self.rawStream, + 'replace') + HTMLUnicodeInputStream.reset(self) + + def openStream(self, source): + """Produces a file object from source. + + source can be either a file object, local filename or a string. + + """ + # Already a file object + if hasattr(source, 'read'): + stream = source + else: + stream = BytesIO(source) + + try: + stream.seek(stream.tell()) + except: + stream = BufferedStream(stream) + + return stream + + def detectEncoding(self, parseMeta=True, chardet=True): + # First look for a BOM + # This will also read past the BOM if present + encoding = self.detectBOM() + confidence = "certain" + # If there is no BOM need to look for meta elements with encoding + # information + if encoding is None and parseMeta: + encoding = self.detectEncodingMeta() + confidence = "tentative" + # Guess with chardet, if avaliable + if encoding is None and chardet: + confidence = "tentative" + try: + try: + from charade.universaldetector import UniversalDetector + except ImportError: + from chardet.universaldetector import UniversalDetector + buffers = [] + detector = UniversalDetector() + while not detector.done: + buffer = self.rawStream.read(self.numBytesChardet) + assert isinstance(buffer, bytes) + if not buffer: + break + buffers.append(buffer) + detector.feed(buffer) + detector.close() + encoding = detector.result['encoding'] + self.rawStream.seek(0) + except ImportError: + pass + # If all else fails use the default encoding + if encoding is None: + confidence = "tentative" + encoding = self.defaultEncoding + + # Substitute for equivalent encodings: + encodingSub = {"iso-8859-1": "windows-1252"} + + if encoding.lower() in encodingSub: + encoding = encodingSub[encoding.lower()] + + return encoding, confidence + + def changeEncoding(self, newEncoding): + assert self.charEncoding[1] != "certain" + newEncoding = codecName(newEncoding) + if newEncoding in ("utf-16", "utf-16-be", "utf-16-le"): + newEncoding = "utf-8" + if newEncoding is None: + return + elif newEncoding == self.charEncoding[0]: + self.charEncoding = (self.charEncoding[0], "certain") + else: + self.rawStream.seek(0) + self.reset() + self.charEncoding = (newEncoding, "certain") + raise ReparseException("Encoding changed from %s to %s" % (self.charEncoding[0], newEncoding)) + + def detectBOM(self): + """Attempts to detect at BOM at the start of the stream. If + an encoding can be determined from the BOM return the name of the + encoding otherwise return None""" + bomDict = { + codecs.BOM_UTF8: 'utf-8', + codecs.BOM_UTF16_LE: 'utf-16-le', codecs.BOM_UTF16_BE: 'utf-16-be', + codecs.BOM_UTF32_LE: 'utf-32-le', codecs.BOM_UTF32_BE: 'utf-32-be' + } + + # Go to beginning of file and read in 4 bytes + string = self.rawStream.read(4) + assert isinstance(string, bytes) + + # Try detecting the BOM using bytes from the string + encoding = bomDict.get(string[:3]) # UTF-8 + seek = 3 + if not encoding: + # Need to detect UTF-32 before UTF-16 + encoding = bomDict.get(string) # UTF-32 + seek = 4 + if not encoding: + encoding = bomDict.get(string[:2]) # UTF-16 + seek = 2 + + # Set the read position past the BOM if one was found, otherwise + # set it to the start of the stream + self.rawStream.seek(encoding and seek or 0) + + return encoding + + def detectEncodingMeta(self): + """Report the encoding declared by the meta element + """ + buffer = self.rawStream.read(self.numBytesMeta) + assert isinstance(buffer, bytes) + parser = EncodingParser(buffer) + self.rawStream.seek(0) + encoding = parser.getEncoding() + + if encoding in ("utf-16", "utf-16-be", "utf-16-le"): + encoding = "utf-8" + + return encoding + + +class EncodingBytes(bytes): """String-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raised""" def __new__(self, value): - return str.__new__(self, value.lower()) + assert isinstance(value, bytes) + return bytes.__new__(self, value.lower()) def __init__(self, value): - self._position=-1 - + self._position = -1 + def __iter__(self): return self - - def next(self): + + def __next__(self): p = self._position = self._position + 1 if p >= len(self): raise StopIteration elif p < 0: raise TypeError - return self[p] + return self[p:p + 1] + + def next(self): + # Py2 compat + return self.__next__() def previous(self): p = self._position @@ -496,13 +576,13 @@ class EncodingBytes(str): elif p < 0: raise TypeError self._position = p = p - 1 - return self[p] - + return self[p:p + 1] + def setPosition(self, position): if self._position >= len(self): raise StopIteration self._position = position - + def getPosition(self): if self._position >= len(self): raise StopIteration @@ -510,19 +590,19 @@ class EncodingBytes(str): return self._position else: return None - + position = property(getPosition, setPosition) def getCurrentByte(self): - return self[self.position] - + return self[self.position:self.position + 1] + currentByte = property(getCurrentByte) def skip(self, chars=spaceCharactersBytes): """Skip past a list of characters""" p = self.position # use property for the error-checking while p < len(self): - c = self[p] + c = self[p:p + 1] if c not in chars: self._position = p return c @@ -533,7 +613,7 @@ class EncodingBytes(str): def skipUntil(self, chars): p = self.position while p < len(self): - c = self[p] + c = self[p:p + 1] if c in chars: self._position = p return c @@ -542,16 +622,16 @@ class EncodingBytes(str): return None def matchBytes(self, bytes): - """Look for a sequence of bytes at the start of a string. If the bytes - are found return True and advance the position to the byte after the + """Look for a sequence of bytes at the start of a string. If the bytes + are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone""" p = self.position - data = self[p:p+len(bytes)] + data = self[p:p + len(bytes)] rv = data.startswith(bytes) if rv: self.position += len(bytes) return rv - + def jumpTo(self, bytes): """Look for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the match""" @@ -560,11 +640,12 @@ class EncodingBytes(str): # XXX: This is ugly, but I can't see a nicer way to fix this. if self._position == -1: self._position = 0 - self._position += (newPosition + len(bytes)-1) + self._position += (newPosition + len(bytes) - 1) return True else: raise StopIteration + class EncodingParser(object): """Mini parser for detecting character encoding from meta elements""" @@ -575,147 +656,158 @@ class EncodingParser(object): def getEncoding(self): methodDispatch = ( - ("<!--",self.handleComment), - ("<meta",self.handleMeta), - ("</",self.handlePossibleEndTag), - ("<!",self.handleOther), - ("<?",self.handleOther), - ("<",self.handlePossibleStartTag)) + (b"<!--", self.handleComment), + (b"<meta", self.handleMeta), + (b"</", self.handlePossibleEndTag), + (b"<!", self.handleOther), + (b"<?", self.handleOther), + (b"<", self.handlePossibleStartTag)) for byte in self.data: keepParsing = True for key, method in methodDispatch: if self.data.matchBytes(key): try: - keepParsing = method() + keepParsing = method() break except StopIteration: - keepParsing=False + keepParsing = False break if not keepParsing: break - + return self.encoding def handleComment(self): """Skip over comments""" - return self.data.jumpTo("-->") + return self.data.jumpTo(b"-->") def handleMeta(self): if self.data.currentByte not in spaceCharactersBytes: - #if we have <meta not followed by a space so just keep going + # if we have <meta not followed by a space so just keep going return True - #We have a valid meta element we want to search for attributes + # We have a valid meta element we want to search for attributes + hasPragma = False + pendingEncoding = None while True: - #Try to find the next attribute after the current position + # Try to find the next attribute after the current position attr = self.getAttribute() if attr is None: return True else: - if attr[0] == "charset": + if attr[0] == b"http-equiv": + hasPragma = attr[1] == b"content-type" + if hasPragma and pendingEncoding is not None: + self.encoding = pendingEncoding + return False + elif attr[0] == b"charset": tentativeEncoding = attr[1] codec = codecName(tentativeEncoding) if codec is not None: self.encoding = codec return False - elif attr[0] == "content": + elif attr[0] == b"content": contentParser = ContentAttrParser(EncodingBytes(attr[1])) tentativeEncoding = contentParser.parse() - codec = codecName(tentativeEncoding) - if codec is not None: - self.encoding = codec - return False + if tentativeEncoding is not None: + codec = codecName(tentativeEncoding) + if codec is not None: + if hasPragma: + self.encoding = codec + return False + else: + pendingEncoding = codec def handlePossibleStartTag(self): return self.handlePossibleTag(False) def handlePossibleEndTag(self): - self.data.next() + next(self.data) return self.handlePossibleTag(True) def handlePossibleTag(self, endTag): data = self.data if data.currentByte not in asciiLettersBytes: - #If the next byte is not an ascii letter either ignore this - #fragment (possible start tag case) or treat it according to - #handleOther + # If the next byte is not an ascii letter either ignore this + # fragment (possible start tag case) or treat it according to + # handleOther if endTag: data.previous() self.handleOther() return True - + c = data.skipUntil(spacesAngleBrackets) - if c == "<": - #return to the first step in the overall "two step" algorithm - #reprocessing the < byte + if c == b"<": + # return to the first step in the overall "two step" algorithm + # reprocessing the < byte data.previous() else: - #Read all attributes + # Read all attributes attr = self.getAttribute() while attr is not None: attr = self.getAttribute() return True def handleOther(self): - return self.data.jumpTo(">") + return self.data.jumpTo(b">") def getAttribute(self): - """Return a name,value pair for the next attribute in the stream, + """Return a name,value pair for the next attribute in the stream, if one is found, or None""" data = self.data # Step 1 (skip chars) - c = data.skip(spaceCharactersBytes | frozenset("/")) + c = data.skip(spaceCharactersBytes | frozenset([b"/"])) + assert c is None or len(c) == 1 # Step 2 - if c in (">", None): + if c in (b">", None): return None # Step 3 attrName = [] attrValue = [] - #Step 4 attribute name + # Step 4 attribute name while True: - if c == "=" and attrName: + if c == b"=" and attrName: break elif c in spaceCharactersBytes: - #Step 6! + # Step 6! c = data.skip() - c = data.next() break - elif c in ("/", ">"): - return "".join(attrName), "" + elif c in (b"/", b">"): + return b"".join(attrName), b"" elif c in asciiUppercaseBytes: attrName.append(c.lower()) - elif c == None: + elif c is None: return None else: attrName.append(c) - #Step 5 - c = data.next() - #Step 7 - if c != "=": + # Step 5 + c = next(data) + # Step 7 + if c != b"=": data.previous() - return "".join(attrName), "" - #Step 8 - data.next() - #Step 9 + return b"".join(attrName), b"" + # Step 8 + next(data) + # Step 9 c = data.skip() - #Step 10 - if c in ("'", '"'): - #10.1 + # Step 10 + if c in (b"'", b'"'): + # 10.1 quoteChar = c while True: - #10.2 - c = data.next() - #10.3 + # 10.2 + c = next(data) + # 10.3 if c == quoteChar: - data.next() - return "".join(attrName), "".join(attrValue) - #10.4 + next(data) + return b"".join(attrName), b"".join(attrValue) + # 10.4 elif c in asciiUppercaseBytes: attrValue.append(c.lower()) - #10.5 + # 10.5 else: attrValue.append(c) - elif c == ">": - return "".join(attrName), "" + elif c == b">": + return b"".join(attrName), b"" elif c in asciiUppercaseBytes: attrValue.append(c.lower()) elif c is None: @@ -724,9 +816,9 @@ class EncodingParser(object): attrValue.append(c) # Step 11 while True: - c = data.next() + c = next(data) if c in spacesAngleBrackets: - return "".join(attrName), "".join(attrValue) + return b"".join(attrName), b"".join(attrValue) elif c in asciiUppercaseBytes: attrValue.append(c.lower()) elif c is None: @@ -737,21 +829,23 @@ class EncodingParser(object): class ContentAttrParser(object): def __init__(self, data): + assert isinstance(data, bytes) self.data = data + def parse(self): try: - #Check if the attr name is charset - #otherwise return - self.data.jumpTo("charset") + # Check if the attr name is charset + # otherwise return + self.data.jumpTo(b"charset") self.data.position += 1 self.data.skip() - if not self.data.currentByte == "=": - #If there is no = sign keep looking for attrs + if not self.data.currentByte == b"=": + # If there is no = sign keep looking for attrs return None self.data.position += 1 self.data.skip() - #Look for an encoding between matching quote marks - if self.data.currentByte in ('"', "'"): + # Look for an encoding between matching quote marks + if self.data.currentByte in (b'"', b"'"): quoteMark = self.data.currentByte self.data.position += 1 oldPosition = self.data.position @@ -760,13 +854,13 @@ class ContentAttrParser(object): else: return None else: - #Unquoted value + # Unquoted value oldPosition = self.data.position try: self.data.skipUntil(spaceCharactersBytes) return self.data[oldPosition:self.data.position] except StopIteration: - #Return the whole remaining value + # Return the whole remaining value return self.data[oldPosition:] except StopIteration: return None @@ -775,7 +869,12 @@ class ContentAttrParser(object): def codecName(encoding): """Return the python codec name corresponding to an encoding or None if the string doesn't correspond to a valid encoding.""" - if (encoding is not None and type(encoding) in types.StringTypes): + if isinstance(encoding, bytes): + try: + encoding = encoding.decode("ascii") + except UnicodeDecodeError: + return None + if encoding: canonicalName = ascii_punctuation_re.sub("", encoding).lower() return encodings.get(canonicalName, None) else: diff --git a/libs/html5lib/sanitizer.py b/libs/html5lib/sanitizer.py index ae4c7d83..71dc5212 100644 --- a/libs/html5lib/sanitizer.py +++ b/libs/html5lib/sanitizer.py @@ -1,142 +1,145 @@ +from __future__ import absolute_import, division, unicode_literals + import re from xml.sax.saxutils import escape, unescape -from tokenizer import HTMLTokenizer -from constants import tokenTypes +from .tokenizer import HTMLTokenizer +from .constants import tokenTypes + class HTMLSanitizerMixin(object): """ sanitization of XHTML+MathML+SVG and of inline style attributes.""" acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', - 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', - 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', - 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', - 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', - 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', - 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', - 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', - 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', - 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', - 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', - 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] - + 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', + 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', + 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', + 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', + 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', + 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', + 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', + 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', + 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', + 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', + 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', + 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video'] + mathml_elements = ['maction', 'math', 'merror', 'mfrac', 'mi', - 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', - 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', - 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', - 'munderover', 'none'] - + 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', + 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', + 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', + 'munderover', 'none'] + svg_elements = ['a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', - 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', - 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] - + 'animateTransform', 'clipPath', 'circle', 'defs', 'desc', 'ellipse', + 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', + 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', + 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', + 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] + acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', - 'background', 'balance', 'bgcolor', 'bgproperties', 'border', - 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', - 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', - 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', - 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', - 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', - 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', - 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', - 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', - 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', - 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', - 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', - 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', - 'optimum', 'pattern', 'ping', 'point-size', 'prompt', 'pqg', - 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', - 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', - 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', - 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', - 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', - 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', - 'width', 'wrap', 'xml:lang'] + 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', + 'background', 'balance', 'bgcolor', 'bgproperties', 'border', + 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', + 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', + 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', + 'cols', 'colspan', 'compact', 'contenteditable', 'controls', 'coords', + 'data', 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', + 'delay', 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', + 'face', 'for', 'form', 'frame', 'galleryimg', 'gutter', 'headers', + 'height', 'hidefocus', 'hidden', 'high', 'href', 'hreflang', 'hspace', + 'icon', 'id', 'inputmode', 'ismap', 'keytype', 'label', 'leftspacing', + 'lang', 'list', 'longdesc', 'loop', 'loopcount', 'loopend', + 'loopstart', 'low', 'lowsrc', 'max', 'maxlength', 'media', 'method', + 'min', 'multiple', 'name', 'nohref', 'noshade', 'nowrap', 'open', + 'optimum', 'pattern', 'ping', 'point-size', 'poster', 'pqg', 'preload', + 'prompt', 'radiogroup', 'readonly', 'rel', 'repeat-max', 'repeat-min', + 'replace', 'required', 'rev', 'rightspacing', 'rows', 'rowspan', + 'rules', 'scope', 'selected', 'shape', 'size', 'span', 'src', 'start', + 'step', 'style', 'summary', 'suppress', 'tabindex', 'target', + 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', + 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', + 'width', 'wrap', 'xml:lang'] mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign', - 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', - 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', - 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', - 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', - 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', - 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', - 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', - 'xlink:type', 'xmlns', 'xmlns:xlink'] - - svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', - 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', - 'fill-opacity', 'fill-rule', 'font-family', 'font-size', - 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', - 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', - 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', - 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', - 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', - 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', - 'opacity', 'orient', 'origin', 'overline-position', - 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', - 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', - 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', - 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', - 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', - 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', - 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', - 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', - 'transform', 'type', 'u1', 'u2', 'underline-position', - 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', - 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', - 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', - 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', - 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', - 'y1', 'y2', 'zoomAndPan'] + 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', + 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', + 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', + 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', + 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', + 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', + 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', + 'xlink:type', 'xmlns', 'xmlns:xlink'] - attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', - 'xlink:href', 'xml:base'] + svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', + 'arabic-form', 'ascent', 'attributeName', 'attributeType', + 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', + 'class', 'clip-path', 'color', 'color-rendering', 'content', 'cx', + 'cy', 'd', 'dx', 'dy', 'descent', 'display', 'dur', 'end', 'fill', + 'fill-opacity', 'fill-rule', 'font-family', 'font-size', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', 'from', + 'fx', 'fy', 'g1', 'g2', 'glyph-name', 'gradientUnits', 'hanging', + 'height', 'horiz-adv-x', 'horiz-origin-x', 'id', 'ideographic', 'k', + 'keyPoints', 'keySplines', 'keyTimes', 'lang', 'marker-end', + 'marker-mid', 'marker-start', 'markerHeight', 'markerUnits', + 'markerWidth', 'mathematical', 'max', 'min', 'name', 'offset', + 'opacity', 'orient', 'origin', 'overline-position', + 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', + 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', + 'repeatDur', 'requiredExtensions', 'requiredFeatures', 'restart', + 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', 'stop-color', + 'stop-opacity', 'strikethrough-position', 'strikethrough-thickness', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', + 'stroke-width', 'systemLanguage', 'target', 'text-anchor', 'to', + 'transform', 'type', 'u1', 'u2', 'underline-position', + 'underline-thickness', 'unicode', 'unicode-range', 'units-per-em', + 'values', 'version', 'viewBox', 'visibility', 'width', 'widths', 'x', + 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', + 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', + 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', + 'y1', 'y2', 'zoomAndPan'] + + attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', + 'xlink:href', 'xml:base'] svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill', - 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', - 'mask', 'stroke'] + 'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end', + 'mask', 'stroke'] svg_allow_local_href = ['altGlyph', 'animate', 'animateColor', - 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', - 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', - 'set', 'use'] - + 'animateMotion', 'animateTransform', 'cursor', 'feImage', 'filter', + 'linearGradient', 'pattern', 'radialGradient', 'textpath', 'tref', + 'set', 'use'] + acceptable_css_properties = ['azimuth', 'background-color', - 'border-bottom-color', 'border-collapse', 'border-color', - 'border-left-color', 'border-right-color', 'border-top-color', 'clear', - 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', - 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', - 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', - 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', - 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', - 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', - 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', - 'white-space', 'width'] - + 'border-bottom-color', 'border-collapse', 'border-color', + 'border-left-color', 'border-right-color', 'border-top-color', 'clear', + 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', + 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', + 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', + 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', + 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', + 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', + 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', + 'white-space', 'width'] + acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue', - 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', - 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', - 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', - 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', - 'transparent', 'underline', 'white', 'yellow'] - - acceptable_svg_properties = [ 'fill', 'fill-opacity', 'fill-rule', - 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', - 'stroke-opacity'] - - acceptable_protocols = [ 'ed2k', 'ftp', 'http', 'https', 'irc', - 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', - 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', - 'ssh', 'sftp', 'rtsp', 'afs' ] - + 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', + 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', + 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', + 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', + 'transparent', 'underline', 'white', 'yellow'] + + acceptable_svg_properties = ['fill', 'fill-opacity', 'fill-rule', + 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', + 'stroke-opacity'] + + acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc', + 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', + 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', + 'ssh', 'sftp', 'rtsp', 'afs'] + # subclasses may define their own versions of these constants allowed_elements = acceptable_elements + mathml_elements + svg_elements allowed_attributes = acceptable_attributes + mathml_attributes + svg_attributes @@ -160,94 +163,104 @@ class HTMLSanitizerMixin(object): # accommodate filters which use token_type differently token_type = token["type"] - if token_type in tokenTypes.keys(): - token_type = tokenTypes[token_type] + if token_type in list(tokenTypes.keys()): + token_type = tokenTypes[token_type] - if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], - tokenTypes["EmptyTag"]): + if token_type in (tokenTypes["StartTag"], tokenTypes["EndTag"], + tokenTypes["EmptyTag"]): if token["name"] in self.allowed_elements: - if token.has_key("data"): - attrs = dict([(name,val) for name,val in - token["data"][::-1] - if name in self.allowed_attributes]) - for attr in self.attr_val_is_uri: - if not attrs.has_key(attr): - continue - val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', - unescape(attrs[attr])).lower() - #remove replacement characters from unescaped characters - val_unescaped = val_unescaped.replace(u"\ufffd", "") - if (re.match("^[a-z0-9][-+.a-z0-9]*:",val_unescaped) and - (val_unescaped.split(':')[0] not in - self.allowed_protocols)): - del attrs[attr] - for attr in self.svg_attr_val_allows_ref: - if attr in attrs: - attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', - ' ', - unescape(attrs[attr])) - if (token["name"] in self.svg_allow_local_href and - 'xlink:href' in attrs and re.search('^\s*[^#\s].*', - attrs['xlink:href'])): - del attrs['xlink:href'] - if attrs.has_key('style'): - attrs['style'] = self.sanitize_css(attrs['style']) - token["data"] = [[name,val] for name,val in attrs.items()] - return token + return self.allowed_token(token, token_type) else: - if token_type == tokenTypes["EndTag"]: - token["data"] = "</%s>" % token["name"] - elif token["data"]: - attrs = ''.join([' %s="%s"' % (k,escape(v)) for k,v in token["data"]]) - token["data"] = "<%s%s>" % (token["name"],attrs) - else: - token["data"] = "<%s>" % token["name"] - if token.get("selfClosing"): - token["data"]=token["data"][:-1] + "/>" - - if token["type"] in tokenTypes.keys(): - token["type"] = "Characters" - else: - token["type"] = tokenTypes["Characters"] - - del token["name"] - return token + return self.disallowed_token(token, token_type) elif token_type == tokenTypes["Comment"]: pass else: return token + def allowed_token(self, token, token_type): + if "data" in token: + attrs = dict([(name, val) for name, val in + token["data"][::-1] + if name in self.allowed_attributes]) + for attr in self.attr_val_is_uri: + if attr not in attrs: + continue + val_unescaped = re.sub("[`\000-\040\177-\240\s]+", '', + unescape(attrs[attr])).lower() + # remove replacement characters from unescaped characters + val_unescaped = val_unescaped.replace("\ufffd", "") + if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and + (val_unescaped.split(':')[0] not in + self.allowed_protocols)): + del attrs[attr] + for attr in self.svg_attr_val_allows_ref: + if attr in attrs: + attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', + ' ', + unescape(attrs[attr])) + if (token["name"] in self.svg_allow_local_href and + 'xlink:href' in attrs and re.search('^\s*[^#\s].*', + attrs['xlink:href'])): + del attrs['xlink:href'] + if 'style' in attrs: + attrs['style'] = self.sanitize_css(attrs['style']) + token["data"] = [[name, val] for name, val in list(attrs.items())] + return token + + def disallowed_token(self, token, token_type): + if token_type == tokenTypes["EndTag"]: + token["data"] = "</%s>" % token["name"] + elif token["data"]: + attrs = ''.join([' %s="%s"' % (k, escape(v)) for k, v in token["data"]]) + token["data"] = "<%s%s>" % (token["name"], attrs) + else: + token["data"] = "<%s>" % token["name"] + if token.get("selfClosing"): + token["data"] = token["data"][:-1] + "/>" + + if token["type"] in list(tokenTypes.keys()): + token["type"] = "Characters" + else: + token["type"] = tokenTypes["Characters"] + + del token["name"] + return token + def sanitize_css(self, style): # disallow urls - style=re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ',style) + style = re.compile('url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style) # gauntlet - if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): return '' - if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): return '' + if not re.match("""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): + return '' + if not re.match("^\s*([-\w]+\s*:[^:;]*(;\s*|$))*$", style): + return '' clean = [] - for prop,value in re.findall("([-\w]+)\s*:\s*([^:;]*)",style): - if not value: continue - if prop.lower() in self.allowed_css_properties: - clean.append(prop + ': ' + value + ';') - elif prop.split('-')[0].lower() in ['background','border','margin', - 'padding']: - for keyword in value.split(): - if not keyword in self.acceptable_css_keywords and \ - not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$",keyword): - break - else: - clean.append(prop + ': ' + value + ';') - elif prop.lower() in self.allowed_svg_properties: - clean.append(prop + ': ' + value + ';') + for prop, value in re.findall("([-\w]+)\s*:\s*([^:;]*)", style): + if not value: + continue + if prop.lower() in self.allowed_css_properties: + clean.append(prop + ': ' + value + ';') + elif prop.split('-')[0].lower() in ['background', 'border', 'margin', + 'padding']: + for keyword in value.split(): + if not keyword in self.acceptable_css_keywords and \ + not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword): + break + else: + clean.append(prop + ': ' + value + ';') + elif prop.lower() in self.allowed_svg_properties: + clean.append(prop + ': ' + value + ';') return ' '.join(clean) + class HTMLSanitizer(HTMLTokenizer, HTMLSanitizerMixin): def __init__(self, stream, encoding=None, parseMeta=True, useChardet=True, lowercaseElementName=False, lowercaseAttrName=False, parser=None): - #Change case matching defaults as we only output lowercase html anyway - #This solution doesn't seem ideal... + # Change case matching defaults as we only output lowercase html anyway + # This solution doesn't seem ideal... HTMLTokenizer.__init__(self, stream, encoding, parseMeta, useChardet, lowercaseElementName, lowercaseAttrName, parser=parser) diff --git a/libs/html5lib/serializer/__init__.py b/libs/html5lib/serializer/__init__.py index 1b746655..8380839a 100644 --- a/libs/html5lib/serializer/__init__.py +++ b/libs/html5lib/serializer/__init__.py @@ -1,17 +1,16 @@ +from __future__ import absolute_import, division, unicode_literals -from html5lib import treewalkers +from .. import treewalkers -from htmlserializer import HTMLSerializer -from xhtmlserializer import XHTMLSerializer +from .htmlserializer import HTMLSerializer -def serialize(input, tree="simpletree", format="html", encoding=None, + +def serialize(input, tree="etree", format="html", encoding=None, **serializer_opts): # XXX: Should we cache this? - walker = treewalkers.getTreeWalker(tree) + walker = treewalkers.getTreeWalker(tree) if format == "html": s = HTMLSerializer(**serializer_opts) - elif format == "xhtml": - s = XHTMLSerializer(**serializer_opts) else: - raise ValueError, "type must be either html or xhtml" + raise ValueError("type must be html") return s.render(walker(input), encoding) diff --git a/libs/html5lib/serializer/htmlserializer.py b/libs/html5lib/serializer/htmlserializer.py index 8dd0a815..412a5a22 100644 --- a/libs/html5lib/serializer/htmlserializer.py +++ b/libs/html5lib/serializer/htmlserializer.py @@ -1,18 +1,20 @@ -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import ImmutableSet as frozenset +from __future__ import absolute_import, division, unicode_literals +from six import text_type import gettext _ = gettext.gettext -from html5lib.constants import voidElements, booleanAttributes, spaceCharacters -from html5lib.constants import rcdataElements, entities, xmlEntities -from html5lib import utils +try: + from functools import reduce +except ImportError: + pass + +from ..constants import voidElements, booleanAttributes, spaceCharacters +from ..constants import rcdataElements, entities, xmlEntities +from .. import utils from xml.sax.saxutils import escape -spaceCharacters = u"".join(spaceCharacters) +spaceCharacters = "".join(spaceCharacters) try: from codecs import register_error, xmlcharrefreplace_errors @@ -21,24 +23,18 @@ except ImportError: else: unicode_encode_errors = "htmlentityreplace" - from html5lib.constants import entities - encode_entity_map = {} - is_ucs4 = len(u"\U0010FFFF") == 1 - for k, v in entities.items(): - #skip multi-character entities + is_ucs4 = len("\U0010FFFF") == 1 + for k, v in list(entities.items()): + # skip multi-character entities if ((is_ucs4 and len(v) > 1) or - (not is_ucs4 and len(v) > 2)): + (not is_ucs4 and len(v) > 2)): continue if v != "&": if len(v) == 2: v = utils.surrogatePairToCodepoint(v) else: - try: - v = ord(v) - except: - print v - raise + v = ord(v) if not v in encode_entity_map or k.islower(): # prefer < over < and similarly for &, >, etc. encode_entity_map[v] = k @@ -53,8 +49,8 @@ else: skip = False continue index = i + exc.start - if utils.isSurrogatePair(exc.object[index:min([exc.end, index+2])]): - codepoint = utils.surrogatePairToCodepoint(exc.object[index:index+2]) + if utils.isSurrogatePair(exc.object[index:min([exc.end, index + 2])]): + codepoint = utils.surrogatePairToCodepoint(exc.object[index:index + 2]) skip = True else: codepoint = ord(c) @@ -67,8 +63,8 @@ else: if not e.endswith(";"): res.append(";") else: - res.append("&#x%s;"%(hex(cp)[2:])) - return (u"".join(res), exc.end) + res.append("&#x%s;" % (hex(cp)[2:])) + return ("".join(res), exc.end) else: return xmlcharrefreplace_errors(exc) @@ -81,7 +77,7 @@ class HTMLSerializer(object): # attribute quoting options quote_attr_values = False - quote_char = u'"' + quote_char = '"' use_best_quote_char = True # tag syntax options @@ -96,15 +92,17 @@ class HTMLSerializer(object): resolve_entities = True # miscellaneous options + alphabetical_attributes = False inject_meta_charset = True strip_whitespace = False sanitize = False options = ("quote_attr_values", "quote_char", "use_best_quote_char", - "minimize_boolean_attributes", "use_trailing_solidus", - "space_before_trailing_solidus", "omit_optional_tags", - "strip_whitespace", "inject_meta_charset", "escape_lt_in_attrs", - "escape_rcdata", "resolve_entities", "sanitize") + "omit_optional_tags", "minimize_boolean_attributes", + "use_trailing_solidus", "space_before_trailing_solidus", + "escape_lt_in_attrs", "escape_rcdata", "resolve_entities", + "alphabetical_attributes", "inject_meta_charset", + "strip_whitespace", "sanitize") def __init__(self, **kwargs): """Initialize HTMLSerializer. @@ -147,10 +145,12 @@ class HTMLSerializer(object): See `html5lib user documentation`_ omit_optional_tags=True|False Omit start/end tags that are optional. + alphabetical_attributes=False|True + Reorder attributes to be in alphabetical order. .. _html5lib user documentation: http://code.google.com/p/html5lib/wiki/UserDocumentation """ - if kwargs.has_key('quote_char'): + if 'quote_char' in kwargs: self.use_best_quote_char = False for attr in self.options: setattr(self, attr, kwargs.get(attr, getattr(self, attr))) @@ -158,14 +158,14 @@ class HTMLSerializer(object): self.strict = False def encode(self, string): - assert(isinstance(string, unicode)) + assert(isinstance(string, text_type)) if self.encoding: return string.encode(self.encoding, unicode_encode_errors) else: return string def encodeStrict(self, string): - assert(isinstance(string, unicode)) + assert(isinstance(string, text_type)) if self.encoding: return string.encode(self.encoding, "strict") else: @@ -175,39 +175,46 @@ class HTMLSerializer(object): self.encoding = encoding in_cdata = False self.errors = [] + if encoding and self.inject_meta_charset: - from html5lib.filters.inject_meta_charset import Filter + from ..filters.inject_meta_charset import Filter treewalker = Filter(treewalker, encoding) - # XXX: WhitespaceFilter should be used before OptionalTagFilter + # WhitespaceFilter should be used before OptionalTagFilter # for maximum efficiently of this latter filter if self.strip_whitespace: - from html5lib.filters.whitespace import Filter + from ..filters.whitespace import Filter treewalker = Filter(treewalker) if self.sanitize: - from html5lib.filters.sanitizer import Filter + from ..filters.sanitizer import Filter treewalker = Filter(treewalker) if self.omit_optional_tags: - from html5lib.filters.optionaltags import Filter + from ..filters.optionaltags import Filter treewalker = Filter(treewalker) + # Alphabetical attributes must be last, as other filters + # could add attributes and alter the order + if self.alphabetical_attributes: + from ..filters.alphabeticalattributes import Filter + treewalker = Filter(treewalker) + for token in treewalker: type = token["type"] if type == "Doctype": - doctype = u"<!DOCTYPE %s" % token["name"] - + doctype = "<!DOCTYPE %s" % token["name"] + if token["publicId"]: - doctype += u' PUBLIC "%s"' % token["publicId"] + doctype += ' PUBLIC "%s"' % token["publicId"] elif token["systemId"]: - doctype += u" SYSTEM" - if token["systemId"]: - if token["systemId"].find(u'"') >= 0: - if token["systemId"].find(u"'") >= 0: + doctype += " SYSTEM" + if token["systemId"]: + if token["systemId"].find('"') >= 0: + if token["systemId"].find("'") >= 0: self.serializeError(_("System identifer contains both single and double quote characters")) - quote_char = u"'" + quote_char = "'" else: - quote_char = u'"' - doctype += u" %s%s%s" % (quote_char, token["systemId"], quote_char) - - doctype += u">" + quote_char = '"' + doctype += " %s%s%s" % (quote_char, token["systemId"], quote_char) + + doctype += ">" yield self.encodeStrict(doctype) elif type in ("Characters", "SpaceCharacters"): @@ -220,41 +227,41 @@ class HTMLSerializer(object): elif type in ("StartTag", "EmptyTag"): name = token["name"] - yield self.encodeStrict(u"<%s" % name) + yield self.encodeStrict("<%s" % name) if name in rcdataElements and not self.escape_rcdata: in_cdata = True elif in_cdata: self.serializeError(_("Unexpected child element of a CDATA element")) - attributes = [] - for (attr_namespace,attr_name),attr_value in sorted(token["data"].items()): - #TODO: Add namespace support here + for (attr_namespace, attr_name), attr_value in token["data"].items(): + # TODO: Add namespace support here k = attr_name v = attr_value - yield self.encodeStrict(u' ') + yield self.encodeStrict(' ') yield self.encodeStrict(k) if not self.minimize_boolean_attributes or \ - (k not in booleanAttributes.get(name, tuple()) \ - and k not in booleanAttributes.get("", tuple())): - yield self.encodeStrict(u"=") + (k not in booleanAttributes.get(name, tuple()) + and k not in booleanAttributes.get("", tuple())): + yield self.encodeStrict("=") if self.quote_attr_values or not v: quote_attr = True else: - quote_attr = reduce(lambda x,y: x or (y in v), - spaceCharacters + u">\"'=", False) - v = v.replace(u"&", u"&") - if self.escape_lt_in_attrs: v = v.replace(u"<", u"<") + quote_attr = reduce(lambda x, y: x or (y in v), + spaceCharacters + ">\"'=", False) + v = v.replace("&", "&") + if self.escape_lt_in_attrs: + v = v.replace("<", "<") if quote_attr: quote_char = self.quote_char if self.use_best_quote_char: - if u"'" in v and u'"' not in v: - quote_char = u'"' - elif u'"' in v and u"'" not in v: - quote_char = u"'" - if quote_char == u"'": - v = v.replace(u"'", u"'") + if "'" in v and '"' not in v: + quote_char = '"' + elif '"' in v and "'" not in v: + quote_char = "'" + if quote_char == "'": + v = v.replace("'", "'") else: - v = v.replace(u'"', u""") + v = v.replace('"', """) yield self.encodeStrict(quote_char) yield self.encode(v) yield self.encodeStrict(quote_char) @@ -262,10 +269,10 @@ class HTMLSerializer(object): yield self.encode(v) if name in voidElements and self.use_trailing_solidus: if self.space_before_trailing_solidus: - yield self.encodeStrict(u" /") + yield self.encodeStrict(" /") else: - yield self.encodeStrict(u"/") - yield self.encode(u">") + yield self.encodeStrict("/") + yield self.encode(">") elif type == "EndTag": name = token["name"] @@ -273,13 +280,13 @@ class HTMLSerializer(object): in_cdata = False elif in_cdata: self.serializeError(_("Unexpected child element of a CDATA element")) - yield self.encodeStrict(u"</%s>" % name) + yield self.encodeStrict("</%s>" % name) elif type == "Comment": data = token["data"] if data.find("--") >= 0: self.serializeError(_("Comment contains --")) - yield self.encodeStrict(u"<!--%s-->" % token["data"]) + yield self.encodeStrict("<!--%s-->" % token["data"]) elif type == "Entity": name = token["name"] @@ -289,7 +296,7 @@ class HTMLSerializer(object): if self.resolve_entities and key not in xmlEntities: data = entities[key] else: - data = u"&%s;" % name + data = "&%s;" % name yield self.encodeStrict(data) else: @@ -297,9 +304,9 @@ class HTMLSerializer(object): def render(self, treewalker, encoding=None): if encoding: - return "".join(list(self.serialize(treewalker, encoding))) + return b"".join(list(self.serialize(treewalker, encoding))) else: - return u"".join(list(self.serialize(treewalker))) + return "".join(list(self.serialize(treewalker))) def serializeError(self, data="XXX ERROR MESSAGE NEEDED"): # XXX The idea is to make data mandatory. @@ -307,6 +314,7 @@ class HTMLSerializer(object): if self.strict: raise SerializeError + def SerializeError(Exception): """Error in serialized tree""" pass diff --git a/libs/html5lib/serializer/xhtmlserializer.py b/libs/html5lib/serializer/xhtmlserializer.py deleted file mode 100644 index 7fdce47b..00000000 --- a/libs/html5lib/serializer/xhtmlserializer.py +++ /dev/null @@ -1,9 +0,0 @@ -from htmlserializer import HTMLSerializer - -class XHTMLSerializer(HTMLSerializer): - quote_attr_values = True - minimize_boolean_attributes = False - use_trailing_solidus = True - escape_lt_in_attrs = True - omit_optional_tags = False - escape_rcdata = True diff --git a/libs/html5lib/tokenizer.py b/libs/html5lib/tokenizer.py index 7e9eca88..79774578 100644 --- a/libs/html5lib/tokenizer.py +++ b/libs/html5lib/tokenizer.py @@ -1,27 +1,25 @@ +from __future__ import absolute_import, division, unicode_literals + try: - frozenset + chr = unichr # flake8: noqa except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset -try: - from collections import deque -except ImportError: - from utils import deque - -from constants import spaceCharacters -from constants import entitiesWindows1252, entities -from constants import asciiLowercase, asciiLetters, asciiUpper2Lower -from constants import digits, hexDigits, EOF -from constants import tokenTypes, tagTokenTypes -from constants import replacementCharacters + pass -from inputstream import HTMLInputStream +from collections import deque + +from .constants import spaceCharacters +from .constants import entities +from .constants import asciiLetters, asciiUpper2Lower +from .constants import digits, hexDigits, EOF +from .constants import tokenTypes, tagTokenTypes +from .constants import replacementCharacters + +from .inputstream import HTMLInputStream + +from .trie import Trie + +entitiesTrie = Trie(entities) -# Group entities by their first character, for faster lookups -entitiesByFirstChar = {} -for e in entities: - entitiesByFirstChar.setdefault(e[0], []).append(e) class HTMLTokenizer(object): """ This class takes care of tokenizing HTML. @@ -42,10 +40,10 @@ class HTMLTokenizer(object): self.stream = HTMLInputStream(stream, encoding, parseMeta, useChardet) self.parser = parser - #Perform case conversions? + # Perform case conversions? self.lowercaseElementName = lowercaseElementName self.lowercaseAttrName = lowercaseAttrName - + # Setup the initial tokenizer state self.escapeFlag = False self.lastFourChars = [] @@ -100,78 +98,79 @@ class HTMLTokenizer(object): if charAsInt in replacementCharacters: char = replacementCharacters[charAsInt] self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", - "datavars": {"charAsInt": charAsInt}}) - elif ((0xD800 <= charAsInt <= 0xDFFF) or + "illegal-codepoint-for-numeric-entity", + "datavars": {"charAsInt": charAsInt}}) + elif ((0xD800 <= charAsInt <= 0xDFFF) or (charAsInt > 0x10FFFF)): - char = u"\uFFFD" + char = "\uFFFD" self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", - "datavars": {"charAsInt": charAsInt}}) + "illegal-codepoint-for-numeric-entity", + "datavars": {"charAsInt": charAsInt}}) else: - #Should speed up this check somehow (e.g. move the set to a constant) - if ((0x0001 <= charAsInt <= 0x0008) or - (0x000E <= charAsInt <= 0x001F) or - (0x007F <= charAsInt <= 0x009F) or - (0xFDD0 <= charAsInt <= 0xFDEF) or - charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE, + # Should speed up this check somehow (e.g. move the set to a constant) + if ((0x0001 <= charAsInt <= 0x0008) or + (0x000E <= charAsInt <= 0x001F) or + (0x007F <= charAsInt <= 0x009F) or + (0xFDD0 <= charAsInt <= 0xFDEF) or + charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE, - 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, + 0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE, 0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE, - 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE, - 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, - 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, + 0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE, + 0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE, + 0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE, 0xFFFFF, 0x10FFFE, 0x10FFFF])): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "illegal-codepoint-for-numeric-entity", + "illegal-codepoint-for-numeric-entity", "datavars": {"charAsInt": charAsInt}}) try: # Try/except needed as UCS-2 Python builds' unichar only works # within the BMP. - char = unichr(charAsInt) + char = chr(charAsInt) except ValueError: - char = eval("u'\\U%08x'" % charAsInt) + v = charAsInt - 0x10000 + char = chr(0xD800 | (v >> 10)) + chr(0xDC00 | (v & 0x3FF)) # Discard the ; if present. Otherwise, put it back on the queue and # invoke parseError on parser. - if c != u";": + if c != ";": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "numeric-entity-without-semicolon"}) + "numeric-entity-without-semicolon"}) self.stream.unget(c) return char def consumeEntity(self, allowedChar=None, fromAttribute=False): # Initialise to the default output for when no entity is matched - output = u"&" + output = "&" charStack = [self.stream.char()] - if (charStack[0] in spaceCharacters or charStack[0] in (EOF, u"<", u"&") - or (allowedChar is not None and allowedChar == charStack[0])): + if (charStack[0] in spaceCharacters or charStack[0] in (EOF, "<", "&") + or (allowedChar is not None and allowedChar == charStack[0])): self.stream.unget(charStack[0]) - elif charStack[0] == u"#": + elif charStack[0] == "#": # Read the next character to see if it's hex or decimal hex = False charStack.append(self.stream.char()) - if charStack[-1] in (u"x", u"X"): + if charStack[-1] in ("x", "X"): hex = True charStack.append(self.stream.char()) # charStack[-1] should be the first digit if (hex and charStack[-1] in hexDigits) \ - or (not hex and charStack[-1] in digits): + or (not hex and charStack[-1] in digits): # At least one digit found, so consume the whole number self.stream.unget(charStack[-1]) output = self.consumeNumberEntity(hex) else: # No digits found self.tokenQueue.append({"type": tokenTypes["ParseError"], - "data": "expected-numeric-entity"}) + "data": "expected-numeric-entity"}) self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) else: # At this point in the process might have named entity. Entities @@ -179,46 +178,40 @@ class HTMLTokenizer(object): # # Consume characters and compare to these to a substring of the # entity names in the list until the substring no longer matches. - filteredEntityList = entitiesByFirstChar.get(charStack[0], []) - - def entitiesStartingWith(name): - return [e for e in filteredEntityList if e.startswith(name)] - - while (charStack[-1] is not EOF and - entitiesStartingWith("".join(charStack))): + while (charStack[-1] is not EOF): + if not entitiesTrie.has_keys_with_prefix("".join(charStack)): + break charStack.append(self.stream.char()) # At this point we have a string that starts with some characters # that may match an entity - entityName = None - # Try to find the longest entity the string will match to take care # of ¬i for instance. - for entityLength in xrange(len(charStack)-1, 1, -1): - possibleEntityName = "".join(charStack[:entityLength]) - if possibleEntityName in entities: - entityName = possibleEntityName - break + try: + entityName = entitiesTrie.longest_prefix("".join(charStack[:-1])) + entityLength = len(entityName) + except KeyError: + entityName = None if entityName is not None: if entityName[-1] != ";": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "named-entity-without-semicolon"}) + "named-entity-without-semicolon"}) if (entityName[-1] != ";" and fromAttribute and (charStack[entityLength] in asciiLetters or charStack[entityLength] in digits or - charStack[entityLength] == "=")): + charStack[entityLength] == "=")): self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) else: output = entities[entityName] self.stream.unget(charStack.pop()) - output += u"".join(charStack[entityLength:]) + output += "".join(charStack[entityLength:]) else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-named-entity"}) + "expected-named-entity"}) self.stream.unget(charStack.pop()) - output = u"&" + u"".join(charStack) + output = "&" + "".join(charStack) if fromAttribute: self.currentToken["data"][-1][1] += output @@ -246,28 +239,26 @@ class HTMLTokenizer(object): token["name"] = token["name"].translate(asciiUpper2Lower) if token["type"] == tokenTypes["EndTag"]: if token["data"]: - self.tokenQueue.append({"type":tokenTypes["ParseError"], - "data":"attributes-in-end-tag"}) + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "attributes-in-end-tag"}) if token["selfClosing"]: - self.tokenQueue.append({"type":tokenTypes["ParseError"], - "data":"self-closing-flag-on-end-tag"}) + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "self-closing-flag-on-end-tag"}) self.tokenQueue.append(token) self.state = self.dataState - # Below are the various tokenizer states worked out. - def dataState(self): data = self.stream.char() if data == "&": self.state = self.entityDataState elif data == "<": self.state = self.tagOpenState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], - "data":"invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\u0000"}) + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], + "data": "invalid-codepoint"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\u0000"}) elif data is EOF: # Tokenization ends. return False @@ -276,21 +267,21 @@ class HTMLTokenizer(object): # state". At that point spaceCharacters are important so they are # emitted separately. self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data": - data + self.stream.charsUntil(spaceCharacters, True)}) + data + self.stream.charsUntil(spaceCharacters, True)}) # No need to update lastFourChars here, since the first space will # have already been appended to lastFourChars and will have broken # any <!-- or --> sequences else: - chars = self.stream.charsUntil((u"&", u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("&", "<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True def entityDataState(self): self.consumeEntity() self.state = self.dataState return True - + def rcdataState(self): data = self.stream.char() if data == "&": @@ -300,113 +291,113 @@ class HTMLTokenizer(object): elif data == EOF: # Tokenization ends. return False - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data in spaceCharacters: # Directly after emitting a token you switch back to the "data # state". At that point spaceCharacters are important so they are # emitted separately. self.tokenQueue.append({"type": tokenTypes["SpaceCharacters"], "data": - data + self.stream.charsUntil(spaceCharacters, True)}) + data + self.stream.charsUntil(spaceCharacters, True)}) # No need to update lastFourChars here, since the first space will # have already been appended to lastFourChars and will have broken # any <!-- or --> sequences else: - chars = self.stream.charsUntil((u"&", u"<")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("&", "<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True def characterReferenceInRcdata(self): self.consumeEntity() self.state = self.rcdataState return True - + def rawtextState(self): data = self.stream.char() if data == "<": self.state = self.rawtextLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: # Tokenization ends. return False else: - chars = self.stream.charsUntil((u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def scriptDataState(self): data = self.stream.char() if data == "<": self.state = self.scriptDataLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: # Tokenization ends. return False else: - chars = self.stream.charsUntil((u"<", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def plaintextState(self): data = self.stream.char() if data == EOF: # Tokenization ends. return False - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + self.stream.charsUntil(u"\u0000")}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + self.stream.charsUntil("\u0000")}) return True def tagOpenState(self): data = self.stream.char() - if data == u"!": + if data == "!": self.state = self.markupDeclarationOpenState - elif data == u"/": + elif data == "/": self.state = self.closeTagOpenState elif data in asciiLetters: - self.currentToken = {"type": tokenTypes["StartTag"], + self.currentToken = {"type": tokenTypes["StartTag"], "name": data, "data": [], "selfClosing": False, "selfClosingAcknowledged": False} self.state = self.tagNameState - elif data == u">": + elif data == ">": # XXX In theory it could be something besides a tag name. But # do we really care? self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name-but-got-right-bracket"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<>"}) + "expected-tag-name-but-got-right-bracket"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<>"}) self.state = self.dataState - elif data == u"?": + elif data == "?": # XXX In theory it could be something besides a tag name. But # do we really care? self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name-but-got-question-mark"}) + "expected-tag-name-but-got-question-mark"}) self.stream.unget(data) self.state = self.bogusCommentState else: # XXX self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-tag-name"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + "expected-tag-name"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.dataState return True @@ -415,22 +406,22 @@ class HTMLTokenizer(object): data = self.stream.char() if data in asciiLetters: self.currentToken = {"type": tokenTypes["EndTag"], "name": data, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.tagNameState - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-right-bracket"}) + "expected-closing-tag-but-got-right-bracket"}) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-eof"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + "expected-closing-tag-but-got-eof"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.state = self.dataState else: # XXX data can be _'_... self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-closing-tag-but-got-char", - "datavars": {"data": data}}) + "expected-closing-tag-but-got-char", + "datavars": {"data": data}}) self.stream.unget(data) self.state = self.bogusCommentState return True @@ -439,229 +430,229 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-tag-name"}) + "eof-in-tag-name"}) self.state = self.dataState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] += u"\uFFFD" + self.currentToken["name"] += "\uFFFD" else: self.currentToken["name"] += data # (Don't use charsUntil here, because tag names are # very short and it's faster to not do anything fancy) return True - + def rcdataLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.rcdataEndTagOpenState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.rcdataState return True - + def rcdataEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.rcdataEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.rcdataState return True - + def rcdataEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.rcdataState return True - + def rawtextLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.rawtextEndTagOpenState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.rawtextState return True - + def rawtextEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.rawtextEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.rawtextState return True - + def rawtextEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.rawtextState return True - + def scriptDataLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.scriptDataEndTagOpenState elif data == "!": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<!"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<!"}) self.state = self.scriptDataEscapeStartState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer += data self.state = self.scriptDataEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapeStartState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapeStartDashState else: self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapeStartDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashDashState else: self.stream.unget(data) self.state = self.scriptDataState return True - + def scriptDataEscapedState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashState elif data == "<": self.state = self.scriptDataEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: self.state = self.dataState else: - chars = self.stream.charsUntil((u"<", u"-", u"\u0000")) - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": - data + chars}) + chars = self.stream.charsUntil(("<", "-", "\u0000")) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": + data + chars}) return True - + def scriptDataEscapedDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataEscapedDashDashState elif data == "<": self.state = self.scriptDataEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataEscapedState elif data == EOF: self.state = self.dataState @@ -669,21 +660,21 @@ class HTMLTokenizer(object): self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedDashDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) elif data == "<": self.state = self.scriptDataEscapedLessThanSignState elif data == ">": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u">"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"}) self.state = self.scriptDataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataEscapedState elif data == EOF: self.state = self.dataState @@ -691,61 +682,61 @@ class HTMLTokenizer(object): self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedLessThanSignState(self): data = self.stream.char() if data == "/": self.temporaryBuffer = "" self.state = self.scriptDataEscapedEndTagOpenState elif data in asciiLetters: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<" + data}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<" + data}) self.temporaryBuffer = data self.state = self.scriptDataDoubleEscapeStartState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedEndTagOpenState(self): data = self.stream.char() if data in asciiLetters: self.temporaryBuffer = data self.state = self.scriptDataEscapedEndTagNameState else: - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"</"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "</"}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataEscapedEndTagNameState(self): appropriate = self.currentToken and self.currentToken["name"].lower() == self.temporaryBuffer.lower() data = self.stream.char() if data in spaceCharacters and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.beforeAttributeNameState elif data == "/" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.state = self.selfClosingStartTagState elif data == ">" and appropriate: self.currentToken = {"type": tokenTypes["EndTag"], "name": self.temporaryBuffer, - "data": [], "selfClosing":False} + "data": [], "selfClosing": False} self.emitCurrentToken() self.state = self.dataState elif data in asciiLetters: self.temporaryBuffer += data else: self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"</" + self.temporaryBuffer}) + "data": "</" + self.temporaryBuffer}) self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataDoubleEscapeStartState(self): data = self.stream.char() if data in (spaceCharacters | frozenset(("/", ">"))): @@ -761,87 +752,87 @@ class HTMLTokenizer(object): self.stream.unget(data) self.state = self.scriptDataEscapedState return True - + def scriptDataDoubleEscapedState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataDoubleEscapedDashState elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) return True - + def scriptDataDoubleEscapedDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) self.state = self.scriptDataDoubleEscapedDashDashState elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataDoubleEscapedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataDoubleEscapedState return True - - def scriptDataDoubleEscapedDashState(self): + + def scriptDataDoubleEscapedDashDashState(self): data = self.stream.char() if data == "-": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"-"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "-"}) elif data == "<": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"<"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "<"}) self.state = self.scriptDataDoubleEscapedLessThanSignState elif data == ">": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u">"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": ">"}) self.state = self.scriptDataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.tokenQueue.append({"type": tokenTypes["Characters"], - "data": u"\uFFFD"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], + "data": "\uFFFD"}) self.state = self.scriptDataDoubleEscapedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-script-in-script"}) + "eof-in-script-in-script"}) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.scriptDataDoubleEscapedState return True - + def scriptDataDoubleEscapedLessThanSignState(self): data = self.stream.char() if data == "/": - self.tokenQueue.append({"type": tokenTypes["Characters"], "data": u"/"}) + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": "/"}) self.temporaryBuffer = "" self.state = self.scriptDataDoubleEscapeEndState else: self.stream.unget(data) self.state = self.scriptDataDoubleEscapedState return True - + def scriptDataDoubleEscapeEndState(self): data = self.stream.char() if data in (spaceCharacters | frozenset(("/", ">"))): @@ -865,23 +856,23 @@ class HTMLTokenizer(object): elif data in asciiLetters: self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data in (u"'", u'"', u"=", u"<"): + elif data in ("'", '"', "=", "<"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-in-attribute-name"}) + "invalid-character-in-attribute-name"}) self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"].append([u"\uFFFD", ""]) + self.currentToken["data"].append(["\uFFFD", ""]) self.state = self.attributeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-name-but-got-eof"}) + "expected-attribute-name-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"].append([data, ""]) @@ -892,34 +883,34 @@ class HTMLTokenizer(object): data = self.stream.char() leavingThisState = True emitToken = False - if data == u"=": + if data == "=": self.state = self.beforeAttributeValueState elif data in asciiLetters: self.currentToken["data"][-1][0] += data +\ - self.stream.charsUntil(asciiLetters, True) + self.stream.charsUntil(asciiLetters, True) leavingThisState = False - elif data == u">": + elif data == ">": # XXX If we emit here the attributes are converted to a dict # without being checked and when the code below runs we error # because data is a dict not a list emitToken = True elif data in spaceCharacters: self.state = self.afterAttributeNameState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][0] += u"\uFFFD" + self.currentToken["data"][-1][0] += "\uFFFD" leavingThisState = False - elif data in (u"'", u'"', u"<"): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data in ("'", '"', "<"): + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-in-attribute-name"}) + "invalid-character-in-attribute-name"}) self.currentToken["data"][-1][0] += data leavingThisState = False elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "eof-in-attribute-name"}) self.state = self.dataState else: @@ -936,7 +927,7 @@ class HTMLTokenizer(object): for name, value in self.currentToken["data"][:-1]: if self.currentToken["data"][-1][0] == name: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "duplicate-attribute"}) + "duplicate-attribute"}) break # XXX Fix for above XXX if emitToken: @@ -947,28 +938,28 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.stream.charsUntil(spaceCharacters, True) - elif data == u"=": + elif data == "=": self.state = self.beforeAttributeValueState - elif data == u">": + elif data == ">": self.emitCurrentToken() elif data in asciiLetters: self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"].append([u"\uFFFD", ""]) + self.currentToken["data"].append(["\uFFFD", ""]) self.state = self.attributeNameState - elif data in (u"'", u'"', u"<"): + elif data in ("'", '"', "<"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "invalid-character-after-attribute-name"}) + "invalid-character-after-attribute-name"}) self.currentToken["data"].append([data, ""]) self.state = self.attributeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-end-of-tag-but-got-eof"}) + "expected-end-of-tag-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"].append([data, ""]) @@ -979,30 +970,30 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: self.stream.charsUntil(spaceCharacters, True) - elif data == u"\"": + elif data == "\"": self.state = self.attributeValueDoubleQuotedState - elif data == u"&": + elif data == "&": self.state = self.attributeValueUnQuotedState - self.stream.unget(data); - elif data == u"'": + self.stream.unget(data) + elif data == "'": self.state = self.attributeValueSingleQuotedState - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-value-but-got-right-bracket"}) + "expected-attribute-value-but-got-right-bracket"}) self.emitCurrentToken() - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" self.state = self.attributeValueUnQuotedState - elif data in (u"=", u"<", u"`"): + elif data in ("=", "<", "`"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "equals-in-unquoted-attribute-value"}) + "equals-in-unquoted-attribute-value"}) self.currentToken["data"][-1][1] += data self.state = self.attributeValueUnQuotedState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-attribute-value-but-got-eof"}) + "expected-attribute-value-but-got-eof"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data @@ -1013,81 +1004,81 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterAttributeValueState - elif data == u"&": - self.processEntityInAttribute(u'"') - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "&": + self.processEntityInAttribute('"') + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-double-quote"}) + "eof-in-attribute-value-double-quote"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data +\ - self.stream.charsUntil(("\"", u"&")) + self.stream.charsUntil(("\"", "&", "\u0000")) return True def attributeValueSingleQuotedState(self): data = self.stream.char() if data == "'": self.state = self.afterAttributeValueState - elif data == u"&": - self.processEntityInAttribute(u"'") - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "&": + self.processEntityInAttribute("'") + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-single-quote"}) + "eof-in-attribute-value-single-quote"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data +\ - self.stream.charsUntil(("'", u"&")) + self.stream.charsUntil(("'", "&", "\u0000")) return True def attributeValueUnQuotedState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u"&": + elif data == "&": self.processEntityInAttribute(">") - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data in (u'"', u"'", u"=", u"<", u"`"): + elif data in ('"', "'", "=", "<", "`"): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-in-unquoted-attribute-value"}) + "unexpected-character-in-unquoted-attribute-value"}) self.currentToken["data"][-1][1] += data - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"][-1][1] += u"\uFFFD" + self.currentToken["data"][-1][1] += "\uFFFD" elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-attribute-value-no-quotes"}) + "eof-in-attribute-value-no-quotes"}) self.state = self.dataState else: self.currentToken["data"][-1][1] += data + self.stream.charsUntil( - frozenset((u"&", u">", u'"', u"'", u"=", u"<", u"`")) | spaceCharacters) + frozenset(("&", ">", '"', "'", "=", "<", "`", "\u0000")) | spaceCharacters) return True def afterAttributeValueState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeAttributeNameState - elif data == u">": + elif data == ">": self.emitCurrentToken() - elif data == u"/": + elif data == "/": self.state = self.selfClosingStartTagState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-EOF-after-attribute-value"}) + "unexpected-EOF-after-attribute-value"}) self.stream.unget(data) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-after-attribute-value"}) + "unexpected-character-after-attribute-value"}) self.stream.unget(data) self.state = self.beforeAttributeNameState return True @@ -1098,14 +1089,14 @@ class HTMLTokenizer(object): self.currentToken["selfClosing"] = True self.emitCurrentToken() elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-EOF-after-solidus-in-tag"}) + "unexpected-EOF-after-solidus-in-tag"}) self.stream.unget(data) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-character-after-soldius-in-tag"}) + "unexpected-character-after-solidus-in-tag"}) self.stream.unget(data) self.state = self.beforeAttributeNameState return True @@ -1114,10 +1105,10 @@ class HTMLTokenizer(object): # Make a new comment token and give it as value all the characters # until the first > or EOF (charsUntil checks for EOF automatically) # and emit it. - data = self.stream.charsUntil(u">") - data = data.replace(u"\u0000", u"\uFFFD") + data = self.stream.charsUntil(">") + data = data.replace("\u0000", "\uFFFD") self.tokenQueue.append( - {"type": tokenTypes["Comment"], "data": data}) + {"type": tokenTypes["Comment"], "data": data}) # Eat the character directly after the bogus comment which is either a # ">" or an EOF. @@ -1127,28 +1118,28 @@ class HTMLTokenizer(object): def markupDeclarationOpenState(self): charStack = [self.stream.char()] - if charStack[-1] == u"-": + if charStack[-1] == "-": charStack.append(self.stream.char()) - if charStack[-1] == u"-": - self.currentToken = {"type": tokenTypes["Comment"], "data": u""} + if charStack[-1] == "-": + self.currentToken = {"type": tokenTypes["Comment"], "data": ""} self.state = self.commentStartState return True - elif charStack[-1] in (u'd', u'D'): + elif charStack[-1] in ('d', 'D'): matched = True - for expected in ((u'o', u'O'), (u'c', u'C'), (u't', u'T'), - (u'y', u'Y'), (u'p', u'P'), (u'e', u'E')): + for expected in (('o', 'O'), ('c', 'C'), ('t', 'T'), + ('y', 'Y'), ('p', 'P'), ('e', 'E')): charStack.append(self.stream.char()) if charStack[-1] not in expected: matched = False break if matched: self.currentToken = {"type": tokenTypes["Doctype"], - "name": u"", - "publicId": None, "systemId": None, + "name": "", + "publicId": None, "systemId": None, "correct": True} self.state = self.doctypeState return True - elif (charStack[-1] == "[" and + elif (charStack[-1] == "[" and self.parser is not None and self.parser.tree.openElements and self.parser.tree.openElements[-1].namespace != self.parser.tree.defaultNamespace): @@ -1163,7 +1154,7 @@ class HTMLTokenizer(object): return True self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-dashes-or-doctype"}) + "expected-dashes-or-doctype"}) while charStack: self.stream.unget(charStack.pop()) @@ -1174,41 +1165,41 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "-": self.state = self.commentStartDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"\uFFFD" + self.currentToken["data"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "incorrect-comment"}) + "incorrect-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment"}) + "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.currentToken["data"] += data self.state = self.commentState return True - + def commentStartDashState(self): data = self.stream.char() if data == "-": self.state = self.commentEndState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"-\uFFFD" + self.currentToken["data"] += "-\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "incorrect-comment"}) + "incorrect-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment"}) + "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: @@ -1216,95 +1207,94 @@ class HTMLTokenizer(object): self.state = self.commentState return True - def commentState(self): data = self.stream.char() - if data == u"-": + if data == "-": self.state = self.commentEndDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"\uFFFD" + self.currentToken["data"] += "\uFFFD" elif data is EOF: - self.tokenQueue.append({"type": tokenTypes["ParseError"], + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "eof-in-comment"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.currentToken["data"] += data + \ - self.stream.charsUntil((u"-", u"\u0000")) + self.stream.charsUntil(("-", "\u0000")) return True def commentEndDashState(self): data = self.stream.char() - if data == u"-": + if data == "-": self.state = self.commentEndState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"-\uFFFD" + self.currentToken["data"] += "-\uFFFD" self.state = self.commentState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-end-dash"}) + "eof-in-comment-end-dash"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - self.currentToken["data"] += u"-" + data + self.currentToken["data"] += "-" + data self.state = self.commentState return True def commentEndState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"--\uFFFD" + self.currentToken["data"] += "--\uFFFD" self.state = self.commentState elif data == "!": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-bang-after-double-dash-in-comment"}) + "unexpected-bang-after-double-dash-in-comment"}) self.state = self.commentEndBangState - elif data == u"-": + elif data == "-": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-dash-after-double-dash-in-comment"}) + "unexpected-dash-after-double-dash-in-comment"}) self.currentToken["data"] += data elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-double-dash"}) + "eof-in-comment-double-dash"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: # XXX self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-comment"}) - self.currentToken["data"] += u"--" + data + "unexpected-char-in-comment"}) + self.currentToken["data"] += "--" + data self.state = self.commentState return True def commentEndBangState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"-": + elif data == "-": self.currentToken["data"] += "--!" self.state = self.commentEndDashState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["data"] += u"--!\uFFFD" + self.currentToken["data"] += "--!\uFFFD" self.state = self.commentState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-comment-end-bang-state"}) + "eof-in-comment-end-bang-state"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - self.currentToken["data"] += u"--!" + data + self.currentToken["data"] += "--!" + data self.state = self.commentState return True @@ -1314,13 +1304,13 @@ class HTMLTokenizer(object): self.state = self.beforeDoctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-eof"}) + "expected-doctype-name-but-got-eof"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "need-space-after-doctype"}) + "need-space-after-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypeNameState return True @@ -1329,20 +1319,20 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: pass - elif data == u">": + elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-right-bracket"}) + "expected-doctype-name-but-got-right-bracket"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] = u"\uFFFD" + self.currentToken["name"] = "\uFFFD" self.state = self.doctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-doctype-name-but-got-eof"}) + "expected-doctype-name-but-got-eof"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1356,18 +1346,18 @@ class HTMLTokenizer(object): if data in spaceCharacters: self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.state = self.afterDoctypeNameState - elif data == u">": + elif data == ">": self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.tokenQueue.append(self.currentToken) self.state = self.dataState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["name"] += u"\uFFFD" + self.currentToken["name"] += "\uFFFD" self.state = self.doctypeNameState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype-name"}) + "eof-in-doctype-name"}) self.currentToken["correct"] = False self.currentToken["name"] = self.currentToken["name"].translate(asciiUpper2Lower) self.tokenQueue.append(self.currentToken) @@ -1380,21 +1370,21 @@ class HTMLTokenizer(object): data = self.stream.char() if data in spaceCharacters: pass - elif data == u">": + elif data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.currentToken["correct"] = False self.stream.unget(data) self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.tokenQueue.append(self.currentToken) self.state = self.dataState else: - if data in (u"p", u"P"): + if data in ("p", "P"): matched = True - for expected in ((u"u", u"U"), (u"b", u"B"), (u"l", u"L"), - (u"i", u"I"), (u"c", u"C")): + for expected in (("u", "U"), ("b", "B"), ("l", "L"), + ("i", "I"), ("c", "C")): data = self.stream.char() if data not in expected: matched = False @@ -1402,10 +1392,10 @@ class HTMLTokenizer(object): if matched: self.state = self.afterDoctypePublicKeywordState return True - elif data in (u"s", u"S"): + elif data in ("s", "S"): matched = True - for expected in ((u"y", u"Y"), (u"s", u"S"), (u"t", u"T"), - (u"e", u"E"), (u"m", u"M")): + for expected in (("y", "Y"), ("s", "S"), ("t", "T"), + ("e", "E"), ("m", "M")): data = self.stream.char() if data not in expected: matched = False @@ -1420,25 +1410,25 @@ class HTMLTokenizer(object): # and needs to be ungetted self.stream.unget(data) self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "expected-space-or-right-bracket-in-doctype", "datavars": - {"data": data}}) + "expected-space-or-right-bracket-in-doctype", "datavars": + {"data": data}}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def afterDoctypePublicKeywordState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeDoctypePublicIdentifierState elif data in ("'", '"'): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypePublicIdentifierState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1452,26 +1442,26 @@ class HTMLTokenizer(object): if data in spaceCharacters: pass elif data == "\"": - self.currentToken["publicId"] = u"" + self.currentToken["publicId"] = "" self.state = self.doctypePublicIdentifierDoubleQuotedState elif data == "'": - self.currentToken["publicId"] = u"" + self.currentToken["publicId"] = "" self.state = self.doctypePublicIdentifierSingleQuotedState elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True @@ -1480,19 +1470,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterDoctypePublicIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["publicId"] += u"\uFFFD" + self.currentToken["publicId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1504,19 +1494,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "'": self.state = self.afterDoctypePublicIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["publicId"] += u"\uFFFD" + self.currentToken["publicId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1533,27 +1523,27 @@ class HTMLTokenizer(object): self.state = self.dataState elif data == '"': self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) - self.currentToken["systemId"] = u"" + "unexpected-char-in-doctype"}) + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) - self.currentToken["systemId"] = u"" + "unexpected-char-in-doctype"}) + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def betweenDoctypePublicAndSystemIdentifiersState(self): data = self.stream.char() if data in spaceCharacters: @@ -1562,36 +1552,36 @@ class HTMLTokenizer(object): self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data == '"': - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data == EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True - + def afterDoctypeSystemKeywordState(self): data = self.stream.char() if data in spaceCharacters: self.state = self.beforeDoctypeSystemIdentifierState elif data in ("'", '"'): self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.stream.unget(data) self.state = self.beforeDoctypeSystemIdentifierState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1599,32 +1589,32 @@ class HTMLTokenizer(object): self.stream.unget(data) self.state = self.beforeDoctypeSystemIdentifierState return True - + def beforeDoctypeSystemIdentifierState(self): data = self.stream.char() if data in spaceCharacters: pass elif data == "\"": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierDoubleQuotedState elif data == "'": - self.currentToken["systemId"] = u"" + self.currentToken["systemId"] = "" self.state = self.doctypeSystemIdentifierSingleQuotedState elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.currentToken["correct"] = False self.state = self.bogusDoctypeState return True @@ -1633,19 +1623,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "\"": self.state = self.afterDoctypeSystemIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["systemId"] += u"\uFFFD" + self.currentToken["systemId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1657,19 +1647,19 @@ class HTMLTokenizer(object): data = self.stream.char() if data == "'": self.state = self.afterDoctypeSystemIdentifierState - elif data == u"\u0000": - self.tokenQueue.append({"type": tokenTypes["ParseError"], + elif data == "\u0000": + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - self.currentToken["systemId"] += u"\uFFFD" + self.currentToken["systemId"] += "\uFFFD" elif data == ">": self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-end-of-doctype"}) + "unexpected-end-of-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState @@ -1686,19 +1676,19 @@ class HTMLTokenizer(object): self.state = self.dataState elif data is EOF: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "eof-in-doctype"}) + "eof-in-doctype"}) self.currentToken["correct"] = False self.tokenQueue.append(self.currentToken) self.state = self.dataState else: self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": - "unexpected-char-in-doctype"}) + "unexpected-char-in-doctype"}) self.state = self.bogusDoctypeState return True def bogusDoctypeState(self): data = self.stream.char() - if data == u">": + if data == ">": self.tokenQueue.append(self.currentToken) self.state = self.dataState elif data is EOF: @@ -1713,32 +1703,29 @@ class HTMLTokenizer(object): def cdataSectionState(self): data = [] while True: - data.append(self.stream.charsUntil(u"]")) - charStack = [] - - for expected in ["]", "]", ">"]: - charStack.append(self.stream.char()) - matched = True - if charStack[-1] == EOF: - data.extend(charStack[:-1]) - break - elif charStack[-1] != expected: - matched = False - data.extend(charStack) - break - - if matched: + data.append(self.stream.charsUntil("]")) + data.append(self.stream.charsUntil(">")) + char = self.stream.char() + if char == EOF: break + else: + assert char == ">" + if data[-1][-2:] == "]]": + data[-1] = data[-1][:-2] + break + else: + data.append(char) + data = "".join(data) - #Deal with null here rather than in the parser - nullCount = data.count(u"\u0000") + # Deal with null here rather than in the parser + nullCount = data.count("\u0000") if nullCount > 0: - for i in xrange(nullCount): - self.tokenQueue.append({"type": tokenTypes["ParseError"], + for i in range(nullCount): + self.tokenQueue.append({"type": tokenTypes["ParseError"], "data": "invalid-codepoint"}) - data = data.replace(u"\u0000", u"\uFFFD") + data = data.replace("\u0000", "\uFFFD") if data: - self.tokenQueue.append({"type": tokenTypes["Characters"], + self.tokenQueue.append({"type": tokenTypes["Characters"], "data": data}) self.state = self.dataState return True diff --git a/libs/html5lib/treeadapters/__init__.py b/libs/html5lib/treeadapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/html5lib/treeadapters/sax.py b/libs/html5lib/treeadapters/sax.py new file mode 100644 index 00000000..ad47df95 --- /dev/null +++ b/libs/html5lib/treeadapters/sax.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import, division, unicode_literals + +from xml.sax.xmlreader import AttributesNSImpl + +from ..constants import adjustForeignAttributes, unadjustForeignAttributes + +prefix_mapping = {} +for prefix, localName, namespace in adjustForeignAttributes.values(): + if prefix is not None: + prefix_mapping[prefix] = namespace + + +def to_sax(walker, handler): + """Call SAX-like content handler based on treewalker walker""" + handler.startDocument() + for prefix, namespace in prefix_mapping.items(): + handler.startPrefixMapping(prefix, namespace) + + for token in walker: + type = token["type"] + if type == "Doctype": + continue + elif type in ("StartTag", "EmptyTag"): + attrs = AttributesNSImpl(token["data"], + unadjustForeignAttributes) + handler.startElementNS((token["namespace"], token["name"]), + token["name"], + attrs) + if type == "EmptyTag": + handler.endElementNS((token["namespace"], token["name"]), + token["name"]) + elif type == "EndTag": + handler.endElementNS((token["namespace"], token["name"]), + token["name"]) + elif type in ("Characters", "SpaceCharacters"): + handler.characters(token["data"]) + elif type == "Comment": + pass + else: + assert False, "Unknown token type" + + for prefix, namespace in prefix_mapping.items(): + handler.endPrefixMapping(prefix) + handler.endDocument() diff --git a/libs/html5lib/treebuilders/__init__.py b/libs/html5lib/treebuilders/__init__.py index 14f66d40..6a6b2a4c 100755 --- a/libs/html5lib/treebuilders/__init__.py +++ b/libs/html5lib/treebuilders/__init__.py @@ -7,7 +7,7 @@ implement several things: 1) A set of classes for various types of elements: Document, Doctype, Comment, Element. These must implement the interface of _base.treebuilders.Node (although comment nodes have a different -signature for their constructor, see treebuilders.simpletree.Comment) +signature for their constructor, see treebuilders.etree.Comment) Textual content may also be implemented as another node type, or not, as your tree implementation requires. @@ -24,73 +24,53 @@ getDocument - Returns the root node of the complete document tree testSerializer method on your treebuilder which accepts a node and returns a string containing Node and its children serialized according to the format used in the unittests - -The supplied simpletree module provides a python-only implementation -of a full treebuilder and is a useful reference for the semantics of -the various methods. """ +from __future__ import absolute_import, division, unicode_literals + +from ..utils import default_etree + treeBuilderCache = {} -import sys def getTreeBuilder(treeType, implementation=None, **kwargs): """Get a TreeBuilder class for various types of tree with built-in support - + treeType - the name of the tree type required (case-insensitive). Supported - values are "simpletree", "dom", "etree" and "beautifulsoup" - - "simpletree" - a built-in DOM-ish tree type with support for some - more pythonic idioms. - "dom" - A generic builder for DOM implementations, defaulting to - a xml.dom.minidom based implementation for the sake of - backwards compatibility (as releases up until 0.10 had a - builder called "dom" that was a minidom implemenation). - "etree" - A generic builder for tree implementations exposing an - elementtree-like interface (known to work with - ElementTree, cElementTree and lxml.etree). - "beautifulsoup" - Beautiful soup (if installed) - + values are: + + "dom" - A generic builder for DOM implementations, defaulting to + a xml.dom.minidom based implementation. + "etree" - A generic builder for tree implementations exposing an + ElementTree-like interface, defaulting to + xml.etree.cElementTree if available and + xml.etree.ElementTree if not. + "lxml" - A etree-based builder for lxml.etree, handling + limitations of lxml's implementation. + implementation - (Currently applies to the "etree" and "dom" tree types). A module implementing the tree type e.g. - xml.etree.ElementTree or lxml.etree.""" - + xml.etree.ElementTree or xml.etree.cElementTree.""" + treeType = treeType.lower() if treeType not in treeBuilderCache: if treeType == "dom": - import dom - # XXX: Keep backwards compatibility by using minidom if no implementation is given - if implementation == None: + from . import dom + # Come up with a sane default (pref. from the stdlib) + if implementation is None: from xml.dom import minidom implementation = minidom - # XXX: NEVER cache here, caching is done in the dom submodule + # NEVER cache here, caching is done in the dom submodule return dom.getDomModule(implementation, **kwargs).TreeBuilder - elif treeType == "simpletree": - import simpletree - treeBuilderCache[treeType] = simpletree.TreeBuilder - elif treeType == "beautifulsoup": - import soup - treeBuilderCache[treeType] = soup.TreeBuilder elif treeType == "lxml": - import etree_lxml + from . import etree_lxml treeBuilderCache[treeType] = etree_lxml.TreeBuilder elif treeType == "etree": - # Come up with a sane default - if implementation == None: - try: - import xml.etree.cElementTree as ET - except ImportError: - try: - import xml.etree.ElementTree as ET - except ImportError: - try: - import cElementTree as ET - except ImportError: - import elementtree.ElementTree as ET - implementation = ET - import etree + from . import etree + if implementation is None: + implementation = default_etree # NEVER cache here, caching is done in the etree submodule return etree.getETreeModule(implementation, **kwargs).TreeBuilder else: - raise ValueError("""Unrecognised treebuilder "%s" """%treeType) + raise ValueError("""Unrecognised treebuilder "%s" """ % treeType) return treeBuilderCache.get(treeType) diff --git a/libs/html5lib/treebuilders/_base.py b/libs/html5lib/treebuilders/_base.py index f3782d28..8b97cc11 100755 --- a/libs/html5lib/treebuilders/_base.py +++ b/libs/html5lib/treebuilders/_base.py @@ -1,25 +1,34 @@ -from html5lib.constants import scopingElements, tableInsertModeElements, namespaces -try: - frozenset -except NameError: - # Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from ..constants import scopingElements, tableInsertModeElements, namespaces # The scope markers are inserted when entering object elements, # marquees, table cells, and table captions, and are used to prevent formatting # from "leaking" into tables, object elements, and marquees. Marker = None +listElementsMap = { + None: (frozenset(scopingElements), False), + "button": (frozenset(scopingElements | set([(namespaces["html"], "button")])), False), + "list": (frozenset(scopingElements | set([(namespaces["html"], "ol"), + (namespaces["html"], "ul")])), False), + "table": (frozenset([(namespaces["html"], "html"), + (namespaces["html"], "table")]), False), + "select": (frozenset([(namespaces["html"], "optgroup"), + (namespaces["html"], "option")]), True) +} + + class Node(object): def __init__(self, name): """Node representing an item in the tree. name - The tag name associated with the node parent - The parent of the current node (or None for the document node) - value - The value of the current node (applies to text nodes and + value - The value of the current node (applies to text nodes and comments attributes - a dict holding name, value pairs for attributes of the node - childNodes - a list of child nodes of the current node. This must + childNodes - a list of child nodes of the current node. This must include all elements but not necessarily other node types _flags - A list of miscellaneous flags that can be set on the node """ @@ -30,14 +39,14 @@ class Node(object): self.childNodes = [] self._flags = [] - def __unicode__(self): - attributesStr = " ".join(["%s=\"%s\""%(name, value) - for name, value in - self.attributes.iteritems()]) + def __str__(self): + attributesStr = " ".join(["%s=\"%s\"" % (name, value) + for name, value in + self.attributes.items()]) if attributesStr: - return "<%s %s>"%(self.name,attributesStr) + return "<%s %s>" % (self.name, attributesStr) else: - return "<%s>"%(self.name) + return "<%s>" % (self.name) def __repr__(self): return "<%s>" % (self.name) @@ -48,14 +57,14 @@ class Node(object): raise NotImplementedError def insertText(self, data, insertBefore=None): - """Insert data as text in the current node, positioned before the + """Insert data as text in the current node, positioned before the start of node insertBefore or to the end of the node's text. """ raise NotImplementedError def insertBefore(self, node, refNode): - """Insert node as a child of the current node, before refNode in the - list of child nodes. Raises ValueError if refNode is not a child of + """Insert node as a child of the current node, before refNode in the + list of child nodes. Raises ValueError if refNode is not a child of the current node""" raise NotImplementedError @@ -65,11 +74,11 @@ class Node(object): raise NotImplementedError def reparentChildren(self, newParent): - """Move all the children of the current node to newParent. - This is needed so that trees that don't store text as nodes move the + """Move all the children of the current node to newParent. + This is needed so that trees that don't store text as nodes move the text in the correct way """ - #XXX - should this method be made more general? + # XXX - should this method be made more general? for child in self.childNodes: newParent.appendChild(child) self.childNodes = [] @@ -80,12 +89,12 @@ class Node(object): """ raise NotImplementedError - def hasContent(self): """Return true if the node has children or text, false otherwise """ raise NotImplementedError + class ActiveFormattingElements(list): def append(self, node): equalCount = 0 @@ -103,12 +112,13 @@ class ActiveFormattingElements(list): def nodesEqual(self, node1, node2): if not node1.nameTuple == node2.nameTuple: return False - + if not node1.attributes == node2.attributes: return False - + return True + class TreeBuilder(object): """Base treebuilder implementation documentClass - the class to use for the bottommost node of a document @@ -117,19 +127,19 @@ class TreeBuilder(object): doctypeClass - the class to use for doctypes """ - #Document class + # Document class documentClass = None - #The class to use for creating a node + # The class to use for creating a node elementClass = None - #The class to use for creating comments + # The class to use for creating comments commentClass = None - #The class to use for creating doctypes + # The class to use for creating doctypes doctypeClass = None - - #Fragment class + + # Fragment class fragmentClass = None def __init__(self, namespaceHTMLElements): @@ -138,12 +148,12 @@ class TreeBuilder(object): else: self.defaultNamespace = None self.reset() - + def reset(self): self.openElements = [] self.activeFormattingElements = ActiveFormattingElements() - #XXX - rename these to headElement, formElement + # XXX - rename these to headElement, formElement self.headPointer = None self.formPointer = None @@ -153,30 +163,20 @@ class TreeBuilder(object): def elementInScope(self, target, variant=None): - #If we pass a node in we match that. if we pass a string - #match any node with that name + # If we pass a node in we match that. if we pass a string + # match any node with that name exactNode = hasattr(target, "nameTuple") - listElementsMap = { - None:(scopingElements, False), - "button":(scopingElements | set([(namespaces["html"], "button")]), False), - "list":(scopingElements | set([(namespaces["html"], "ol"), - (namespaces["html"], "ul")]), False), - "table":(set([(namespaces["html"], "html"), - (namespaces["html"], "table")]), False), - "select":(set([(namespaces["html"], "optgroup"), - (namespaces["html"], "option")]), True) - } listElements, invert = listElementsMap[variant] for node in reversed(self.openElements): if (node.name == target and not exactNode or - node == target and exactNode): + node == target and exactNode): return True - elif (invert ^ (node.nameTuple in listElements)): + elif (invert ^ (node.nameTuple in listElements)): return False - assert False # We should never reach this point + assert False # We should never reach this point def reconstructActiveFormattingElements(self): # Within this algorithm the order of steps described in the @@ -196,7 +196,7 @@ class TreeBuilder(object): # Step 6 while entry != Marker and entry not in self.openElements: if i == 0: - #This will be reset to 0 below + # This will be reset to 0 below i = -1 break i -= 1 @@ -209,13 +209,13 @@ class TreeBuilder(object): # Step 8 entry = self.activeFormattingElements[i] - clone = entry.cloneNode() #Mainly to get a new copy of the attributes + clone = entry.cloneNode() # Mainly to get a new copy of the attributes # Step 9 - element = self.insertElement({"type":"StartTag", - "name":clone.name, - "namespace":clone.namespace, - "data":clone.attributes}) + element = self.insertElement({"type": "StartTag", + "name": clone.name, + "namespace": clone.namespace, + "data": clone.attributes}) # Step 10 self.activeFormattingElements[i] = element @@ -260,7 +260,7 @@ class TreeBuilder(object): if parent is None: parent = self.openElements[-1] parent.appendChild(self.commentClass(token["data"])) - + def createElement(self, token): """Create an element but don't insert it anywhere""" name = token["name"] @@ -282,10 +282,10 @@ class TreeBuilder(object): self.insertElement = self.insertElementNormal insertFromTable = property(_getInsertFromTable, _setInsertFromTable) - + def insertElementNormal(self, token): name = token["name"] - assert type(name) == unicode, "Element %s not unicode"%name + assert isinstance(name, text_type), "Element %s not unicode" % name namespace = token.get("namespace", self.defaultNamespace) element = self.elementClass(name, namespace) element.attributes = token["data"] @@ -294,13 +294,13 @@ class TreeBuilder(object): return element def insertElementTable(self, token): - """Create an element and insert it into the tree""" + """Create an element and insert it into the tree""" element = self.createElement(token) if self.openElements[-1].name not in tableInsertModeElements: return self.insertElementNormal(token) else: - #We should be in the InTable mode. This means we want to do - #special magic element rearranging + # We should be in the InTable mode. This means we want to do + # special magic element rearranging parent, insertBefore = self.getTableMisnestedNodePosition() if insertBefore is None: parent.appendChild(element) @@ -315,7 +315,7 @@ class TreeBuilder(object): parent = self.openElements[-1] if (not self.insertFromTable or (self.insertFromTable and - self.openElements[-1].name + self.openElements[-1].name not in tableInsertModeElements)): parent.insertText(data) else: @@ -323,14 +323,14 @@ class TreeBuilder(object): # special magic element rearranging parent, insertBefore = self.getTableMisnestedNodePosition() parent.insertText(data, insertBefore) - + def getTableMisnestedNodePosition(self): """Get the foster parent element, and sibling to insert before (or None) when inserting a misnested table node""" # The foster parent element is the one which comes before the most # recently opened table element # XXX - this is really inelegant - lastTable=None + lastTable = None fosterParent = None insertBefore = None for elm in self.openElements[::-1]: @@ -354,7 +354,7 @@ class TreeBuilder(object): name = self.openElements[-1].name # XXX td, th and tr are not actually needed if (name in frozenset(("dd", "dt", "li", "option", "optgroup", "p", "rp", "rt")) - and name != exclude): + and name != exclude): self.openElements.pop() # XXX This is not entirely what the specification says. We should # investigate it more closely. @@ -363,10 +363,10 @@ class TreeBuilder(object): def getDocument(self): "Return the final tree" return self.document - + def getFragment(self): "Return the final fragment" - #assert self.innerHTML + # assert self.innerHTML fragment = self.fragmentClass() self.openElements[0].reparentChildren(fragment) return fragment diff --git a/libs/html5lib/treebuilders/dom.py b/libs/html5lib/treebuilders/dom.py index 9578da2b..61e5ed79 100644 --- a/libs/html5lib/treebuilders/dom.py +++ b/libs/html5lib/treebuilders/dom.py @@ -1,45 +1,38 @@ +from __future__ import absolute_import, division, unicode_literals -from xml.dom import minidom, Node, XML_NAMESPACE, XMLNS_NAMESPACE -try: - from types import ModuleType -except: - from new import module as ModuleType -import re + +from xml.dom import minidom, Node import weakref -import _base -from html5lib import constants, ihatexml -from html5lib.constants import namespaces +from . import _base +from .. import constants +from ..constants import namespaces +from ..utils import moduleFactoryFactory -moduleCache = {} - -def getDomModule(DomImplementation): - name = "_" + DomImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType(name) - objs = getDomBuilder(DomImplementation) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getDomBuilder(DomImplementation): Dom = DomImplementation + class AttrList(object): def __init__(self, element): self.element = element + def __iter__(self): - return self.element.attributes.items().__iter__() + return list(self.element.attributes.items()).__iter__() + def __setitem__(self, name, value): self.element.setAttribute(name, value) + def __len__(self): - return len(self.element.attributes.items()) + return len(list(self.element.attributes.items())) + def items(self): return [(item[0], item[1]) for item in - self.element.attributes.items()] + list(self.element.attributes.items())] + def keys(self): - return self.element.attributes.keys() + return list(self.element.attributes.keys()) + def __getitem__(self, name): return self.element.getAttribute(name) @@ -48,68 +41,68 @@ def getDomBuilder(DomImplementation): raise NotImplementedError else: return self.element.hasAttribute(name) - + class NodeBuilder(_base.Node): def __init__(self, element): _base.Node.__init__(self, element.nodeName) self.element = element - namespace = property(lambda self:hasattr(self.element, "namespaceURI") + namespace = property(lambda self: hasattr(self.element, "namespaceURI") and self.element.namespaceURI or None) def appendChild(self, node): node.parent = self self.element.appendChild(node.element) - + def insertText(self, data, insertBefore=None): text = self.element.ownerDocument.createTextNode(data) if insertBefore: self.element.insertBefore(text, insertBefore.element) else: self.element.appendChild(text) - + def insertBefore(self, node, refNode): self.element.insertBefore(node.element, refNode.element) node.parent = self - + def removeChild(self, node): if node.element.parentNode == self.element: self.element.removeChild(node.element) node.parent = None - + def reparentChildren(self, newParent): while self.element.hasChildNodes(): child = self.element.firstChild self.element.removeChild(child) newParent.element.appendChild(child) self.childNodes = [] - + def getAttributes(self): return AttrList(self.element) - + def setAttributes(self, attributes): if attributes: - for name, value in attributes.items(): + for name, value in list(attributes.items()): if isinstance(name, tuple): if name[0] is not None: qualifiedName = (name[0] + ":" + name[1]) else: qualifiedName = name[1] - self.element.setAttributeNS(name[2], qualifiedName, + self.element.setAttributeNS(name[2], qualifiedName, value) else: self.element.setAttribute( name, value) attributes = property(getAttributes, setAttributes) - + def cloneNode(self): return NodeBuilder(self.element.cloneNode(False)) - + def hasContent(self): return self.element.hasChildNodes() def getNameTuple(self): - if self.namespace == None: + if self.namespace is None: return namespaces["html"], self.name else: return self.namespace, self.name @@ -118,9 +111,9 @@ def getDomBuilder(DomImplementation): class TreeBuilder(_base.TreeBuilder): def documentClass(self): - self.dom = Dom.getDOMImplementation().createDocument(None,None,None) + self.dom = Dom.getDOMImplementation().createDocument(None, None, None) return weakref.proxy(self) - + def insertDoctype(self, token): name = token["name"] publicId = token["publicId"] @@ -131,7 +124,7 @@ def getDomBuilder(DomImplementation): self.document.appendChild(NodeBuilder(doctype)) if Dom == minidom: doctype.ownerDocument = self.dom - + def elementClass(self, name, namespace=None): if namespace is None and self.defaultNamespace is None: node = self.dom.createElement(name) @@ -139,70 +132,72 @@ def getDomBuilder(DomImplementation): node = self.dom.createElementNS(namespace, name) return NodeBuilder(node) - + def commentClass(self, data): return NodeBuilder(self.dom.createComment(data)) - + def fragmentClass(self): return NodeBuilder(self.dom.createDocumentFragment()) - + def appendChild(self, node): self.dom.appendChild(node.element) - + def testSerializer(self, element): return testSerializer(element) - + def getDocument(self): return self.dom - + def getFragment(self): return _base.TreeBuilder.getFragment(self).element - + def insertText(self, data, parent=None): - data=data - if parent <> self: + data = data + if parent != self: _base.TreeBuilder.insertText(self, data, parent) else: # HACK: allow text nodes as children of the document node if hasattr(self.dom, '_child_node_types'): if not Node.TEXT_NODE in self.dom._child_node_types: - self.dom._child_node_types=list(self.dom._child_node_types) + self.dom._child_node_types = list(self.dom._child_node_types) self.dom._child_node_types.append(Node.TEXT_NODE) self.dom.appendChild(self.dom.createTextNode(data)) - + + implementation = DomImplementation name = None - + def testSerializer(element): element.normalize() rv = [] + def serializeElement(element, indent=0): if element.nodeType == Node.DOCUMENT_TYPE_NODE: if element.name: if element.publicId or element.systemId: publicId = element.publicId or "" systemId = element.systemId or "" - rv.append( """|%s<!DOCTYPE %s "%s" "%s">"""%( - ' '*indent, element.name, publicId, systemId)) + rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" % + (' ' * indent, element.name, publicId, systemId)) else: - rv.append("|%s<!DOCTYPE %s>"%(' '*indent, element.name)) + rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, element.name)) else: - rv.append("|%s<!DOCTYPE >"%(' '*indent,)) + rv.append("|%s<!DOCTYPE >" % (' ' * indent,)) elif element.nodeType == Node.DOCUMENT_NODE: rv.append("#document") elif element.nodeType == Node.DOCUMENT_FRAGMENT_NODE: rv.append("#document-fragment") elif element.nodeType == Node.COMMENT_NODE: - rv.append("|%s<!-- %s -->"%(' '*indent, element.nodeValue)) + rv.append("|%s<!-- %s -->" % (' ' * indent, element.nodeValue)) elif element.nodeType == Node.TEXT_NODE: - rv.append("|%s\"%s\"" %(' '*indent, element.nodeValue)) + rv.append("|%s\"%s\"" % (' ' * indent, element.nodeValue)) else: if (hasattr(element, "namespaceURI") and - element.namespaceURI != None): - name = "%s %s"%(constants.prefixes[element.namespaceURI], - element.nodeName) + element.namespaceURI is not None): + name = "%s %s" % (constants.prefixes[element.namespaceURI], + element.nodeName) else: name = element.nodeName - rv.append("|%s<%s>"%(' '*indent, name)) + rv.append("|%s<%s>" % (' ' * indent, name)) if element.hasAttributes(): attributes = [] for i in range(len(element.attributes)): @@ -211,81 +206,22 @@ def getDomBuilder(DomImplementation): value = attr.value ns = attr.namespaceURI if ns: - name = "%s %s"%(constants.prefixes[ns], attr.localName) + name = "%s %s" % (constants.prefixes[ns], attr.localName) else: name = attr.nodeName attributes.append((name, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) indent += 2 for child in element.childNodes: serializeElement(child, indent) serializeElement(element, 0) - + return "\n".join(rv) - - def dom2sax(node, handler, nsmap={'xml':XML_NAMESPACE}): - if node.nodeType == Node.ELEMENT_NODE: - if not nsmap: - handler.startElement(node.nodeName, node.attributes) - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endElement(node.nodeName) - else: - attributes = dict(node.attributes.itemsNS()) - - # gather namespace declarations - prefixes = [] - for attrname in node.attributes.keys(): - attr = node.getAttributeNode(attrname) - if (attr.namespaceURI == XMLNS_NAMESPACE or - (attr.namespaceURI == None and attr.nodeName.startswith('xmlns'))): - prefix = (attr.nodeName != 'xmlns' and attr.nodeName or None) - handler.startPrefixMapping(prefix, attr.nodeValue) - prefixes.append(prefix) - nsmap = nsmap.copy() - nsmap[prefix] = attr.nodeValue - del attributes[(attr.namespaceURI, attr.nodeName)] - - # apply namespace declarations - for attrname in node.attributes.keys(): - attr = node.getAttributeNode(attrname) - if attr.namespaceURI == None and ':' in attr.nodeName: - prefix = attr.nodeName.split(':')[0] - if nsmap.has_key(prefix): - del attributes[(attr.namespaceURI, attr.nodeName)] - attributes[(nsmap[prefix],attr.nodeName)]=attr.nodeValue - - # SAX events - ns = node.namespaceURI or nsmap.get(None,None) - handler.startElementNS((ns,node.nodeName), node.nodeName, attributes) - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endElementNS((ns, node.nodeName), node.nodeName) - for prefix in prefixes: handler.endPrefixMapping(prefix) - - elif node.nodeType in [Node.TEXT_NODE, Node.CDATA_SECTION_NODE]: - handler.characters(node.nodeValue) - - elif node.nodeType == Node.DOCUMENT_NODE: - handler.startDocument() - for child in node.childNodes: dom2sax(child, handler, nsmap) - handler.endDocument() - - elif node.nodeType == Node.DOCUMENT_FRAGMENT_NODE: - for child in node.childNodes: dom2sax(child, handler, nsmap) - - else: - # ATTRIBUTE_NODE - # ENTITY_NODE - # PROCESSING_INSTRUCTION_NODE - # COMMENT_NODE - # DOCUMENT_TYPE_NODE - # NOTATION_NODE - pass - + return locals() -# Keep backwards compatibility with things that directly load -# classes/functions from this module -for key, value in getDomModule(minidom).__dict__.items(): - globals()[key] = value + +# The actual means to get a module! +getDomModule = moduleFactoryFactory(getDomBuilder) diff --git a/libs/html5lib/treebuilders/etree.py b/libs/html5lib/treebuilders/etree.py index 95be4755..2c8ed19f 100755 --- a/libs/html5lib/treebuilders/etree.py +++ b/libs/html5lib/treebuilders/etree.py @@ -1,32 +1,21 @@ -try: - from types import ModuleType -except: - from new import module as ModuleType -import re -import types +from __future__ import absolute_import, division, unicode_literals +from six import text_type -import _base -from html5lib import ihatexml -from html5lib import constants -from html5lib.constants import namespaces +import re + +from . import _base +from .. import ihatexml +from .. import constants +from ..constants import namespaces +from ..utils import moduleFactoryFactory tag_regexp = re.compile("{([^}]*)}(.*)") -moduleCache = {} - -def getETreeModule(ElementTreeImplementation, fullTree=False): - name = "_" + ElementTreeImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder") - objs = getETreeBuilder(ElementTreeImplementation, fullTree) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getETreeBuilder(ElementTreeImplementation, fullTree=False): ElementTree = ElementTreeImplementation + ElementTreeCommentType = ElementTree.Comment("asd").tag + class Element(_base.Node): def __init__(self, name, namespace=None): self._name = name @@ -45,16 +34,16 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): if namespace is None: etree_tag = name else: - etree_tag = "{%s}%s"%(namespace, name) + etree_tag = "{%s}%s" % (namespace, name) return etree_tag - + def _setName(self, name): self._name = name self._element.tag = self._getETreeTag(self._name, self._namespace) - + def _getName(self): return self._name - + name = property(_getName, _setName) def _setNamespace(self, namespace): @@ -65,81 +54,82 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): return self._namespace namespace = property(_getNamespace, _setNamespace) - + def _getAttributes(self): return self._element.attrib - + def _setAttributes(self, attributes): - #Delete existing attributes first - #XXX - there may be a better way to do this... - for key in self._element.attrib.keys(): + # Delete existing attributes first + # XXX - there may be a better way to do this... + for key in list(self._element.attrib.keys()): del self._element.attrib[key] - for key, value in attributes.iteritems(): + for key, value in attributes.items(): if isinstance(key, tuple): - name = "{%s}%s"%(key[2], key[1]) + name = "{%s}%s" % (key[2], key[1]) else: name = key self._element.set(name, value) - + attributes = property(_getAttributes, _setAttributes) - + def _getChildNodes(self): - return self._childNodes + return self._childNodes + def _setChildNodes(self, value): del self._element[:] self._childNodes = [] for element in value: self.insertChild(element) - + childNodes = property(_getChildNodes, _setChildNodes) - + def hasContent(self): """Return true if the node has children or text""" return bool(self._element.text or len(self._element)) - + def appendChild(self, node): self._childNodes.append(node) self._element.append(node._element) node.parent = self - + def insertBefore(self, node, refNode): index = list(self._element).index(refNode._element) self._element.insert(index, node._element) node.parent = self - + def removeChild(self, node): self._element.remove(node._element) - node.parent=None - + node.parent = None + def insertText(self, data, insertBefore=None): if not(len(self._element)): if not self._element.text: self._element.text = "" self._element.text += data elif insertBefore is None: - #Insert the text as the tail of the last child element + # Insert the text as the tail of the last child element if not self._element[-1].tail: self._element[-1].tail = "" self._element[-1].tail += data else: - #Insert the text before the specified node + # Insert the text before the specified node children = list(self._element) index = children.index(insertBefore._element) if index > 0: - if not self._element[index-1].tail: - self._element[index-1].tail = "" - self._element[index-1].tail += data + if not self._element[index - 1].tail: + self._element[index - 1].tail = "" + self._element[index - 1].tail += data else: if not self._element.text: self._element.text = "" self._element.text += data - + def cloneNode(self): element = type(self)(self.name, self.namespace) - for name, value in self.attributes.iteritems(): + for name, value in self.attributes.items(): element.attributes[name] = value return element - + def reparentChildren(self, newParent): if newParent.childNodes: newParent.childNodes[-1]._element.tail += self._element.text @@ -150,60 +140,60 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): newParent._element.text += self._element.text self._element.text = "" _base.Node.reparentChildren(self, newParent) - + class Comment(Element): def __init__(self, data): - #Use the superclass constructor to set all properties on the - #wrapper element + # Use the superclass constructor to set all properties on the + # wrapper element self._element = ElementTree.Comment(data) self.parent = None self._childNodes = [] self._flags = [] - + def _getData(self): return self._element.text - + def _setData(self, value): self._element.text = value - + data = property(_getData, _setData) - + class DocumentType(Element): def __init__(self, name, publicId, systemId): - Element.__init__(self, "<!DOCTYPE>") + Element.__init__(self, "<!DOCTYPE>") self._element.text = name self.publicId = publicId self.systemId = systemId def _getPublicId(self): - return self._element.get(u"publicId", "") + return self._element.get("publicId", "") def _setPublicId(self, value): if value is not None: - self._element.set(u"publicId", value) + self._element.set("publicId", value) publicId = property(_getPublicId, _setPublicId) - + def _getSystemId(self): - return self._element.get(u"systemId", "") + return self._element.get("systemId", "") def _setSystemId(self, value): if value is not None: - self._element.set(u"systemId", value) + self._element.set("systemId", value) systemId = property(_getSystemId, _setSystemId) - + class Document(Element): def __init__(self): - Element.__init__(self, "<DOCUMENT_ROOT>") - + Element.__init__(self, "DOCUMENT_ROOT") + class DocumentFragment(Element): def __init__(self): - Element.__init__(self, "<DOCUMENT_FRAGMENT>") - + Element.__init__(self, "DOCUMENT_FRAGMENT") + def testSerializer(element): rv = [] - finalText = None + def serializeElement(element, indent=0): if not(hasattr(element, "tag")): element = element.getroot() @@ -211,20 +201,23 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): if element.get("publicId") or element.get("systemId"): publicId = element.get("publicId") or "" systemId = element.get("systemId") or "" - rv.append( """<!DOCTYPE %s "%s" "%s">"""%( - element.text, publicId, systemId)) - else: - rv.append("<!DOCTYPE %s>"%(element.text,)) - elif element.tag == "<DOCUMENT_ROOT>": + rv.append("""<!DOCTYPE %s "%s" "%s">""" % + (element.text, publicId, systemId)) + else: + rv.append("<!DOCTYPE %s>" % (element.text,)) + elif element.tag == "DOCUMENT_ROOT": rv.append("#document") - if element.text: - rv.append("|%s\"%s\""%(' '*(indent+2), element.text)) - if element.tail: - finalText = element.tail - elif element.tag == ElementTree.Comment: - rv.append("|%s<!-- %s -->"%(' '*indent, element.text)) + if element.text is not None: + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) + if element.tail is not None: + raise TypeError("Document node cannot have tail") + if hasattr(element, "attrib") and len(element.attrib): + raise TypeError("Document node cannot have attributes") + elif element.tag == ElementTreeCommentType: + rv.append("|%s<!-- %s -->" % (' ' * indent, element.text)) else: - assert type(element.tag) in types.StringTypes, "Expected unicode, got %s"%type(element.tag) + assert isinstance(element.tag, text_type), \ + "Expected unicode, got %s, %s" % (type(element.tag), element.tag) nsmatch = tag_regexp.match(element.tag) if nsmatch is None: @@ -232,113 +225,113 @@ def getETreeBuilder(ElementTreeImplementation, fullTree=False): else: ns, name = nsmatch.groups() prefix = constants.prefixes[ns] - name = "%s %s"%(prefix, name) - rv.append("|%s<%s>"%(' '*indent, name)) + name = "%s %s" % (prefix, name) + rv.append("|%s<%s>" % (' ' * indent, name)) if hasattr(element, "attrib"): attributes = [] - for name, value in element.attrib.iteritems(): + for name, value in element.attrib.items(): nsmatch = tag_regexp.match(name) if nsmatch is not None: ns, name = nsmatch.groups() prefix = constants.prefixes[ns] - attr_string = "%s %s"%(prefix, name) + attr_string = "%s %s" % (prefix, name) else: attr_string = name attributes.append((attr_string, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) if element.text: - rv.append("|%s\"%s\"" %(' '*(indent+2), element.text)) + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) indent += 2 for child in element: serializeElement(child, indent) if element.tail: - rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail)) + rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail)) serializeElement(element, 0) - - if finalText is not None: - rv.append("|%s\"%s\""%(' '*2, finalText)) - + return "\n".join(rv) - + def tostring(element): """Serialize an element and its child nodes to a string""" rv = [] - finalText = None filter = ihatexml.InfosetFilter() + def serializeElement(element): - if type(element) == type(ElementTree.ElementTree): + if isinstance(element, ElementTree.ElementTree): element = element.getroot() - + if element.tag == "<!DOCTYPE>": if element.get("publicId") or element.get("systemId"): publicId = element.get("publicId") or "" systemId = element.get("systemId") or "" - rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%( - element.text, publicId, systemId)) - else: - rv.append("<!DOCTYPE %s>"%(element.text,)) - elif element.tag == "<DOCUMENT_ROOT>": - if element.text: - rv.append(element.text) - if element.tail: - finalText = element.tail - - for child in element: - serializeElement(child) - - elif type(element.tag) == type(ElementTree.Comment): - rv.append("<!--%s-->"%(element.text,)) - else: - #This is assumed to be an ordinary element - if not element.attrib: - rv.append("<%s>"%(filter.fromXmlName(element.tag),)) + rv.append("""<!DOCTYPE %s PUBLIC "%s" "%s">""" % + (element.text, publicId, systemId)) else: - attr = " ".join(["%s=\"%s\""%( - filter.fromXmlName(name), value) - for name, value in element.attrib.iteritems()]) - rv.append("<%s %s>"%(element.tag, attr)) - if element.text: + rv.append("<!DOCTYPE %s>" % (element.text,)) + elif element.tag == "DOCUMENT_ROOT": + if element.text is not None: rv.append(element.text) - + if element.tail is not None: + raise TypeError("Document node cannot have tail") + if hasattr(element, "attrib") and len(element.attrib): + raise TypeError("Document node cannot have attributes") + for child in element: serializeElement(child) - - rv.append("</%s>"%(element.tag,)) - + + elif element.tag == ElementTreeCommentType: + rv.append("<!--%s-->" % (element.text,)) + else: + # This is assumed to be an ordinary element + if not element.attrib: + rv.append("<%s>" % (filter.fromXmlName(element.tag),)) + else: + attr = " ".join(["%s=\"%s\"" % ( + filter.fromXmlName(name), value) + for name, value in element.attrib.items()]) + rv.append("<%s %s>" % (element.tag, attr)) + if element.text: + rv.append(element.text) + + for child in element: + serializeElement(child) + + rv.append("</%s>" % (element.tag,)) + if element.tail: rv.append(element.tail) - + serializeElement(element) - - if finalText is not None: - rv.append("%s\""%(' '*2, finalText)) - + return "".join(rv) - + class TreeBuilder(_base.TreeBuilder): documentClass = Document doctypeClass = DocumentType elementClass = Element commentClass = Comment fragmentClass = DocumentFragment - + implementation = ElementTreeImplementation + def testSerializer(self, element): return testSerializer(element) - + def getDocument(self): if fullTree: return self.document._element else: if self.defaultNamespace is not None: return self.document._element.find( - "{%s}html"%self.defaultNamespace) + "{%s}html" % self.defaultNamespace) else: return self.document._element.find("html") - + def getFragment(self): return _base.TreeBuilder.getFragment(self)._element - + return locals() + + +getETreeModule = moduleFactoryFactory(getETreeBuilder) diff --git a/libs/html5lib/treebuilders/etree_lxml.py b/libs/html5lib/treebuilders/etree_lxml.py index eee1e3b2..35d08efa 100644 --- a/libs/html5lib/treebuilders/etree_lxml.py +++ b/libs/html5lib/treebuilders/etree_lxml.py @@ -1,20 +1,3 @@ -import warnings -import re - -import _base -from html5lib.constants import DataLossWarning -import html5lib.constants as constants -import etree as etree_builders -from html5lib import ihatexml - -try: - import lxml.etree as etree -except ImportError: - pass - -fullTree = True -tag_regexp = re.compile("{([^}]*)}(.*)") - """Module for supporting the lxml.etree library. The idea here is to use as much of the native library as possible, without using fragile hacks like custom element names that break between releases. The downside of this is that we cannot represent @@ -26,12 +9,34 @@ Docypes with no name When any of these things occur, we emit a DataLossWarning """ +from __future__ import absolute_import, division, unicode_literals + +import warnings +import re +import sys + +from . import _base +from ..constants import DataLossWarning +from .. import constants +from . import etree as etree_builders +from .. import ihatexml + +import lxml.etree as etree + + +fullTree = True +tag_regexp = re.compile("{([^}]*)}(.*)") + +comment_type = etree.Comment("asd").tag + + class DocumentType(object): def __init__(self, name, publicId, systemId): - self.name = name + self.name = name self.publicId = publicId self.systemId = systemId + class Document(object): def __init__(self): self._elementTree = None @@ -42,118 +47,126 @@ class Document(object): def _getChildNodes(self): return self._childNodes - + childNodes = property(_getChildNodes) + def testSerializer(element): rv = [] finalText = None - filter = ihatexml.InfosetFilter() + infosetFilter = ihatexml.InfosetFilter() + def serializeElement(element, indent=0): if not hasattr(element, "tag"): - if hasattr(element, "getroot"): - #Full tree case + if hasattr(element, "getroot"): + # Full tree case rv.append("#document") if element.docinfo.internalDTD: - if not (element.docinfo.public_id or + if not (element.docinfo.public_id or element.docinfo.system_url): - dtd_str = "<!DOCTYPE %s>"%element.docinfo.root_name + dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name else: - dtd_str = """<!DOCTYPE %s "%s" "%s">"""%( - element.docinfo.root_name, + dtd_str = """<!DOCTYPE %s "%s" "%s">""" % ( + element.docinfo.root_name, element.docinfo.public_id, element.docinfo.system_url) - rv.append("|%s%s"%(' '*(indent+2), dtd_str)) + rv.append("|%s%s" % (' ' * (indent + 2), dtd_str)) next_element = element.getroot() while next_element.getprevious() is not None: next_element = next_element.getprevious() while next_element is not None: - serializeElement(next_element, indent+2) + serializeElement(next_element, indent + 2) next_element = next_element.getnext() - elif isinstance(element, basestring): - #Text in a fragment - rv.append("|%s\"%s\""%(' '*indent, element)) + elif isinstance(element, str) or isinstance(element, bytes): + # Text in a fragment + assert isinstance(element, str) or sys.version_info.major == 2 + rv.append("|%s\"%s\"" % (' ' * indent, element)) else: - #Fragment case + # Fragment case rv.append("#document-fragment") for next_element in element: - serializeElement(next_element, indent+2) - elif type(element.tag) == type(etree.Comment): - rv.append("|%s<!-- %s -->"%(' '*indent, element.text)) + serializeElement(next_element, indent + 2) + elif element.tag == comment_type: + rv.append("|%s<!-- %s -->" % (' ' * indent, element.text)) + if hasattr(element, "tail") and element.tail: + rv.append("|%s\"%s\"" % (' ' * indent, element.tail)) else: + assert isinstance(element, etree._Element) nsmatch = etree_builders.tag_regexp.match(element.tag) if nsmatch is not None: ns = nsmatch.group(1) tag = nsmatch.group(2) prefix = constants.prefixes[ns] - rv.append("|%s<%s %s>"%(' '*indent, prefix, - filter.fromXmlName(tag))) + rv.append("|%s<%s %s>" % (' ' * indent, prefix, + infosetFilter.fromXmlName(tag))) else: - rv.append("|%s<%s>"%(' '*indent, - filter.fromXmlName(element.tag))) + rv.append("|%s<%s>" % (' ' * indent, + infosetFilter.fromXmlName(element.tag))) if hasattr(element, "attrib"): attributes = [] - for name, value in element.attrib.iteritems(): + for name, value in element.attrib.items(): nsmatch = tag_regexp.match(name) if nsmatch is not None: ns, name = nsmatch.groups() - name = filter.fromXmlName(name) + name = infosetFilter.fromXmlName(name) prefix = constants.prefixes[ns] - attr_string = "%s %s"%(prefix, name) + attr_string = "%s %s" % (prefix, name) else: - attr_string = filter.fromXmlName(name) + attr_string = infosetFilter.fromXmlName(name) attributes.append((attr_string, value)) for name, value in sorted(attributes): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) + rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value)) if element.text: - rv.append("|%s\"%s\"" %(' '*(indent+2), element.text)) + rv.append("|%s\"%s\"" % (' ' * (indent + 2), element.text)) indent += 2 - for child in element.getchildren(): + for child in element: serializeElement(child, indent) - if hasattr(element, "tail") and element.tail: - rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail)) + if hasattr(element, "tail") and element.tail: + rv.append("|%s\"%s\"" % (' ' * (indent - 2), element.tail)) serializeElement(element, 0) if finalText is not None: - rv.append("|%s\"%s\""%(' '*2, finalText)) + rv.append("|%s\"%s\"" % (' ' * 2, finalText)) return "\n".join(rv) + def tostring(element): """Serialize an element and its child nodes to a string""" rv = [] finalText = None + def serializeElement(element): if not hasattr(element, "tag"): if element.docinfo.internalDTD: if element.docinfo.doctype: dtd_str = element.docinfo.doctype else: - dtd_str = "<!DOCTYPE %s>"%element.docinfo.root_name + dtd_str = "<!DOCTYPE %s>" % element.docinfo.root_name rv.append(dtd_str) serializeElement(element.getroot()) - - elif type(element.tag) == type(etree.Comment): - rv.append("<!--%s-->"%(element.text,)) - + + elif element.tag == comment_type: + rv.append("<!--%s-->" % (element.text,)) + else: - #This is assumed to be an ordinary element + # This is assumed to be an ordinary element if not element.attrib: - rv.append("<%s>"%(element.tag,)) + rv.append("<%s>" % (element.tag,)) else: - attr = " ".join(["%s=\"%s\""%(name, value) - for name, value in element.attrib.iteritems()]) - rv.append("<%s %s>"%(element.tag, attr)) + attr = " ".join(["%s=\"%s\"" % (name, value) + for name, value in element.attrib.items()]) + rv.append("<%s %s>" % (element.tag, attr)) if element.text: rv.append(element.text) - for child in element.getchildren(): + for child in element: serializeElement(child) - rv.append("</%s>"%(element.tag,)) + rv.append("</%s>" % (element.tag,)) if hasattr(element, "tail") and element.tail: rv.append(element.tail) @@ -161,56 +174,57 @@ def tostring(element): serializeElement(element) if finalText is not None: - rv.append("%s\""%(' '*2, finalText)) + rv.append("%s\"" % (' ' * 2, finalText)) return "".join(rv) - + class TreeBuilder(_base.TreeBuilder): documentClass = Document doctypeClass = DocumentType elementClass = None commentClass = None - fragmentClass = Document + fragmentClass = Document + implementation = etree - def __init__(self, namespaceHTMLElements, fullTree = False): + def __init__(self, namespaceHTMLElements, fullTree=False): builder = etree_builders.getETreeModule(etree, fullTree=fullTree) - filter = self.filter = ihatexml.InfosetFilter() + infosetFilter = self.infosetFilter = ihatexml.InfosetFilter() self.namespaceHTMLElements = namespaceHTMLElements class Attributes(dict): def __init__(self, element, value={}): self._element = element dict.__init__(self, value) - for key, value in self.iteritems(): + for key, value in self.items(): if isinstance(key, tuple): - name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1])) + name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1])) else: - name = filter.coerceAttribute(key) + name = infosetFilter.coerceAttribute(key) self._element._element.attrib[name] = value def __setitem__(self, key, value): dict.__setitem__(self, key, value) if isinstance(key, tuple): - name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1])) + name = "{%s}%s" % (key[2], infosetFilter.coerceAttribute(key[1])) else: - name = filter.coerceAttribute(key) + name = infosetFilter.coerceAttribute(key) self._element._element.attrib[name] = value class Element(builder.Element): def __init__(self, name, namespace): - name = filter.coerceElement(name) + name = infosetFilter.coerceElement(name) builder.Element.__init__(self, name, namespace=namespace) self._attributes = Attributes(self) def _setName(self, name): - self._name = filter.coerceElement(name) + self._name = infosetFilter.coerceElement(name) self._element.tag = self._getETreeTag( self._name, self._namespace) - + def _getName(self): - return filter.fromXmlName(self._name) - + return infosetFilter.fromXmlName(self._name) + name = property(_getName, _setName) def _getAttributes(self): @@ -218,24 +232,23 @@ class TreeBuilder(_base.TreeBuilder): def _setAttributes(self, attributes): self._attributes = Attributes(self, attributes) - + attributes = property(_getAttributes, _setAttributes) def insertText(self, data, insertBefore=None): - data = filter.coerceCharacters(data) + data = infosetFilter.coerceCharacters(data) builder.Element.insertText(self, data, insertBefore) def appendChild(self, child): builder.Element.appendChild(self, child) - class Comment(builder.Comment): def __init__(self, data): - data = filter.coerceComment(data) + data = infosetFilter.coerceComment(data) builder.Comment.__init__(self, data) def _setData(self, data): - data = filter.coerceComment(data) + data = infosetFilter.coerceComment(data) self._element.text = data def _getData(self): @@ -245,9 +258,9 @@ class TreeBuilder(_base.TreeBuilder): self.elementClass = Element self.commentClass = builder.Comment - #self.fragmentClass = builder.DocumentFragment + # self.fragmentClass = builder.DocumentFragment _base.TreeBuilder.__init__(self, namespaceHTMLElements) - + def reset(self): _base.TreeBuilder.reset(self) self.insertComment = self.insertCommentInitial @@ -262,13 +275,13 @@ class TreeBuilder(_base.TreeBuilder): return self.document._elementTree else: return self.document._elementTree.getroot() - + def getFragment(self): fragment = [] element = self.openElements[0]._element if element.text: fragment.append(element.text) - fragment.extend(element.getchildren()) + fragment.extend(list(element)) if element.tail: fragment.append(element.tail) return fragment @@ -278,59 +291,79 @@ class TreeBuilder(_base.TreeBuilder): publicId = token["publicId"] systemId = token["systemId"] - if not name or ihatexml.nonXmlNameBMPRegexp.search(name) or name[0] == '"': - warnings.warn("lxml cannot represent null or non-xml doctype", DataLossWarning) + if not name: + warnings.warn("lxml cannot represent empty doctype", DataLossWarning) + self.doctype = None + else: + coercedName = self.infosetFilter.coerceElement(name) + if coercedName != name: + warnings.warn("lxml cannot represent non-xml doctype", DataLossWarning) + + doctype = self.doctypeClass(coercedName, publicId, systemId) + self.doctype = doctype - doctype = self.doctypeClass(name, publicId, systemId) - self.doctype = doctype - def insertCommentInitial(self, data, parent=None): self.initial_comments.append(data) - + + def insertCommentMain(self, data, parent=None): + if (parent == self.document and + self.document._elementTree.getroot()[-1].tag == comment_type): + warnings.warn("lxml cannot represent adjacent comments beyond the root elements", DataLossWarning) + super(TreeBuilder, self).insertComment(data, parent) + def insertRoot(self, token): """Create the document root""" - #Because of the way libxml2 works, it doesn't seem to be possible to - #alter information like the doctype after the tree has been parsed. - #Therefore we need to use the built-in parser to create our iniial - #tree, after which we can add elements like normal + # Because of the way libxml2 works, it doesn't seem to be possible to + # alter information like the doctype after the tree has been parsed. + # Therefore we need to use the built-in parser to create our iniial + # tree, after which we can add elements like normal docStr = "" - if self.doctype and self.doctype.name and not self.doctype.name.startswith('"'): - docStr += "<!DOCTYPE %s"%self.doctype.name - if (self.doctype.publicId is not None or - self.doctype.systemId is not None): - docStr += ' PUBLIC "%s" "%s"'%(self.doctype.publicId or "", - self.doctype.systemId or "") + if self.doctype: + assert self.doctype.name + docStr += "<!DOCTYPE %s" % self.doctype.name + if (self.doctype.publicId is not None or + self.doctype.systemId is not None): + docStr += (' PUBLIC "%s" ' % + (self.infosetFilter.coercePubid(self.doctype.publicId or ""))) + if self.doctype.systemId: + sysid = self.doctype.systemId + if sysid.find("'") >= 0 and sysid.find('"') >= 0: + warnings.warn("DOCTYPE system cannot contain single and double quotes", DataLossWarning) + sysid = sysid.replace("'", 'U00027') + if sysid.find("'") >= 0: + docStr += '"%s"' % sysid + else: + docStr += "'%s'" % sysid + else: + docStr += "''" docStr += ">" + if self.doctype.name != token["name"]: + warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning) docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>" - - try: - root = etree.fromstring(docStr) - except etree.XMLSyntaxError: - print docStr - raise - - #Append the initial comments: + root = etree.fromstring(docStr) + + # Append the initial comments: for comment_token in self.initial_comments: root.addprevious(etree.Comment(comment_token["data"])) - - #Create the root document and add the ElementTree to it + + # Create the root document and add the ElementTree to it self.document = self.documentClass() self.document._elementTree = root.getroottree() - + # Give the root element the right name name = token["name"] namespace = token.get("namespace", self.defaultNamespace) if namespace is None: etree_tag = name else: - etree_tag = "{%s}%s"%(namespace, name) + etree_tag = "{%s}%s" % (namespace, name) root.tag = etree_tag - - #Add the root element to the internal child/open data structures + + # Add the root element to the internal child/open data structures root_element = self.elementClass(name, namespace) root_element._element = root self.document._childNodes.append(root_element) self.openElements.append(root_element) - - #Reset to the default insert comment function - self.insertComment = super(TreeBuilder, self).insertComment + + # Reset to the default insert comment function + self.insertComment = self.insertCommentMain diff --git a/libs/html5lib/treebuilders/simpletree.py b/libs/html5lib/treebuilders/simpletree.py deleted file mode 100755 index 67fe7583..00000000 --- a/libs/html5lib/treebuilders/simpletree.py +++ /dev/null @@ -1,256 +0,0 @@ -import _base -from html5lib.constants import voidElements, namespaces, prefixes -from xml.sax.saxutils import escape - -# Really crappy basic implementation of a DOM-core like thing -class Node(_base.Node): - type = -1 - def __init__(self, name): - self.name = name - self.parent = None - self.value = None - self.childNodes = [] - self._flags = [] - - def __iter__(self): - for node in self.childNodes: - yield node - for item in node: - yield item - - def __unicode__(self): - return self.name - - def toxml(self): - raise NotImplementedError - - def printTree(self, indent=0): - tree = '\n|%s%s' % (' '* indent, unicode(self)) - for child in self.childNodes: - tree += child.printTree(indent + 2) - return tree - - def appendChild(self, node): - assert isinstance(node, Node) - if (isinstance(node, TextNode) and self.childNodes and - isinstance(self.childNodes[-1], TextNode)): - self.childNodes[-1].value += node.value - else: - self.childNodes.append(node) - node.parent = self - - def insertText(self, data, insertBefore=None): - assert isinstance(data, unicode), "data %s is of type %s expected unicode"%(repr(data), type(data)) - if insertBefore is None: - self.appendChild(TextNode(data)) - else: - self.insertBefore(TextNode(data), insertBefore) - - def insertBefore(self, node, refNode): - index = self.childNodes.index(refNode) - if (isinstance(node, TextNode) and index > 0 and - isinstance(self.childNodes[index - 1], TextNode)): - self.childNodes[index - 1].value += node.value - else: - self.childNodes.insert(index, node) - node.parent = self - - def removeChild(self, node): - try: - self.childNodes.remove(node) - except: - # XXX - raise - node.parent = None - - def cloneNode(self): - raise NotImplementedError - - def hasContent(self): - """Return true if the node has children or text""" - return bool(self.childNodes) - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class Document(Node): - type = 1 - def __init__(self): - Node.__init__(self, None) - - def __str__(self): - return "#document" - - def __unicode__(self): - return str(self) - - def appendChild(self, child): - Node.appendChild(self, child) - - def toxml(self, encoding="utf=8"): - result = "" - for child in self.childNodes: - result += child.toxml() - return result.encode(encoding) - - def hilite(self, encoding="utf-8"): - result = "<pre>" - for child in self.childNodes: - result += child.hilite() - return result.encode(encoding) + "</pre>" - - def printTree(self): - tree = unicode(self) - for child in self.childNodes: - tree += child.printTree(2) - return tree - - def cloneNode(self): - return Document() - -class DocumentFragment(Document): - type = 2 - def __str__(self): - return "#document-fragment" - - def __unicode__(self): - return str(self) - - def cloneNode(self): - return DocumentFragment() - -class DocumentType(Node): - type = 3 - def __init__(self, name, publicId, systemId): - Node.__init__(self, name) - self.publicId = publicId - self.systemId = systemId - - def __unicode__(self): - if self.publicId or self.systemId: - publicId = self.publicId or "" - systemId = self.systemId or "" - return """<!DOCTYPE %s "%s" "%s">"""%( - self.name, publicId, systemId) - - else: - return u"<!DOCTYPE %s>" % self.name - - - toxml = __unicode__ - - def hilite(self): - return '<code class="markup doctype"><!DOCTYPE %s></code>' % self.name - - def cloneNode(self): - return DocumentType(self.name, self.publicId, self.systemId) - -class TextNode(Node): - type = 4 - def __init__(self, value): - Node.__init__(self, None) - self.value = value - - def __unicode__(self): - return u"\"%s\"" % self.value - - def toxml(self): - return escape(self.value) - - hilite = toxml - - def cloneNode(self): - return TextNode(self.value) - -class Element(Node): - type = 5 - def __init__(self, name, namespace=None): - Node.__init__(self, name) - self.namespace = namespace - self.attributes = {} - - def __unicode__(self): - if self.namespace == None: - return u"<%s>" % self.name - else: - return u"<%s %s>"%(prefixes[self.namespace], self.name) - - def toxml(self): - result = '<' + self.name - if self.attributes: - for name,value in self.attributes.iteritems(): - result += u' %s="%s"' % (name, escape(value,{'"':'"'})) - if self.childNodes: - result += '>' - for child in self.childNodes: - result += child.toxml() - result += u'</%s>' % self.name - else: - result += u'/>' - return result - - def hilite(self): - result = '<<code class="markup element-name">%s</code>' % self.name - if self.attributes: - for name, value in self.attributes.iteritems(): - result += ' <code class="markup attribute-name">%s</code>=<code class="markup attribute-value">"%s"</code>' % (name, escape(value, {'"':'"'})) - if self.childNodes: - result += ">" - for child in self.childNodes: - result += child.hilite() - elif self.name in voidElements: - return result + ">" - return result + '</<code class="markup element-name">%s</code>>' % self.name - - def printTree(self, indent): - tree = '\n|%s%s' % (' '*indent, unicode(self)) - indent += 2 - if self.attributes: - for name, value in sorted(self.attributes.iteritems()): - if isinstance(name, tuple): - name = "%s %s"%(name[0], name[1]) - tree += '\n|%s%s="%s"' % (' ' * indent, name, value) - for child in self.childNodes: - tree += child.printTree(indent) - return tree - - def cloneNode(self): - newNode = Element(self.name) - if hasattr(self, 'namespace'): - newNode.namespace = self.namespace - for attr, value in self.attributes.iteritems(): - newNode.attributes[attr] = value - return newNode - -class CommentNode(Node): - type = 6 - def __init__(self, data): - Node.__init__(self, None) - self.data = data - - def __unicode__(self): - return "<!-- %s -->" % self.data - - def toxml(self): - return "<!--%s-->" % self.data - - def hilite(self): - return '<code class="markup comment"><!--%s--></code>' % escape(self.data) - - def cloneNode(self): - return CommentNode(self.data) - -class TreeBuilder(_base.TreeBuilder): - documentClass = Document - doctypeClass = DocumentType - elementClass = Element - commentClass = CommentNode - fragmentClass = DocumentFragment - - def testSerializer(self, node): - return node.printTree() diff --git a/libs/html5lib/treebuilders/soup.py b/libs/html5lib/treebuilders/soup.py deleted file mode 100644 index 9bc5ff0e..00000000 --- a/libs/html5lib/treebuilders/soup.py +++ /dev/null @@ -1,236 +0,0 @@ -import warnings - -warnings.warn("BeautifulSoup 3.x (as of 3.1) is not fully compatible with html5lib and support will be removed in the future", DeprecationWarning) - -from BeautifulSoup import BeautifulSoup, Tag, NavigableString, Comment, Declaration - -import _base -from html5lib.constants import namespaces, DataLossWarning - -class AttrList(object): - def __init__(self, element): - self.element = element - self.attrs = dict(self.element.attrs) - def __iter__(self): - return self.attrs.items().__iter__() - def __setitem__(self, name, value): - "set attr", name, value - self.element[name] = value - def items(self): - return self.attrs.items() - def keys(self): - return self.attrs.keys() - def __getitem__(self, name): - return self.attrs[name] - def __contains__(self, name): - return name in self.attrs.keys() - def __eq__(self, other): - if len(self.keys()) != len(other.keys()): - return False - for item in self.keys(): - if item not in other: - return False - if self[item] != other[item]: - return False - return True - -class Element(_base.Node): - def __init__(self, element, soup, namespace): - _base.Node.__init__(self, element.name) - self.element = element - self.soup = soup - self.namespace = namespace - - def _nodeIndex(self, node, refNode): - # Finds a node by identity rather than equality - for index in range(len(self.element.contents)): - if id(self.element.contents[index]) == id(refNode.element): - return index - return None - - def appendChild(self, node): - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[-1].__class__ == NavigableString): - # Concatenate new text onto old text node - # (TODO: This has O(n^2) performance, for input like "a</a>a</a>a</a>...") - newStr = NavigableString(self.element.contents[-1]+node.element) - - # Remove the old text node - # (Can't simply use .extract() by itself, because it fails if - # an equal text node exists within the parent node) - oldElement = self.element.contents[-1] - del self.element.contents[-1] - oldElement.parent = None - oldElement.extract() - - self.element.insert(len(self.element.contents), newStr) - else: - self.element.insert(len(self.element.contents), node.element) - node.parent = self - - def getAttributes(self): - return AttrList(self.element) - - def setAttributes(self, attributes): - if attributes: - for name, value in attributes.items(): - self.element[name] = value - - attributes = property(getAttributes, setAttributes) - - def insertText(self, data, insertBefore=None): - text = TextNode(NavigableString(data), self.soup) - if insertBefore: - self.insertBefore(text, insertBefore) - else: - self.appendChild(text) - - def insertBefore(self, node, refNode): - index = self._nodeIndex(node, refNode) - if (node.element.__class__ == NavigableString and self.element.contents - and self.element.contents[index-1].__class__ == NavigableString): - # (See comments in appendChild) - newStr = NavigableString(self.element.contents[index-1]+node.element) - oldNode = self.element.contents[index-1] - del self.element.contents[index-1] - oldNode.parent = None - oldNode.extract() - - self.element.insert(index-1, newStr) - else: - self.element.insert(index, node.element) - node.parent = self - - def removeChild(self, node): - index = self._nodeIndex(node.parent, node) - del node.parent.element.contents[index] - node.element.parent = None - node.element.extract() - node.parent = None - - def reparentChildren(self, newParent): - while self.element.contents: - child = self.element.contents[0] - child.extract() - if isinstance(child, Tag): - newParent.appendChild(Element(child, self.soup, namespaces["html"])) - else: - newParent.appendChild(TextNode(child, self.soup)) - - def cloneNode(self): - node = Element(Tag(self.soup, self.element.name), self.soup, self.namespace) - for key,value in self.attributes: - node.attributes[key] = value - return node - - def hasContent(self): - return self.element.contents - - def getNameTuple(self): - if self.namespace == None: - return namespaces["html"], self.name - else: - return self.namespace, self.name - - nameTuple = property(getNameTuple) - -class TextNode(Element): - def __init__(self, element, soup): - _base.Node.__init__(self, None) - self.element = element - self.soup = soup - - def cloneNode(self): - raise NotImplementedError - -class TreeBuilder(_base.TreeBuilder): - def __init__(self, namespaceHTMLElements): - if namespaceHTMLElements: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - _base.TreeBuilder.__init__(self, namespaceHTMLElements) - - def documentClass(self): - self.soup = BeautifulSoup("") - return Element(self.soup, self.soup, None) - - def insertDoctype(self, token): - name = token["name"] - publicId = token["publicId"] - systemId = token["systemId"] - - if publicId: - self.soup.insert(0, Declaration("DOCTYPE %s PUBLIC \"%s\" \"%s\""%(name, publicId, systemId or ""))) - elif systemId: - self.soup.insert(0, Declaration("DOCTYPE %s SYSTEM \"%s\""% - (name, systemId))) - else: - self.soup.insert(0, Declaration("DOCTYPE %s"%name)) - - def elementClass(self, name, namespace): - if namespace is not None: - warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning) - return Element(Tag(self.soup, name), self.soup, namespace) - - def commentClass(self, data): - return TextNode(Comment(data), self.soup) - - def fragmentClass(self): - self.soup = BeautifulSoup("") - self.soup.name = "[document_fragment]" - return Element(self.soup, self.soup, None) - - def appendChild(self, node): - self.soup.insert(len(self.soup.contents), node.element) - - def testSerializer(self, element): - return testSerializer(element) - - def getDocument(self): - return self.soup - - def getFragment(self): - return _base.TreeBuilder.getFragment(self).element - -def testSerializer(element): - import re - rv = [] - def serializeElement(element, indent=0): - if isinstance(element, Declaration): - doctype_regexp = r'DOCTYPE\s+(?P<name>[^\s]*)( PUBLIC "(?P<publicId>.*)" "(?P<systemId1>.*)"| SYSTEM "(?P<systemId2>.*)")?' - m = re.compile(doctype_regexp).match(element.string) - assert m is not None, "DOCTYPE did not match expected format" - name = m.group('name') - publicId = m.group('publicId') - if publicId is not None: - systemId = m.group('systemId1') or "" - else: - systemId = m.group('systemId2') - - if publicId is not None or systemId is not None: - rv.append("""|%s<!DOCTYPE %s "%s" "%s">"""% - (' '*indent, name, publicId or "", systemId or "")) - else: - rv.append("|%s<!DOCTYPE %s>"%(' '*indent, name)) - - elif isinstance(element, BeautifulSoup): - if element.name == "[document_fragment]": - rv.append("#document-fragment") - else: - rv.append("#document") - - elif isinstance(element, Comment): - rv.append("|%s<!-- %s -->"%(' '*indent, element.string)) - elif isinstance(element, unicode): - rv.append("|%s\"%s\"" %(' '*indent, element)) - else: - rv.append("|%s<%s>"%(' '*indent, element.name)) - if element.attrs: - for name, value in sorted(element.attrs): - rv.append('|%s%s="%s"' % (' '*(indent+2), name, value)) - indent += 2 - if hasattr(element, "contents"): - for child in element.contents: - serializeElement(child, indent) - serializeElement(element, 0) - - return "\n".join(rv) diff --git a/libs/html5lib/treewalkers/__init__.py b/libs/html5lib/treewalkers/__init__.py index 3a606a8b..18124e75 100644 --- a/libs/html5lib/treewalkers/__init__.py +++ b/libs/html5lib/treewalkers/__init__.py @@ -8,23 +8,27 @@ implements a 'serialize' method taking a tree as sole argument and returning an iterator generating tokens. """ +from __future__ import absolute_import, division, unicode_literals + +import sys + +from ..utils import default_etree + treeWalkerCache = {} + def getTreeWalker(treeType, implementation=None, **kwargs): """Get a TreeWalker class for various types of tree with built-in support treeType - the name of the tree type required (case-insensitive). Supported - values are "simpletree", "dom", "etree" and "beautifulsoup" + values are: - "simpletree" - a built-in DOM-ish tree type with support for some - more pythonic idioms. "dom" - The xml.dom.minidom DOM implementation "pulldom" - The xml.dom.pulldom event stream "etree" - A generic walker for tree implementations exposing an elementtree-like interface (known to work with ElementTree, cElementTree and lxml.etree). "lxml" - Optimized walker for lxml.etree - "beautifulsoup" - Beautiful soup (if installed) "genshi" - a Genshi stream implementation - (Currently applies to the "etree" tree type only). A module @@ -33,20 +37,21 @@ def getTreeWalker(treeType, implementation=None, **kwargs): treeType = treeType.lower() if treeType not in treeWalkerCache: - if treeType in ("dom", "pulldom", "simpletree"): - mod = __import__(treeType, globals()) + if treeType in ("dom", "pulldom"): + name = "%s.%s" % (__name__, treeType) + __import__(name) + mod = sys.modules[name] treeWalkerCache[treeType] = mod.TreeWalker elif treeType == "genshi": - import genshistream + from . import genshistream treeWalkerCache[treeType] = genshistream.TreeWalker - elif treeType == "beautifulsoup": - import soup - treeWalkerCache[treeType] = soup.TreeWalker elif treeType == "lxml": - import lxmletree + from . import lxmletree treeWalkerCache[treeType] = lxmletree.TreeWalker elif treeType == "etree": - import etree + from . import etree + if implementation is None: + implementation = default_etree # XXX: NEVER cache here, caching is done in the etree submodule return etree.getETreeModule(implementation, **kwargs).TreeWalker return treeWalkerCache.get(treeType) diff --git a/libs/html5lib/treewalkers/_base.py b/libs/html5lib/treewalkers/_base.py index 5929ba05..34252e50 100644 --- a/libs/html5lib/treewalkers/_base.py +++ b/libs/html5lib/treewalkers/_base.py @@ -1,94 +1,9 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type, string_types + import gettext _ = gettext.gettext -from html5lib.constants import voidElements, spaceCharacters -spaceCharacters = u"".join(spaceCharacters) - -class TreeWalker(object): - def __init__(self, tree): - self.tree = tree - - def __iter__(self): - raise NotImplementedError - - def error(self, msg): - return {"type": "SerializeError", "data": msg} - - def normalizeAttrs(self, attrs): - newattrs = {} - if attrs: - #TODO: treewalkers should always have attrs - for (namespace,name),value in attrs.iteritems(): - namespace = unicode(namespace) if namespace else None - name = unicode(name) - value = unicode(value) - newattrs[(namespace,name)] = value - return newattrs - - def emptyTag(self, namespace, name, attrs, hasChildren=False): - yield {"type": "EmptyTag", "name": unicode(name), - "namespace":unicode(namespace), - "data": self.normalizeAttrs(attrs)} - if hasChildren: - yield self.error(_("Void element has children")) - - def startTag(self, namespace, name, attrs): - return {"type": "StartTag", - "name": unicode(name), - "namespace":unicode(namespace), - "data": self.normalizeAttrs(attrs)} - - def endTag(self, namespace, name): - return {"type": "EndTag", - "name": unicode(name), - "namespace":unicode(namespace), - "data": {}} - - def text(self, data): - data = unicode(data) - middle = data.lstrip(spaceCharacters) - left = data[:len(data)-len(middle)] - if left: - yield {"type": "SpaceCharacters", "data": left} - data = middle - middle = data.rstrip(spaceCharacters) - right = data[len(middle):] - if middle: - yield {"type": "Characters", "data": middle} - if right: - yield {"type": "SpaceCharacters", "data": right} - - def comment(self, data): - return {"type": "Comment", "data": unicode(data)} - - def doctype(self, name, publicId=None, systemId=None, correct=True): - return {"type": "Doctype", - "name": name is not None and unicode(name) or u"", - "publicId": publicId, - "systemId": systemId, - "correct": correct} - - def entity(self, name): - return {"type": "Entity", "name": unicode(name)} - - def unknown(self, nodeType): - return self.error(_("Unknown node type: ") + nodeType) - -class RecursiveTreeWalker(TreeWalker): - def walkChildren(self, node): - raise NodeImplementedError - - def element(self, node, namespace, name, attrs, hasChildren): - if name in voidElements: - for token in self.emptyTag(namespace, name, attrs, hasChildren): - yield token - else: - yield self.startTag(name, attrs) - if hasChildren: - for token in self.walkChildren(node): - yield token - yield self.endTag(name) - from xml.dom import Node DOCUMENT = Node.DOCUMENT_NODE @@ -99,16 +14,127 @@ COMMENT = Node.COMMENT_NODE ENTITY = Node.ENTITY_NODE UNKNOWN = "<#UNKNOWN#>" +from ..constants import voidElements, spaceCharacters +spaceCharacters = "".join(spaceCharacters) + + +def to_text(s, blank_if_none=True): + """Wrapper around six.text_type to convert None to empty string""" + if s is None: + if blank_if_none: + return "" + else: + return None + elif isinstance(s, text_type): + return s + else: + return text_type(s) + + +def is_text_or_none(string): + """Wrapper around isinstance(string_types) or is None""" + return string is None or isinstance(string, string_types) + + +class TreeWalker(object): + def __init__(self, tree): + self.tree = tree + + def __iter__(self): + raise NotImplementedError + + def error(self, msg): + return {"type": "SerializeError", "data": msg} + + def emptyTag(self, namespace, name, attrs, hasChildren=False): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(name) + assert all((namespace is None or isinstance(namespace, string_types)) and + isinstance(name, string_types) and + isinstance(value, string_types) + for (namespace, name), value in attrs.items()) + + yield {"type": "EmptyTag", "name": to_text(name, False), + "namespace": to_text(namespace), + "data": attrs} + if hasChildren: + yield self.error(_("Void element has children")) + + def startTag(self, namespace, name, attrs): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(name) + assert all((namespace is None or isinstance(namespace, string_types)) and + isinstance(name, string_types) and + isinstance(value, string_types) + for (namespace, name), value in attrs.items()) + + return {"type": "StartTag", + "name": text_type(name), + "namespace": to_text(namespace), + "data": dict(((to_text(namespace, False), to_text(name)), + to_text(value, False)) + for (namespace, name), value in attrs.items())} + + def endTag(self, namespace, name): + assert namespace is None or isinstance(namespace, string_types), type(namespace) + assert isinstance(name, string_types), type(namespace) + + return {"type": "EndTag", + "name": to_text(name, False), + "namespace": to_text(namespace), + "data": {}} + + def text(self, data): + assert isinstance(data, string_types), type(data) + + data = to_text(data) + middle = data.lstrip(spaceCharacters) + left = data[:len(data) - len(middle)] + if left: + yield {"type": "SpaceCharacters", "data": left} + data = middle + middle = data.rstrip(spaceCharacters) + right = data[len(middle):] + if middle: + yield {"type": "Characters", "data": middle} + if right: + yield {"type": "SpaceCharacters", "data": right} + + def comment(self, data): + assert isinstance(data, string_types), type(data) + + return {"type": "Comment", "data": text_type(data)} + + def doctype(self, name, publicId=None, systemId=None, correct=True): + assert is_text_or_none(name), type(name) + assert is_text_or_none(publicId), type(publicId) + assert is_text_or_none(systemId), type(systemId) + + return {"type": "Doctype", + "name": to_text(name), + "publicId": to_text(publicId), + "systemId": to_text(systemId), + "correct": to_text(correct)} + + def entity(self, name): + assert isinstance(name, string_types), type(name) + + return {"type": "Entity", "name": text_type(name)} + + def unknown(self, nodeType): + return self.error(_("Unknown node type: ") + nodeType) + + class NonRecursiveTreeWalker(TreeWalker): def getNodeDetails(self, node): raise NotImplementedError - + def getFirstChild(self, node): raise NotImplementedError - + def getNextSibling(self, node): raise NotImplementedError - + def getParentNode(self, node): raise NotImplementedError @@ -118,7 +144,6 @@ class NonRecursiveTreeWalker(TreeWalker): details = self.getNodeDetails(currentNode) type, details = details[0], details[1:] hasChildren = False - endTag = None if type == DOCTYPE: yield self.doctype(*details) @@ -130,12 +155,11 @@ class NonRecursiveTreeWalker(TreeWalker): elif type == ELEMENT: namespace, name, attributes, hasChildren = details if name in voidElements: - for token in self.emptyTag(namespace, name, attributes, + for token in self.emptyTag(namespace, name, attributes, hasChildren): yield token hasChildren = False else: - endTag = name yield self.startTag(namespace, name, attributes) elif type == COMMENT: @@ -149,12 +173,12 @@ class NonRecursiveTreeWalker(TreeWalker): else: yield self.unknown(details[0]) - + if hasChildren: firstChild = self.getFirstChild(currentNode) else: firstChild = None - + if firstChild is not None: currentNode = firstChild else: diff --git a/libs/html5lib/treewalkers/dom.py b/libs/html5lib/treewalkers/dom.py index 383b46cb..a01287a9 100644 --- a/libs/html5lib/treewalkers/dom.py +++ b/libs/html5lib/treewalkers/dom.py @@ -1,10 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + from xml.dom import Node import gettext _ = gettext.gettext -import _base -from html5lib.constants import voidElements +from . import _base + class TreeWalker(_base.NonRecursiveTreeWalker): def getNodeDetails(self, node): @@ -16,10 +18,13 @@ class TreeWalker(_base.NonRecursiveTreeWalker): elif node.nodeType == Node.ELEMENT_NODE: attrs = {} - for attr in node.attributes.keys(): + for attr in list(node.attributes.keys()): attr = node.getAttributeNode(attr) - attrs[(attr.namespaceURI,attr.localName)] = attr.value - return (_base.ELEMENT, node.namespaceURI, node.nodeName, + if attr.namespaceURI: + attrs[(attr.namespaceURI, attr.localName)] = attr.value + else: + attrs[(None, attr.name)] = attr.value + return (_base.ELEMENT, node.namespaceURI, node.nodeName, attrs, node.hasChildNodes()) elif node.nodeType == Node.COMMENT_NODE: diff --git a/libs/html5lib/treewalkers/etree.py b/libs/html5lib/treewalkers/etree.py index 13b03194..fd8a9cc9 100644 --- a/libs/html5lib/treewalkers/etree.py +++ b/libs/html5lib/treewalkers/etree.py @@ -1,33 +1,28 @@ +from __future__ import absolute_import, division, unicode_literals + +try: + from collections import OrderedDict +except ImportError: + try: + from ordereddict import OrderedDict + except ImportError: + OrderedDict = dict import gettext _ = gettext.gettext -try: - from types import ModuleType -except: - from new import module as ModuleType -import copy import re -import _base -from html5lib.constants import voidElements +from six import text_type + +from . import _base +from ..utils import moduleFactoryFactory tag_regexp = re.compile("{([^}]*)}(.*)") -moduleCache = {} - -def getETreeModule(ElementTreeImplementation): - name = "_" + ElementTreeImplementation.__name__+"builder" - if name in moduleCache: - return moduleCache[name] - else: - mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder") - objs = getETreeBuilder(ElementTreeImplementation) - mod.__dict__.update(objs) - moduleCache[name] = mod - return mod def getETreeBuilder(ElementTreeImplementation): ElementTree = ElementTreeImplementation + ElementTreeCommentType = ElementTree.Comment("asd").tag class TreeWalker(_base.NonRecursiveTreeWalker): """Given the particular ElementTree representation, this implementation, @@ -35,16 +30,16 @@ def getETreeBuilder(ElementTreeImplementation): content: 1. The current element - + 2. The index of the element relative to its parent - + 3. A stack of ancestor elements - + 4. A flag "text", "tail" or None to indicate if the current node is a text node; either the text or tail of the current element (1) """ def getNodeDetails(self, node): - if isinstance(node, tuple): # It might be the root Element + if isinstance(node, tuple): # It might be the root Element elt, key, parents, flag = node if flag in ("text", "tail"): return _base.TEXT, getattr(elt, flag) @@ -54,41 +49,41 @@ def getETreeBuilder(ElementTreeImplementation): if not(hasattr(node, "tag")): node = node.getroot() - if node.tag in ("<DOCUMENT_ROOT>", "<DOCUMENT_FRAGMENT>"): + if node.tag in ("DOCUMENT_ROOT", "DOCUMENT_FRAGMENT"): return (_base.DOCUMENT,) elif node.tag == "<!DOCTYPE>": - return (_base.DOCTYPE, node.text, + return (_base.DOCTYPE, node.text, node.get("publicId"), node.get("systemId")) - elif node.tag == ElementTree.Comment: + elif node.tag == ElementTreeCommentType: return _base.COMMENT, node.text else: - assert type(node.tag) in (str, unicode), type(node.tag) - #This is assumed to be an ordinary element + assert type(node.tag) == text_type, type(node.tag) + # This is assumed to be an ordinary element match = tag_regexp.match(node.tag) if match: namespace, tag = match.groups() else: namespace = None tag = node.tag - attrs = {} - for name, value in node.attrib.items(): + attrs = OrderedDict() + for name, value in list(node.attrib.items()): match = tag_regexp.match(name) if match: - attrs[(match.group(1),match.group(2))] = value + attrs[(match.group(1), match.group(2))] = value else: - attrs[(None,name)] = value - return (_base.ELEMENT, namespace, tag, + attrs[(None, name)] = value + return (_base.ELEMENT, namespace, tag, attrs, len(node) or node.text) - + def getFirstChild(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: element, key, parents, flag = node, None, [], None - + if flag in ("text", "tail"): return None else: @@ -99,13 +94,13 @@ def getETreeBuilder(ElementTreeImplementation): return element[0], 0, parents, None else: return None - + def getNextSibling(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: return None - + if flag == "text": if len(element): parents.append(element) @@ -116,16 +111,16 @@ def getETreeBuilder(ElementTreeImplementation): if element.tail and flag != "tail": return element, key, parents, "tail" elif key < len(parents[-1]) - 1: - return parents[-1][key+1], key+1, parents, None + return parents[-1][key + 1], key + 1, parents, None else: return None - + def getParentNode(self, node): if isinstance(node, tuple): element, key, parents, flag = node else: return None - + if flag == "text": if not parents: return element @@ -139,3 +134,5 @@ def getETreeBuilder(ElementTreeImplementation): return parent, list(parents[-1]).index(parent), parents, None return locals() + +getETreeModule = moduleFactoryFactory(getETreeBuilder) diff --git a/libs/html5lib/treewalkers/genshistream.py b/libs/html5lib/treewalkers/genshistream.py index ef71a83e..f559c45d 100644 --- a/libs/html5lib/treewalkers/genshistream.py +++ b/libs/html5lib/treewalkers/genshistream.py @@ -1,50 +1,49 @@ +from __future__ import absolute_import, division, unicode_literals + +from genshi.core import QName from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT -from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT -from genshi.output import NamespaceFlattener +from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT -import _base +from . import _base + +from ..constants import voidElements, namespaces -from html5lib.constants import voidElements class TreeWalker(_base.TreeWalker): def __iter__(self): - depth = 0 - ignore_until = None + # Buffer the events so we can pass in the following one previous = None for event in self.tree: if previous is not None: - if previous[0] == START: - depth += 1 - if ignore_until <= depth: - ignore_until = None - if ignore_until is None: - for token in self.tokens(previous, event): - yield token - if token["type"] == "EmptyTag": - ignore_until = depth - if previous[0] == END: - depth -= 1 - previous = event - if previous is not None: - if ignore_until is None or ignore_until <= depth: - for token in self.tokens(previous, None): + for token in self.tokens(previous, event): yield token - elif ignore_until is not None: - raise ValueError("Illformed DOM event stream: void element without END_ELEMENT") + previous = event + + # Don't forget the final event! + if previous is not None: + for token in self.tokens(previous, None): + yield token def tokens(self, event, next): kind, data, pos = event if kind == START: - tag, attrib = data + tag, attribs = data name = tag.localname namespace = tag.namespace - if tag in voidElements: - for token in self.emptyTag(namespace, name, list(attrib), - not next or next[0] != END + converted_attribs = {} + for k, v in attribs: + if isinstance(k, QName): + converted_attribs[(k.namespace, k.localname)] = v + else: + converted_attribs[(None, k)] = v + + if namespace == namespaces["html"] and name in voidElements: + for token in self.emptyTag(namespace, name, converted_attribs, + not next or next[0] != END or next[1] != tag): yield token else: - yield self.startTag(namespace, name, list(attrib)) + yield self.startTag(namespace, name, converted_attribs) elif kind == END: name = data.localname @@ -62,8 +61,8 @@ class TreeWalker(_base.TreeWalker): elif kind == DOCTYPE: yield self.doctype(*data) - elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \ - START_CDATA, END_CDATA, PI): + elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, + START_CDATA, END_CDATA, PI): pass else: diff --git a/libs/html5lib/treewalkers/lxmletree.py b/libs/html5lib/treewalkers/lxmletree.py index 46f4908c..375cc2e8 100644 --- a/libs/html5lib/treewalkers/lxmletree.py +++ b/libs/html5lib/treewalkers/lxmletree.py @@ -1,186 +1,208 @@ -from lxml import etree -from html5lib.treebuilders.etree import tag_regexp - -from gettext import gettext -_ = gettext - -import _base - -from html5lib.constants import voidElements -from html5lib import ihatexml - -class Root(object): - def __init__(self, et): - self.elementtree = et - self.children = [] - if et.docinfo.internalDTD: - self.children.append(Doctype(self, et.docinfo.root_name, - et.docinfo.public_id, - et.docinfo.system_url)) - root = et.getroot() - node = root - - while node.getprevious() is not None: - node = node.getprevious() - while node is not None: - self.children.append(node) - node = node.getnext() - - self.text = None - self.tail = None - - def __getitem__(self, key): - return self.children[key] - - def getnext(self): - return None - - def __len__(self): - return 1 - -class Doctype(object): - def __init__(self, root_node, name, public_id, system_id): - self.root_node = root_node - self.name = name - self.public_id = public_id - self.system_id = system_id - - self.text = None - self.tail = None - - def getnext(self): - return self.root_node.children[1] - -class FragmentRoot(Root): - def __init__(self, children): - self.children = [FragmentWrapper(self, child) for child in children] - self.text = self.tail = None - - def getnext(self): - return None - -class FragmentWrapper(object): - def __init__(self, fragment_root, obj): - self.root_node = fragment_root - self.obj = obj - if hasattr(self.obj, 'text'): - self.text = self.obj.text - else: - self.text = None - if hasattr(self.obj, 'tail'): - self.tail = self.obj.tail - else: - self.tail = None - self.isstring = isinstance(obj, basestring) - - def __getattr__(self, name): - return getattr(self.obj, name) - - def getnext(self): - siblings = self.root_node.children - idx = siblings.index(self) - if idx < len(siblings) - 1: - return siblings[idx + 1] - else: - return None - - def __getitem__(self, key): - return self.obj[key] - - def __nonzero__(self): - return bool(self.obj) - - def getparent(self): - return None - - def __str__(self): - return str(self.obj) - - def __unicode__(self): - return unicode(self.obj) - - def __len__(self): - return len(self.obj) - - -class TreeWalker(_base.NonRecursiveTreeWalker): - def __init__(self, tree): - if hasattr(tree, "getroot"): - tree = Root(tree) - elif isinstance(tree, list): - tree = FragmentRoot(tree) - _base.NonRecursiveTreeWalker.__init__(self, tree) - self.filter = ihatexml.InfosetFilter() - def getNodeDetails(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - return _base.TEXT, getattr(node, key) - - elif isinstance(node, Root): - return (_base.DOCUMENT,) - - elif isinstance(node, Doctype): - return _base.DOCTYPE, node.name, node.public_id, node.system_id - - elif isinstance(node, FragmentWrapper) and node.isstring: - return _base.TEXT, node - - elif node.tag == etree.Comment: - return _base.COMMENT, node.text - - elif node.tag == etree.Entity: - return _base.ENTITY, node.text[1:-1] # strip &; - - else: - #This is assumed to be an ordinary element - match = tag_regexp.match(node.tag) - if match: - namespace, tag = match.groups() - else: - namespace = None - tag = node.tag - attrs = {} - for name, value in node.attrib.items(): - match = tag_regexp.match(name) - if match: - attrs[(match.group(1),match.group(2))] = value - else: - attrs[(None,name)] = value - return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), - attrs, len(node) > 0 or node.text) - - def getFirstChild(self, node): - assert not isinstance(node, tuple), _("Text nodes have no children") - - assert len(node) or node.text, "Node has no children" - if node.text: - return (node, "text") - else: - return node[0] - - def getNextSibling(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - if key == "text": - # XXX: we cannot use a "bool(node) and node[0] or None" construct here - # because node[0] might evaluate to False if it has no child element - if len(node): - return node[0] - else: - return None - else: # tail - return node.getnext() - - return node.tail and (node, "tail") or node.getnext() - - def getParentNode(self, node): - if isinstance(node, tuple): # Text node - node, key = node - assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key - if key == "text": - return node - # else: fallback to "normal" processing - - return node.getparent() +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from lxml import etree +from ..treebuilders.etree import tag_regexp + +from gettext import gettext +_ = gettext + +from . import _base + +from .. import ihatexml + + +def ensure_str(s): + if s is None: + return None + elif isinstance(s, text_type): + return s + else: + return s.decode("utf-8", "strict") + + +class Root(object): + def __init__(self, et): + self.elementtree = et + self.children = [] + if et.docinfo.internalDTD: + self.children.append(Doctype(self, + ensure_str(et.docinfo.root_name), + ensure_str(et.docinfo.public_id), + ensure_str(et.docinfo.system_url))) + root = et.getroot() + node = root + + while node.getprevious() is not None: + node = node.getprevious() + while node is not None: + self.children.append(node) + node = node.getnext() + + self.text = None + self.tail = None + + def __getitem__(self, key): + return self.children[key] + + def getnext(self): + return None + + def __len__(self): + return 1 + + +class Doctype(object): + def __init__(self, root_node, name, public_id, system_id): + self.root_node = root_node + self.name = name + self.public_id = public_id + self.system_id = system_id + + self.text = None + self.tail = None + + def getnext(self): + return self.root_node.children[1] + + +class FragmentRoot(Root): + def __init__(self, children): + self.children = [FragmentWrapper(self, child) for child in children] + self.text = self.tail = None + + def getnext(self): + return None + + +class FragmentWrapper(object): + def __init__(self, fragment_root, obj): + self.root_node = fragment_root + self.obj = obj + if hasattr(self.obj, 'text'): + self.text = ensure_str(self.obj.text) + else: + self.text = None + if hasattr(self.obj, 'tail'): + self.tail = ensure_str(self.obj.tail) + else: + self.tail = None + self.isstring = isinstance(obj, str) or isinstance(obj, bytes) + # Support for bytes here is Py2 + if self.isstring: + self.obj = ensure_str(self.obj) + + def __getattr__(self, name): + return getattr(self.obj, name) + + def getnext(self): + siblings = self.root_node.children + idx = siblings.index(self) + if idx < len(siblings) - 1: + return siblings[idx + 1] + else: + return None + + def __getitem__(self, key): + return self.obj[key] + + def __bool__(self): + return bool(self.obj) + + def getparent(self): + return None + + def __str__(self): + return str(self.obj) + + def __unicode__(self): + return str(self.obj) + + def __len__(self): + return len(self.obj) + + +class TreeWalker(_base.NonRecursiveTreeWalker): + def __init__(self, tree): + if hasattr(tree, "getroot"): + tree = Root(tree) + elif isinstance(tree, list): + tree = FragmentRoot(tree) + _base.NonRecursiveTreeWalker.__init__(self, tree) + self.filter = ihatexml.InfosetFilter() + + def getNodeDetails(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + return _base.TEXT, ensure_str(getattr(node, key)) + + elif isinstance(node, Root): + return (_base.DOCUMENT,) + + elif isinstance(node, Doctype): + return _base.DOCTYPE, node.name, node.public_id, node.system_id + + elif isinstance(node, FragmentWrapper) and node.isstring: + return _base.TEXT, node.obj + + elif node.tag == etree.Comment: + return _base.COMMENT, ensure_str(node.text) + + elif node.tag == etree.Entity: + return _base.ENTITY, ensure_str(node.text)[1:-1] # strip &; + + else: + # This is assumed to be an ordinary element + match = tag_regexp.match(ensure_str(node.tag)) + if match: + namespace, tag = match.groups() + else: + namespace = None + tag = ensure_str(node.tag) + attrs = {} + for name, value in list(node.attrib.items()): + name = ensure_str(name) + value = ensure_str(value) + match = tag_regexp.match(name) + if match: + attrs[(match.group(1), match.group(2))] = value + else: + attrs[(None, name)] = value + return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag), + attrs, len(node) > 0 or node.text) + + def getFirstChild(self, node): + assert not isinstance(node, tuple), _("Text nodes have no children") + + assert len(node) or node.text, "Node has no children" + if node.text: + return (node, "text") + else: + return node[0] + + def getNextSibling(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + if key == "text": + # XXX: we cannot use a "bool(node) and node[0] or None" construct here + # because node[0] might evaluate to False if it has no child element + if len(node): + return node[0] + else: + return None + else: # tail + return node.getnext() + + return (node, "tail") if node.tail else node.getnext() + + def getParentNode(self, node): + if isinstance(node, tuple): # Text node + node, key = node + assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key + if key == "text": + return node + # else: fallback to "normal" processing + + return node.getparent() diff --git a/libs/html5lib/treewalkers/pulldom.py b/libs/html5lib/treewalkers/pulldom.py index 1f8b95b8..0b0f515f 100644 --- a/libs/html5lib/treewalkers/pulldom.py +++ b/libs/html5lib/treewalkers/pulldom.py @@ -1,9 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \ COMMENT, IGNORABLE_WHITESPACE, CHARACTERS -import _base +from . import _base + +from ..constants import voidElements -from html5lib.constants import voidElements class TreeWalker(_base.TreeWalker): def __iter__(self): @@ -11,7 +14,7 @@ class TreeWalker(_base.TreeWalker): previous = None for event in self.tree: if previous is not None and \ - (ignore_until is None or previous[1] is ignore_until): + (ignore_until is None or previous[1] is ignore_until): if previous[1] is ignore_until: ignore_until = None for token in self.tokens(previous, event): @@ -31,9 +34,9 @@ class TreeWalker(_base.TreeWalker): name = node.nodeName namespace = node.namespaceURI attrs = {} - for attr in node.attributes.keys(): + for attr in list(node.attributes.keys()): attr = node.getAttributeNode(attr) - attrs[(attr.namespaceURI,attr.localName)] = attr.value + attrs[(attr.namespaceURI, attr.localName)] = attr.value if name in voidElements: for token in self.emptyTag(namespace, name, diff --git a/libs/html5lib/treewalkers/simpletree.py b/libs/html5lib/treewalkers/simpletree.py deleted file mode 100644 index 9e6bd4c5..00000000 --- a/libs/html5lib/treewalkers/simpletree.py +++ /dev/null @@ -1,78 +0,0 @@ -import gettext -_ = gettext.gettext - -import _base - -class TreeWalker(_base.NonRecursiveTreeWalker): - """Given that simpletree has no performant way of getting a node's - next sibling, this implementation returns "nodes" as tuples with the - following content: - - 1. The parent Node (Element, Document or DocumentFragment) - - 2. The child index of the current node in its parent's children list - - 3. A list used as a stack of all ancestors. It is a pair tuple whose - first item is a parent Node and second item is a child index. - """ - - def getNodeDetails(self, node): - if isinstance(node, tuple): # It might be the root Node - parent, idx, parents = node - node = parent.childNodes[idx] - - # testing node.type allows us not to import treebuilders.simpletree - if node.type in (1, 2): # Document or DocumentFragment - return (_base.DOCUMENT,) - - elif node.type == 3: # DocumentType - return _base.DOCTYPE, node.name, node.publicId, node.systemId - - elif node.type == 4: # TextNode - return _base.TEXT, node.value - - elif node.type == 5: # Element - attrs = {} - for name, value in node.attributes.items(): - if isinstance(name, tuple): - attrs[(name[2],name[1])] = value - else: - attrs[(None,name)] = value - return (_base.ELEMENT, node.namespace, node.name, - attrs, node.hasContent()) - - elif node.type == 6: # CommentNode - return _base.COMMENT, node.data - - else: - return _node.UNKNOWN, node.type - - def getFirstChild(self, node): - if isinstance(node, tuple): # It might be the root Node - parent, idx, parents = node - parents.append((parent, idx)) - node = parent.childNodes[idx] - else: - parents = [] - - assert node.hasContent(), "Node has no children" - return (node, 0, parents) - - def getNextSibling(self, node): - assert isinstance(node, tuple), "Node is not a tuple: " + str(node) - parent, idx, parents = node - idx += 1 - if len(parent.childNodes) > idx: - return (parent, idx, parents) - else: - return None - - def getParentNode(self, node): - assert isinstance(node, tuple) - parent, idx, parents = node - if parents: - parent, idx = parents.pop() - return parent, idx, parents - else: - # HACK: We could return ``parent`` but None will stop the algorithm the same way - return None diff --git a/libs/html5lib/treewalkers/soup.py b/libs/html5lib/treewalkers/soup.py deleted file mode 100644 index fca65ecb..00000000 --- a/libs/html5lib/treewalkers/soup.py +++ /dev/null @@ -1,60 +0,0 @@ -import re -import gettext -_ = gettext.gettext - -from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag -from html5lib.constants import namespaces -import _base - -class TreeWalker(_base.NonRecursiveTreeWalker): - doctype_regexp = re.compile( - r'DOCTYPE\s+(?P<name>[^\s]*)(\s*PUBLIC\s*"(?P<publicId>.*)"\s*"(?P<systemId1>.*)"|\s*SYSTEM\s*"(?P<systemId2>.*)")?') - def getNodeDetails(self, node): - if isinstance(node, BeautifulSoup): # Document or DocumentFragment - return (_base.DOCUMENT,) - - elif isinstance(node, Declaration): # DocumentType - string = unicode(node.string) - #Slice needed to remove markup added during unicode conversion, - #but only in some versions of BeautifulSoup/Python - if string.startswith('<!') and string.endswith('>'): - string = string[2:-1] - m = self.doctype_regexp.match(string) - #This regexp approach seems wrong and fragile - #but beautiful soup stores the doctype as a single thing and we want the seperate bits - #It should work as long as the tree is created by html5lib itself but may be wrong if it's - #been modified at all - #We could just feed to it a html5lib tokenizer, I guess... - assert m is not None, "DOCTYPE did not match expected format" - - name = m.group('name') - publicId = m.group('publicId') - if publicId is not None: - systemId = m.group('systemId1') - else: - systemId = m.group('systemId2') - return _base.DOCTYPE, name, publicId or "", systemId or "" - - elif isinstance(node, Comment): - string = unicode(node.string) - if string.startswith('<!--') and string.endswith('-->'): - string = string[4:-3] - return _base.COMMENT, string - - elif isinstance(node, unicode): # TextNode - return _base.TEXT, node - - elif isinstance(node, Tag): # Element - return (_base.ELEMENT, namespaces["html"], node.name, - dict(node.attrs).items(), node.contents) - else: - return _base.UNKNOWN, node.__class__.__name__ - - def getFirstChild(self, node): - return node.contents[0] - - def getNextSibling(self, node): - return node.nextSibling - - def getParentNode(self, node): - return node.parent diff --git a/libs/html5lib/trie/__init__.py b/libs/html5lib/trie/__init__.py new file mode 100644 index 00000000..a8cca8a9 --- /dev/null +++ b/libs/html5lib/trie/__init__.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import, division, unicode_literals + +from .py import Trie as PyTrie + +Trie = PyTrie + +try: + from .datrie import Trie as DATrie +except ImportError: + pass +else: + Trie = DATrie diff --git a/libs/html5lib/trie/_base.py b/libs/html5lib/trie/_base.py new file mode 100644 index 00000000..724486b1 --- /dev/null +++ b/libs/html5lib/trie/_base.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import, division, unicode_literals + +from collections import Mapping + + +class Trie(Mapping): + """Abstract base class for tries""" + + def keys(self, prefix=None): + keys = super().keys() + + if prefix is None: + return set(keys) + + # Python 2.6: no set comprehensions + return set([x for x in keys if x.startswith(prefix)]) + + def has_keys_with_prefix(self, prefix): + for key in self.keys(): + if key.startswith(prefix): + return True + + return False + + def longest_prefix(self, prefix): + if prefix in self: + return prefix + + for i in range(1, len(prefix) + 1): + if prefix[:-i] in self: + return prefix[:-i] + + raise KeyError(prefix) + + def longest_prefix_item(self, prefix): + lprefix = self.longest_prefix(prefix) + return (lprefix, self[lprefix]) diff --git a/libs/html5lib/trie/datrie.py b/libs/html5lib/trie/datrie.py new file mode 100644 index 00000000..51f3d046 --- /dev/null +++ b/libs/html5lib/trie/datrie.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import, division, unicode_literals + +from datrie import Trie as DATrie +from six import text_type + +from ._base import Trie as ABCTrie + + +class Trie(ABCTrie): + def __init__(self, data): + chars = set() + for key in data.keys(): + if not isinstance(key, text_type): + raise TypeError("All keys must be strings") + for char in key: + chars.add(char) + + self._data = DATrie("".join(chars)) + for key, value in data.items(): + self._data[key] = value + + def __contains__(self, key): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + raise NotImplementedError() + + def __getitem__(self, key): + return self._data[key] + + def keys(self, prefix=None): + return self._data.keys(prefix) + + def has_keys_with_prefix(self, prefix): + return self._data.has_keys_with_prefix(prefix) + + def longest_prefix(self, prefix): + return self._data.longest_prefix(prefix) + + def longest_prefix_item(self, prefix): + return self._data.longest_prefix_item(prefix) diff --git a/libs/html5lib/trie/py.py b/libs/html5lib/trie/py.py new file mode 100644 index 00000000..c2ba3da7 --- /dev/null +++ b/libs/html5lib/trie/py.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import, division, unicode_literals +from six import text_type + +from bisect import bisect_left + +from ._base import Trie as ABCTrie + + +class Trie(ABCTrie): + def __init__(self, data): + if not all(isinstance(x, text_type) for x in data.keys()): + raise TypeError("All keys must be strings") + + self._data = data + self._keys = sorted(data.keys()) + self._cachestr = "" + self._cachepoints = (0, len(data)) + + def __contains__(self, key): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + return iter(self._data) + + def __getitem__(self, key): + return self._data[key] + + def keys(self, prefix=None): + if prefix is None or prefix == "" or not self._keys: + return set(self._keys) + + if prefix.startswith(self._cachestr): + lo, hi = self._cachepoints + start = i = bisect_left(self._keys, prefix, lo, hi) + else: + start = i = bisect_left(self._keys, prefix) + + keys = set() + if start == len(self._keys): + return keys + + while self._keys[i].startswith(prefix): + keys.add(self._keys[i]) + i += 1 + + self._cachestr = prefix + self._cachepoints = (start, i) + + return keys + + def has_keys_with_prefix(self, prefix): + if prefix in self._data: + return True + + if prefix.startswith(self._cachestr): + lo, hi = self._cachepoints + i = bisect_left(self._keys, prefix, lo, hi) + else: + i = bisect_left(self._keys, prefix) + + if i == len(self._keys): + return False + + return self._keys[i].startswith(prefix) diff --git a/libs/html5lib/utils.py b/libs/html5lib/utils.py index d53f6788..2f41f4df 100644 --- a/libs/html5lib/utils.py +++ b/libs/html5lib/utils.py @@ -1,9 +1,16 @@ +from __future__ import absolute_import, division, unicode_literals + +from types import ModuleType + try: - frozenset -except NameError: - #Import from the sets module for python 2.3 - from sets import Set as set - from sets import ImmutableSet as frozenset + import xml.etree.cElementTree as default_etree +except ImportError: + import xml.etree.ElementTree as default_etree + + +__all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair", + "surrogatePairToCodepoint", "moduleFactoryFactory"] + class MethodDispatcher(dict): """Dict with 2 special properties: @@ -23,7 +30,7 @@ class MethodDispatcher(dict): # twice as fast. Please do careful performance testing before changing # anything here. _dictEntries = [] - for name,value in items: + for name, value in items: if type(name) in (list, tuple, frozenset, set): for item in name: _dictEntries.append((item, value)) @@ -35,141 +42,41 @@ class MethodDispatcher(dict): def __getitem__(self, key): return dict.get(self, key, self.default) -#Pure python implementation of deque taken from the ASPN Python Cookbook -#Original code by Raymond Hettinger -class deque(object): +# Some utility functions to dal with weirdness around UCS2 vs UCS4 +# python builds - def __init__(self, iterable=(), maxsize=-1): - if not hasattr(self, 'data'): - self.left = self.right = 0 - self.data = {} - self.maxsize = maxsize - self.extend(iterable) - - def append(self, x): - self.data[self.right] = x - self.right += 1 - if self.maxsize != -1 and len(self) > self.maxsize: - self.popleft() - - def appendleft(self, x): - self.left -= 1 - self.data[self.left] = x - if self.maxsize != -1 and len(self) > self.maxsize: - self.pop() - - def pop(self): - if self.left == self.right: - raise IndexError('cannot pop from empty deque') - self.right -= 1 - elem = self.data[self.right] - del self.data[self.right] - return elem - - def popleft(self): - if self.left == self.right: - raise IndexError('cannot pop from empty deque') - elem = self.data[self.left] - del self.data[self.left] - self.left += 1 - return elem - - def clear(self): - self.data.clear() - self.left = self.right = 0 - - def extend(self, iterable): - for elem in iterable: - self.append(elem) - - def extendleft(self, iterable): - for elem in iterable: - self.appendleft(elem) - - def rotate(self, n=1): - if self: - n %= len(self) - for i in xrange(n): - self.appendleft(self.pop()) - - def __getitem__(self, i): - if i < 0: - i += len(self) - try: - return self.data[i + self.left] - except KeyError: - raise IndexError - - def __setitem__(self, i, value): - if i < 0: - i += len(self) - try: - self.data[i + self.left] = value - except KeyError: - raise IndexError - - def __delitem__(self, i): - size = len(self) - if not (-size <= i < size): - raise IndexError - data = self.data - if i < 0: - i += size - for j in xrange(self.left+i, self.right-1): - data[j] = data[j+1] - self.pop() - - def __len__(self): - return self.right - self.left - - def __cmp__(self, other): - if type(self) != type(other): - return cmp(type(self), type(other)) - return cmp(list(self), list(other)) - - def __repr__(self, _track=[]): - if id(self) in _track: - return '...' - _track.append(id(self)) - r = 'deque(%r)' % (list(self),) - _track.remove(id(self)) - return r - - def __getstate__(self): - return (tuple(self),) - - def __setstate__(self, s): - self.__init__(s[0]) - - def __hash__(self): - raise TypeError - - def __copy__(self): - return self.__class__(self) - - def __deepcopy__(self, memo={}): - from copy import deepcopy - result = self.__class__() - memo[id(self)] = result - result.__init__(deepcopy(tuple(self), memo)) - return result - -#Some utility functions to dal with weirdness around UCS2 vs UCS4 -#python builds - -def encodingType(): - if len() == 2: - return "UCS2" - else: - return "UCS4" - -def isSurrogatePair(data): +def isSurrogatePair(data): return (len(data) == 2 and ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF) + def surrogatePairToCodepoint(data): - char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + + char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 + (ord(data[1]) - 0xDC00)) return char_val + +# Module Factory Factory (no, this isn't Java, I know) +# Here to stop this being duplicated all over the place. + + +def moduleFactoryFactory(factory): + moduleCache = {} + + def moduleFactory(baseModule, *args, **kwargs): + if isinstance(ModuleType.__name__, type("")): + name = "_%s_factory" % baseModule.__name__ + else: + name = b"_%s_factory" % baseModule.__name__ + + if name in moduleCache: + return moduleCache[name] + else: + mod = ModuleType(name) + objs = factory(baseModule, *args, **kwargs) + mod.__dict__.update(objs) + moduleCache[name] = mod + return mod + + return moduleFactory diff --git a/libs/httplib2/__init__.py b/libs/httplib2/__init__.py index 01151f7f..9780d4e5 100644 --- a/libs/httplib2/__init__.py +++ b/libs/httplib2/__init__.py @@ -3,7 +3,7 @@ from __future__ import generators httplib2 A caching http interface that supports ETags and gzip -to conserve bandwidth. +to conserve bandwidth. Requires Python 2.3 or later @@ -15,17 +15,17 @@ Changelog: __author__ = "Joe Gregorio (joe@bitworking.org)" __copyright__ = "Copyright 2006, Joe Gregorio" __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", - "James Antill", - "Xavier Verges Farrero", - "Jonathan Feinberg", - "Blair Zajac", - "Sam Ruby", - "Louis Nyffenegger"] + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] __license__ = "MIT" -__version__ = "$Rev$" +__version__ = "0.8" -import re -import sys +import re +import sys import email import email.Utils import email.Message @@ -35,6 +35,7 @@ import gzip import zlib import httplib import urlparse +import urllib import base64 import os import copy @@ -42,10 +43,10 @@ import calendar import time import random import errno -# remove depracated warning in python2.6 try: from hashlib import sha1 as _sha, md5 as _md5 except ImportError: + # prior to Python 2.5, these were separate modules import sha import md5 _sha = sha.new @@ -54,21 +55,38 @@ import hmac from gettext import gettext as _ import socket -# Try using local version, followed by system, and none if neither are found try: - import lib.socks as socks + from httplib2 import socks except ImportError: try: - import socks as socks - except ImportError: + import socks + except (ImportError, AttributeError): socks = None # Build the appropriate socket wrapper for ssl try: import ssl # python 2.6 - _ssl_wrap_socket = ssl.wrap_socket -except ImportError: - def _ssl_wrap_socket(sock, key_file, cert_file): + ssl_SSLError = ssl.SSLError + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if disable_validation: + cert_reqs = ssl.CERT_NONE + else: + cert_reqs = ssl.CERT_REQUIRED + # We should be specifying SSL version 3 or TLS v1, but the ssl module + # doesn't expose the necessary knobs. So we need to go with the default + # of SSLv23. + return ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, + cert_reqs=cert_reqs, ca_certs=ca_certs) +except (AttributeError, ImportError): + ssl_SSLError = None + def _ssl_wrap_socket(sock, key_file, cert_file, + disable_validation, ca_certs): + if not disable_validation: + raise CertificateValidationUnsupported( + "SSL certificate validation is not supported without " + "the ssl module installed. To avoid this error, install " + "the ssl module, or explicity disable validation.") ssl_sock = socket.ssl(sock, key_file, cert_file) return httplib.FakeSocket(sock, ssl_sock) @@ -84,15 +102,19 @@ def has_timeout(timeout): # python 2.6 return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) return (timeout is not None) -__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', - 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', - 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', - 'debuglevel'] +__all__ = [ + 'Http', 'Response', 'ProxyInfo', 'HttpLib2Error', 'RedirectMissingLocation', + 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', + 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel', 'ProxiesUnavailableError'] # The httplib debug level, set to a non-zero value to get debug output debuglevel = 0 +# A request will be tried 'RETRIES' times if it fails at the socket/connection level. +RETRIES = 2 # Python 2.3 support if sys.version_info < (2,4): @@ -113,8 +135,8 @@ if not hasattr(httplib.HTTPResponse, 'getheaders'): # All exceptions raised here derive from HttpLib2Error class HttpLib2Error(Exception): pass -# Some exceptions can be caught and optionally -# be turned back into responses. +# Some exceptions can be caught and optionally +# be turned back into responses. class HttpLib2ErrorWithResponse(HttpLib2Error): def __init__(self, desc, response, content): self.response = response @@ -127,8 +149,18 @@ class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class MalformedHeader(HttpLib2Error): pass class RelativeURIError(HttpLib2Error): pass class ServerNotFoundError(HttpLib2Error): pass +class ProxiesUnavailableError(HttpLib2Error): pass +class CertificateValidationUnsupported(HttpLib2Error): pass +class SSLHandshakeError(HttpLib2Error): pass +class NotSupportedOnThisPlatform(HttpLib2Error): pass +class CertificateHostnameMismatch(SSLHandshakeError): + def __init__(self, desc, host, cert): + HttpLib2Error.__init__(self, desc) + self.host = host + self.cert = cert # Open Items: # ----------- @@ -152,6 +184,16 @@ class ServerNotFoundError(HttpLib2Error): pass # requesting that URI again. DEFAULT_MAX_REDIRECTS = 5 +try: + # Users can optionally provide a module that tells us where the CA_CERTS + # are located. + import ca_certs_locater + CA_CERTS = ca_certs_locater.get() +except ImportError: + # Default CA certificates file bundled with httplib2. + CA_CERTS = os.path.join( + os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt") + # Which headers are hop-by-hop headers by default HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] @@ -176,7 +218,7 @@ def urlnorm(uri): raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) authority = authority.lower() scheme = scheme.lower() - if not path: + if not path: path = "/" # Could do syntax based normalization of the URI before # computing the digest. See Section 6.2.2 of Std 66. @@ -228,7 +270,7 @@ def _parse_cache_control(headers): parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] retval = dict(parts_with_args + parts_wo_args) - return retval + return retval # Whether to use a strict mode to parse WWW-Authenticate headers # Might lead to bad results in case of ill-formed header value, @@ -249,25 +291,30 @@ def _parse_www_authenticate(headers, headername='www-authenticate'): per auth_scheme.""" retval = {} if headers.has_key(headername): - authenticate = headers[headername].strip() - www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED - while authenticate: - # Break off the scheme at the beginning of the line - if headername == 'authentication-info': - (auth_scheme, the_rest) = ('digest', authenticate) - else: - (auth_scheme, the_rest) = authenticate.split(" ", 1) - # Now loop over all the key value pairs that come after the scheme, - # being careful not to roll into the next scheme - match = www_auth.search(the_rest) - auth_params = {} - while match: - if match and len(match.groups()) == 3: - (key, value, the_rest) = match.groups() - auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) + try: + + authenticate = headers[headername].strip() + www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED + while authenticate: + # Break off the scheme at the beginning of the line + if headername == 'authentication-info': + (auth_scheme, the_rest) = ('digest', authenticate) + else: + (auth_scheme, the_rest) = authenticate.split(" ", 1) + # Now loop over all the key value pairs that come after the scheme, + # being careful not to roll into the next scheme match = www_auth.search(the_rest) - retval[auth_scheme.lower()] = auth_params - authenticate = the_rest.strip() + auth_params = {} + while match: + if match and len(match.groups()) == 3: + (key, value, the_rest) = match.groups() + auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) + match = www_auth.search(the_rest) + retval[auth_scheme.lower()] = auth_params + authenticate = the_rest.strip() + + except ValueError: + raise MalformedHeader("WWW-Authenticate") return retval @@ -279,17 +326,17 @@ def _entry_disposition(response_headers, request_headers): 1. Cache-Control: max-stale 2. Age: headers are not used in the calculations. - Not that this algorithm is simpler than you might think + Not that this algorithm is simpler than you might think because we are operating as a private (non-shared) cache. This lets us ignore 's-maxage'. We can also ignore 'proxy-invalidate' since we aren't a proxy. - We will never return a stale document as - fresh as a design decision, and thus the non-implementation - of 'max-stale'. This also lets us safely ignore 'must-revalidate' + We will never return a stale document as + fresh as a design decision, and thus the non-implementation + of 'max-stale'. This also lets us safely ignore 'must-revalidate' since we operate as if every server has sent 'must-revalidate'. Since we are private we get to ignore both 'public' and 'private' parameters. We also ignore 'no-transform' since - we don't do any transformations. + we don't do any transformations. The 'no-store' parameter is handled at a higher level. So the only Cache-Control parameters we look at are: @@ -298,7 +345,7 @@ def _entry_disposition(response_headers, request_headers): max-age min-fresh """ - + retval = "STALE" cc = _parse_cache_control(request_headers) cc_response = _parse_cache_control(response_headers) @@ -340,10 +387,10 @@ def _entry_disposition(response_headers, request_headers): min_fresh = int(cc['min-fresh']) except ValueError: min_fresh = 0 - current_age += min_fresh + current_age += min_fresh if freshness_lifetime > current_age: retval = "FRESH" - return retval + return retval def _decompressContent(response, new_content): content = new_content @@ -391,7 +438,7 @@ def _updateCache(request_headers, response_headers, content, cache, cachekey): if status == 304: status = 200 - status_header = 'status: %d\r\n' % response_headers.status + status_header = 'status: %d\r\n' % status header_str = info.as_string() @@ -408,10 +455,10 @@ def _wsse_username_token(cnonce, iso_now, password): return base64.b64encode(_sha("%s%s%s" % (cnonce, iso_now, password)).digest()).strip() -# For credentials we need two things, first +# For credentials we need two things, first # a pool of credential to try (not necesarily tied to BAsic, Digest, etc.) # Then we also need a list of URIs that have already demanded authentication -# That list is tricky since sub-URIs can take the same auth, or the +# That list is tricky since sub-URIs can take the same auth, or the # auth scheme may change as you descend the tree. # So we also need each Auth instance to be able to tell us # how close to the 'top' it is. @@ -435,7 +482,7 @@ class Authentication(object): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate - Authorization header. Over-rise this in sub-classes.""" + Authorization header. Over-ride this in sub-classes.""" pass def response(self, response, content): @@ -443,7 +490,7 @@ class Authentication(object): or such returned from the last authorized response. Over-rise this in sub-classes if necessary. - Return TRUE is the request is to be retried, for + Return TRUE is the request is to be retried, for example Digest may return stale=true. """ return False @@ -461,7 +508,7 @@ class BasicAuthentication(Authentication): class DigestAuthentication(Authentication): - """Only do qop='auth' and MD5, since that + """Only do qop='auth' and MD5, since that is all Apache currently implements""" def __init__(self, credentials, host, request_uri, headers, response, content, http): Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) @@ -474,7 +521,7 @@ class DigestAuthentication(Authentication): self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper() if self.challenge['algorithm'] != 'MD5': raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) - self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) + self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) self.challenge['nc'] = 1 def request(self, method, request_uri, headers, content, cnonce = None): @@ -482,23 +529,24 @@ class DigestAuthentication(Authentication): H = lambda x: _md5(x).hexdigest() KD = lambda s, d: H("%s:%s" % (s, d)) A2 = "".join([method, ":", request_uri]) - self.challenge['cnonce'] = cnonce or _cnonce() - request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % (self.challenge['nonce'], - '%08x' % self.challenge['nc'], - self.challenge['cnonce'], - self.challenge['qop'], H(A2) - )) - headers['Authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( - self.credentials[0], + self.challenge['cnonce'] = cnonce or _cnonce() + request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % ( + self.challenge['nonce'], + '%08x' % self.challenge['nc'], + self.challenge['cnonce'], + self.challenge['qop'], H(A2))) + headers['authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( + self.credentials[0], self.challenge['realm'], self.challenge['nonce'], - request_uri, + request_uri, self.challenge['algorithm'], request_digest, self.challenge['qop'], self.challenge['nc'], - self.challenge['cnonce'], - ) + self.challenge['cnonce']) + if self.challenge.get('opaque'): + headers['authorization'] += ', opaque="%s"' % self.challenge['opaque'] self.challenge['nc'] += 1 def response(self, response, content): @@ -506,14 +554,14 @@ class DigestAuthentication(Authentication): challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {}) if 'true' == challenge.get('stale'): self.challenge['nonce'] = challenge['nonce'] - self.challenge['nc'] = 1 + self.challenge['nc'] = 1 return True else: updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {}) if updated_challenge.has_key('nextnonce'): self.challenge['nonce'] = updated_challenge['nextnonce'] - self.challenge['nc'] = 1 + self.challenge['nc'] = 1 return False @@ -547,9 +595,8 @@ class HmacDigestAuthentication(Authentication): else: self.pwhashmod = _sha self.key = "".join([self.credentials[0], ":", - self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), - ":", self.challenge['realm'] - ]) + self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), + ":", self.challenge['realm']]) self.key = self.pwhashmod.new(self.key).hexdigest().lower() def request(self, method, request_uri, headers, content): @@ -561,16 +608,15 @@ class HmacDigestAuthentication(Authentication): cnonce = _cnonce() request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val) request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() - headers['Authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( - self.credentials[0], + headers['authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( + self.credentials[0], self.challenge['realm'], self.challenge['snonce'], cnonce, - request_uri, + request_uri, created, request_digest, - keylist, - ) + keylist) def response(self, response, content): challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {}) @@ -583,7 +629,7 @@ class WsseAuthentication(Authentication): """This is thinly tested and should not be relied upon. At this time there isn't any third party server to test against. Blogger and TypePad implemented this algorithm at one point - but Blogger has since switched to Basic over HTTPS and + but Blogger has since switched to Basic over HTTPS and TypePad has implemented it wrong, by never issuing a 401 challenge but instead requiring your client to telepathically know that their endpoint is expecting WSSE profile="UsernameToken".""" @@ -593,7 +639,7 @@ class WsseAuthentication(Authentication): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers['Authorization'] = 'WSSE profile="UsernameToken"' + headers['authorization'] = 'WSSE profile="UsernameToken"' iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) cnonce = _cnonce() password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) @@ -629,7 +675,7 @@ class GoogleLoginAuthentication(Authentication): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate Authorization header.""" - headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth AUTH_SCHEME_CLASSES = { @@ -644,13 +690,13 @@ AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] class FileCache(object): """Uses a local directory as a store for cached files. - Not really safe to use if multiple threads or processes are going to + Not really safe to use if multiple threads or processes are going to be running on the same cache. """ def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior self.cache = cache self.safe = safe - if not os.path.exists(cache): + if not os.path.exists(cache): os.makedirs(self.cache) def get(self, key): @@ -660,7 +706,7 @@ class FileCache(object): f = file(cacheFullPath, "rb") retval = f.read() f.close() - except IOError, e: + except IOError: pass return retval @@ -688,34 +734,127 @@ class Credentials(object): def iter(self, domain): for (cdomain, name, password) in self.credentials: if cdomain == "" or domain == cdomain: - yield (name, password) + yield (name, password) class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" pass +class AllHosts(object): + pass class ProxyInfo(object): - """Collect information required to use a proxy.""" - def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): - """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX - constants. For example: + """Collect information required to use a proxy.""" + bypass_hosts = () -p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) - """ - self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass + def __init__(self, proxy_type, proxy_host, proxy_port, + proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: - def astuple(self): - return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, - self.proxy_user, self.proxy_pass) + p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, + proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type = proxy_type + self.proxy_host = proxy_host + self.proxy_port = proxy_port + self.proxy_rdns = proxy_rdns + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass - def isgood(self): - return socks and (self.proxy_host != None) and (self.proxy_port != None) + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, + self.proxy_rdns, self.proxy_user, self.proxy_pass) + + def isgood(self): + return (self.proxy_host != None) and (self.proxy_port != None) + + def applies_to(self, hostname): + return not self.bypass_host(hostname) + + def bypass_host(self, hostname): + """Has this host been excluded from the proxy config""" + if self.bypass_hosts is AllHosts: + return True + + bypass = False + for domain in self.bypass_hosts: + if hostname.endswith(domain): + bypass = True + + return bypass + + +def proxy_info_from_environment(method='http'): + """ + Read proxy info from the environment variables. + """ + if method not in ['http', 'https']: + return + + env_var = method + '_proxy' + url = os.environ.get(env_var, os.environ.get(env_var.upper())) + if not url: + return + pi = proxy_info_from_url(url, method) + + no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) + bypass_hosts = [] + if no_proxy: + bypass_hosts = no_proxy.split(',') + # special case, no_proxy=* means all hosts bypassed + if no_proxy == '*': + bypass_hosts = AllHosts + + pi.bypass_hosts = bypass_hosts + return pi + +def proxy_info_from_url(url, method='http'): + """ + Construct a ProxyInfo from a URL (such as http_proxy env var) + """ + url = urlparse.urlparse(url) + username = None + password = None + port = None + if '@' in url[1]: + ident, host_port = url[1].split('@', 1) + if ':' in ident: + username, password = ident.split(':', 1) + else: + password = ident + else: + host_port = url[1] + if ':' in host_port: + host, port = host_port.split(':', 1) + else: + host = host_port + + if port: + port = int(port) + else: + port = dict(https=443, http=80)[method] + + proxy_type = 3 # socks.PROXY_TYPE_HTTP + return ProxyInfo( + proxy_type = proxy_type, + proxy_host = host, + proxy_port = port, + proxy_user = username or None, + proxy_pass = password or None, + ) class HTTPConnectionWithTimeout(httplib.HTTPConnection): - """HTTPConnection subclass that supports timeouts""" + """ + HTTPConnection subclass that supports timeouts + + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): httplib.HTTPConnection.__init__(self, host, port, strict) @@ -725,27 +864,46 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): def connect(self): """Connect to the host and port specified in __init__.""" # Mostly verbatim from httplib.py. + if self.proxy_info and socks is None: + raise ProxiesUnavailableError( + 'Proxy support missing but proxy use was requested!') msg = "getaddrinfo returns an empty list" - for res in socket.getaddrinfo(self.host, self.port, 0, - socket.SOCK_STREAM): + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: - if self.proxy_info and self.proxy_info.isgood(): + if use_proxy: self.sock = socks.socksocket(af, socktype, proto) - self.sock.setproxy(*self.proxy_info.astuple()) + self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) else: self.sock = socket.socket(af, socktype, proto) + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Different from httplib: support timeouts. if has_timeout(self.timeout): self.sock.settimeout(self.timeout) # End of difference from httplib. if self.debuglevel > 0: - print "connect: (%s, %s)" % (self.host, self.port) + print "connect: (%s, %s) ************" % (self.host, self.port) + if use_proxy: + print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) - self.sock.connect(sa) + self.sock.connect((self.host, self.port) + sa[2:]) except socket.error, msg: if self.debuglevel > 0: - print 'connect fail:', (self.host, self.port) + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) if self.sock: self.sock.close() self.sock = None @@ -755,56 +913,265 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): raise socket.error, msg class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): - "This class allows communication via SSL." + """ + This class allows communication via SSL. + All timeouts are in seconds. If None is passed for timeout then + Python's default timeout for sockets will be used. See for example + the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + """ def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=None, proxy_info=None): - httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, - cert_file=cert_file, strict=strict) + strict=None, timeout=None, proxy_info=None, + ca_certs=None, disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, + key_file=key_file, + cert_file=cert_file, strict=strict) self.timeout = timeout self.proxy_info = proxy_info + if ca_certs is None: + ca_certs = CA_CERTS + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + + # The following two methods were adapted from https_wrapper.py, released + # with the Google Appengine SDK at + # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py + # under the following license: + # + # Copyright 2007 Google Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # + + def _GetValidHostsForCert(self, cert): + """Returns a list of valid host globs for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + Returns: + list: A list of valid host globs. + """ + if 'subjectAltName' in cert: + return [x[1] for x in cert['subjectAltName'] + if x[0].lower() == 'dns'] + else: + return [x[0][1] for x in cert['subject'] + if x[0][0].lower() == 'commonname'] + + def _ValidateCertificateHostname(self, cert, hostname): + """Validates that a given hostname is valid for an SSL certificate. + + Args: + cert: A dictionary representing an SSL certificate. + hostname: The hostname to test. + Returns: + bool: Whether or not the hostname is valid for this certificate. + """ + hosts = self._GetValidHostsForCert(cert) + for host in hosts: + host_re = host.replace('.', '\.').replace('*', '[^.]*') + if re.search('^%s$' % (host_re,), hostname, re.I): + return True + return False def connect(self): "Connect to a host on a given (SSL) port." + msg = "getaddrinfo returns an empty list" if self.proxy_info and self.proxy_info.isgood(): - sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) - sock.setproxy(*self.proxy_info.astuple()) + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() else: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - if has_timeout(self.timeout): - sock.settimeout(self.timeout) - sock.connect((self.host, self.port)) - self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + address_info = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) + for family, socktype, proto, canonname, sockaddr in address_info: + try: + if use_proxy: + sock = socks.socksocket(family, socktype, proto) + + sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) + else: + sock = socket.socket(family, socktype, proto) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket( + sock, self.key_file, self.cert_file, + self.disable_ssl_certificate_validation, self.ca_certs) + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if not self.disable_ssl_certificate_validation: + cert = self.sock.getpeercert() + hostname = self.host.split(':', 0)[0] + if not self._ValidateCertificateHostname(cert, hostname): + raise CertificateHostnameMismatch( + 'Server presented certificate that does not match ' + 'host %s: %s' % (hostname, cert), hostname, cert) + except ssl_SSLError, e: + if sock: + sock.close() + if self.sock: + self.sock.close() + self.sock = None + # Unfortunately the ssl module doesn't seem to provide any way + # to get at more detailed error information, in particular + # whether the error is due to certificate validation or + # something else (such as SSL protocol mismatch). + if e.errno == ssl.SSL_ERROR_SSL: + raise SSLHandshakeError(e) + else: + raise + except (socket.timeout, socket.gaierror): + raise + except socket.error, msg: + if self.debuglevel > 0: + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +SCHEME_TO_CONNECTION = { + 'http': HTTPConnectionWithTimeout, + 'https': HTTPSConnectionWithTimeout +} + +# Use a different connection object for Google App Engine +try: + try: + from google.appengine.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google.appengine.api.urlfetch import fetch + from google.appengine.api.urlfetch import InvalidURLError + except (ImportError, AttributeError): + from google3.apphosting.api import apiproxy_stub_map + if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: + raise ImportError # Bail out; we're not actually running on App Engine. + from google3.apphosting.api.urlfetch import fetch + from google3.apphosting.api.urlfetch import InvalidURLError + + def _new_fixed_fetch(validate_certificate): + def fixed_fetch(url, payload=None, method="GET", headers={}, + allow_truncated=False, follow_redirects=True, + deadline=5): + return fetch(url, payload=payload, method=method, headers=headers, + allow_truncated=allow_truncated, + follow_redirects=follow_redirects, deadline=deadline, + validate_certificate=validate_certificate) + return fixed_fetch + + class AppEngineHttpConnection(httplib.HTTPConnection): + """Use httplib on App Engine, but compensate for its weirdness. + + The parameters key_file, cert_file, proxy_info, ca_certs, and + disable_ssl_certificate_validation are all dropped on the ground. + """ + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + httplib.HTTPConnection.__init__(self, host, port=port, + strict=strict, timeout=timeout) + + class AppEngineHttpsConnection(httplib.HTTPSConnection): + """Same as AppEngineHttpConnection, but for HTTPS URIs.""" + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): + httplib.HTTPSConnection.__init__(self, host, port=port, + key_file=key_file, + cert_file=cert_file, strict=strict, + timeout=timeout) + self._fetch = _new_fixed_fetch( + not disable_ssl_certificate_validation) + + # Update the connection classes to use the Googel App Engine specific ones. + SCHEME_TO_CONNECTION = { + 'http': AppEngineHttpConnection, + 'https': AppEngineHttpsConnection + } +except (ImportError, AttributeError): + pass class Http(object): """An HTTP client that handles: -- all methods -- caching -- ETags -- compression, -- HTTPS -- Basic -- Digest -- WSSE -and more. + - all methods + - caching + - ETags + - compression, + - HTTPS + - Basic + - Digest + - WSSE + + and more. """ - def __init__(self, cache=None, timeout=None, proxy_info=None): - """The value of proxy_info is a ProxyInfo instance. + def __init__(self, cache=None, timeout=None, + proxy_info=proxy_info_from_environment, + ca_certs=None, disable_ssl_certificate_validation=False): + """If 'cache' is a string then it is used as a directory name for + a disk cache. Otherwise it must be an object that supports the + same interface as FileCache. -If 'cache' is a string then it is used as a directory name -for a disk cache. Otherwise it must be an object that supports -the same interface as FileCache.""" + All timeouts are in seconds. If None is passed for timeout + then Python's default timeout for sockets will be used. See + for example the docs of socket.setdefaulttimeout(): + http://docs.python.org/library/socket.html#socket.setdefaulttimeout + + `proxy_info` may be: + - a callable that takes the http scheme ('http' or 'https') and + returns a ProxyInfo instance per request. By default, uses + proxy_nfo_from_environment. + - a ProxyInfo instance (static proxy config). + - None (proxy disabled). + + ca_certs is the path of a file containing root CA certificates for SSL + server certificate validation. By default, a CA cert file bundled with + httplib2 is used. + + If disable_ssl_certificate_validation is true, SSL cert validation will + not be performed. + """ self.proxy_info = proxy_info + self.ca_certs = ca_certs + self.disable_ssl_certificate_validation = \ + disable_ssl_certificate_validation + # Map domain name to an httplib connection self.connections = {} # The location of the cache, for now a directory # where cached responses are held. - if cache and isinstance(cache, str): + if cache and isinstance(cache, basestring): self.cache = FileCache(cache) else: self.cache = cache @@ -820,10 +1187,10 @@ the same interface as FileCache.""" # If set to False then no redirects are followed, even safe ones. self.follow_redirects = True - + # Which HTTP methods do we apply optimistic concurrency to, i.e. # which methods get an "if-match:" etag header added to them. - self.optimistic_concurrency_methods = ["PUT"] + self.optimistic_concurrency_methods = ["PUT", "PATCH"] # If 'follow_redirects' is True, and this is set to True then # all redirecs are followed, including unsafe ones. @@ -831,10 +1198,27 @@ the same interface as FileCache.""" self.ignore_etag = False - self.force_exception_to_status_code = False + self.force_exception_to_status_code = False self.timeout = timeout + # Keep Authorization: headers on a redirect. + self.forward_authorization_headers = False + + def __getstate__(self): + state_dict = copy.copy(self.__dict__) + # In case request is augmented by some foreign object such as + # credentials which handle auth + if 'request' in state_dict: + del state_dict['request'] + if 'connections' in state_dict: + del state_dict['connections'] + return state_dict + + def __setstate__(self, state): + self.__dict__.update(state) + self.connections = {} + def _auth_from_challenge(self, host, request_uri, headers, response, content): """A generator that creates Authorization objects that can be applied to requests. @@ -862,37 +1246,56 @@ the same interface as FileCache.""" self.authorizations = [] def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(2): + for i in range(RETRIES): try: + if hasattr(conn, 'sock') and conn.sock is None: + conn.connect() conn.request(method, request_uri, body, headers) + except socket.timeout: + raise except socket.gaierror: conn.close() raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except ssl_SSLError: + conn.close() + raise except socket.error, e: - if not hasattr(e, 'errno'): # I don't know what this is so lets raise it if it happens + err = 0 + if hasattr(e, 'args'): + err = getattr(e, 'args')[0] + else: + err = e.errno + if err == errno.ECONNREFUSED: # Connection refused raise - elif e.errno == errno.ECONNREFUSED: # Connection refused - raise - # Just because the server closed the connection doesn't apparently mean - # that the server didn't send a response. - pass except httplib.HTTPException: # Just because the server closed the connection doesn't apparently mean # that the server didn't send a response. - pass + if hasattr(conn, 'sock') and conn.sock is None: + if i < RETRIES-1: + conn.close() + conn.connect() + continue + else: + conn.close() + raise + if i < RETRIES-1: + conn.close() + conn.connect() + continue try: response = conn.getresponse() except (socket.error, httplib.HTTPException): - if i == 0: + if i < RETRIES-1: conn.close() conn.connect() continue else: + conn.close() raise else: content = "" if method == "HEAD": - response.close() + conn.close() else: content = response.read() response = Response(response) @@ -908,12 +1311,12 @@ the same interface as FileCache.""" auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] auth = auths and sorted(auths)[0][1] or None - if auth: + if auth: auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) - if auth: + if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers ) @@ -921,7 +1324,7 @@ the same interface as FileCache.""" if response.status == 401: for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): - authorization.request(method, request_uri, headers, body) + authorization.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) if response.status != 401: self.authorizations.append(authorization) @@ -944,26 +1347,31 @@ the same interface as FileCache.""" if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri _updateCache(headers, response, content, self.cache, cachekey) if headers.has_key('if-none-match'): del headers['if-none-match'] if headers.has_key('if-modified-since'): del headers['if-modified-since'] + if 'authorization' in headers and not self.forward_authorization_headers: + del headers['authorization'] if response.has_key('location'): location = response['location'] old_response = copy.deepcopy(response) if not old_response.has_key('content-location'): - old_response['content-location'] = absolute_uri - redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method + old_response['content-location'] = absolute_uri + redirect_method = method + if response.status in [302, 303]: + redirect_method = "GET" + body = None (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) response.previous = old_response else: - raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) - elif response.status in [200, 203] and method == "GET": + raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content) + elif response.status in [200, 203] and method in ["GET", "HEAD"]: # Don't cache 206's since we aren't going to handle byte range requests if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri _updateCache(headers, response, content, self.cache, cachekey) return (response, content) @@ -978,24 +1386,25 @@ the same interface as FileCache.""" def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): """ Performs a single HTTP request. -The 'uri' is the URI of the HTTP resource and can begin -with either 'http' or 'https'. The value of 'uri' must be an absolute URI. -The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. -There is no restriction on the methods allowed. + The 'uri' is the URI of the HTTP resource and can begin with either + 'http' or 'https'. The value of 'uri' must be an absolute URI. -The 'body' is the entity body to be sent with the request. It is a string -object. + The 'method' is the HTTP method to perform, such as GET, POST, DELETE, + etc. There is no restriction on the methods allowed. -Any extra headers that are to be sent with the request should be provided in the -'headers' dictionary. + The 'body' is the entity body to be sent with the request. It is a + string object. -The maximum number of redirect to follow before raising an -exception is 'redirections. The default is 5. + Any extra headers that are to be sent with the request should be + provided in the 'headers' dictionary. -The return value is a tuple of (response, content), the first -being and instance of the 'Response' class, the second being -a string that contains the response entity body. + The maximum number of redirect to follow before raising an + exception is 'redirections. The default is 5. + + The return value is a tuple of (response, content), the first + being and instance of the 'Response' class, the second being + a string that contains the response entity body. """ try: if headers is None: @@ -1004,7 +1413,7 @@ a string that contains the response entity body. headers = self._normalize_headers(headers) if not headers.has_key('user-agent'): - headers['user-agent'] = "Python-httplib2/%s" % __version__ + headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__ uri = iri2uri(uri) @@ -1014,21 +1423,38 @@ a string that contains the response entity body. scheme = 'https' authority = domain_port[0] + proxy_info = self._get_proxy_info(scheme, authority) + conn_key = scheme+":"+authority if conn_key in self.connections: conn = self.connections[conn_key] else: if not connection_type: - connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout + connection_type = SCHEME_TO_CONNECTION[scheme] certs = list(self.certificates.iter(authority)) - if scheme == 'https' and certs: - conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], - cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) + if scheme == 'https': + if certs: + conn = self.connections[conn_key] = connection_type( + authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) + else: + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info, + ca_certs=self.ca_certs, + disable_ssl_certificate_validation= + self.disable_ssl_certificate_validation) else: - conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) + conn = self.connections[conn_key] = connection_type( + authority, timeout=self.timeout, + proxy_info=proxy_info) conn.set_debuglevel(debuglevel) - if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: + if 'range' not in headers and 'accept-encoding' not in headers: headers['accept-encoding'] = 'gzip, deflate' info = email.Message.Message() @@ -1048,7 +1474,7 @@ a string that contains the response entity body. feedparser.feed(info) info = feedparser.close() feedparser._parse = None - except IndexError, ValueError: + except (IndexError, ValueError): self.cache.delete(cachekey) cachekey = None cached_value = None @@ -1071,13 +1497,15 @@ a string that contains the response entity body. for header in vary_headers: key = '-varied-%s' % header value = info[key] - if headers.get(header, '') != value: - cached_value = None - break + if headers.get(header, None) != value: + cached_value = None + break if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: if info.has_key('-x-permanent-redirect-url'): # Should cached permanent redirects be counted in our redirection count? For now, yes. + if redirections <= 0: + raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "") (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) response.previous = Response(info) response.previous.fromcache = True @@ -1085,13 +1513,13 @@ a string that contains the response entity body. # Determine our course of action: # Is the cached entry fresh or stale? # Has the client requested a non-cached response? - # - # There seems to be three possible answers: + # + # There seems to be three possible answers: # 1. [FRESH] Return the cache entry w/o doing a GET # 2. [STALE] Do the GET (but add in cache validators if available) # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request - entry_disposition = _entry_disposition(info, headers) - + entry_disposition = _entry_disposition(info, headers) + if entry_disposition == "FRESH": if not cached_value: info['status'] = '504' @@ -1113,7 +1541,7 @@ a string that contains the response entity body. if response.status == 304 and method == "GET": # Rewrite the cache entry with the new end-to-end headers - # Take all headers that are in response + # Take all headers that are in response # and overwrite their values in info. # unless they are hop-by-hop, or are listed in the connection header. @@ -1125,14 +1553,14 @@ a string that contains the response entity body. _updateCache(headers, merged_response, content, self.cache, cachekey) response = merged_response response.status = 200 - response.fromcache = True + response.fromcache = True elif response.status == 200: content = new_content else: self.cache.delete(cachekey) - content = new_content - else: + content = new_content + else: cc = _parse_cache_control(headers) if cc.has_key('only-if-cached'): info['status'] = '504' @@ -1146,34 +1574,47 @@ a string that contains the response entity body. response = e.response content = e.content response.status = 500 - response.reason = str(e) - elif isinstance(e, socket.timeout) or (isinstance(e, socket.error) and 'timed out' in str(e)): + response.reason = str(e) + elif isinstance(e, socket.timeout): content = "Request Timeout" - response = Response( { - "content-type": "text/plain", - "status": "408", - "content-length": len(content) - }) + response = Response({ + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) response.reason = "Request Timeout" else: - content = str(e) - response = Response( { - "content-type": "text/plain", - "status": "400", - "content-length": len(content) - }) - response.reason = "Bad Request" + content = str(e) + response = Response({ + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" else: raise - + return (response, content) - + def _get_proxy_info(self, scheme, authority): + """Return a ProxyInfo instance (or None) based on the scheme + and authority. + """ + hostname, port = urllib.splitport(authority) + proxy_info = self.proxy_info + if callable(proxy_info): + proxy_info = proxy_info(scheme) + + if (hasattr(proxy_info, 'applies_to') + and not proxy_info.applies_to(hostname)): + proxy_info = None + return proxy_info + class Response(dict): """An object more like email.Message than httplib.HTTPResponse.""" - + """Is this response from our local cache""" fromcache = False @@ -1189,27 +1630,28 @@ class Response(dict): previous = None def __init__(self, info): - # info is either an email.Message or + # info is either an email.Message or # an httplib.HTTPResponse object. if isinstance(info, httplib.HTTPResponse): - for key, value in info.getheaders(): - self[key.lower()] = value + for key, value in info.getheaders(): + self[key.lower()] = value self.status = info.status self['status'] = str(self.status) self.reason = info.reason self.version = info.version elif isinstance(info, email.Message.Message): - for key, value in info.items(): - self[key] = value + for key, value in info.items(): + self[key.lower()] = value self.status = int(self['status']) else: - for key, value in info.iteritems(): - self[key] = value + for key, value in info.iteritems(): + self[key.lower()] = value self.status = int(self.get('status', self.status)) + self.reason = self.get('reason', self.reason) def __getattr__(self, name): if name == 'dict': - return self - else: - raise AttributeError, name + return self + else: + raise AttributeError, name diff --git a/libs/httplib2/cacerts.txt b/libs/httplib2/cacerts.txt new file mode 100644 index 00000000..d8a0027c --- /dev/null +++ b/libs/httplib2/cacerts.txt @@ -0,0 +1,739 @@ +# Certifcate Authority certificates for validating SSL connections. +# +# This file contains PEM format certificates generated from +# http://mxr.mozilla.org/seamonkey/source/security/nss/lib/ckfw/builtins/certdata.txt +# +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Netscape security libraries. +# +# The Initial Developer of the Original Code is +# Netscape Communications Corporation. +# Portions created by the Initial Developer are Copyright (C) 1994-2000 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +Verisign/RSA Secure Server CA +============================= + +-----BEGIN CERTIFICATE----- +MIICNDCCAaECEAKtZn5ORf5eV288mBle3cAwDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxIDAeBgNVBAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYD +VQQLEyVTZWN1cmUgU2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk0 +MTEwOTAwMDAwMFoXDTEwMDEwNzIzNTk1OVowXzELMAkGA1UEBhMCVVMxIDAeBgNV +BAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYDVQQLEyVTZWN1cmUgU2Vy +dmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGbMA0GCSqGSIb3DQEBAQUAA4GJ +ADCBhQJ+AJLOesGugz5aqomDV6wlAXYMra6OLDfO6zV4ZFQD5YRAUcm/jwjiioII +0haGN1XpsSECrXZogZoFokvJSyVmIlZsiAeP94FZbYQHZXATcXY+m3dM41CJVphI +uR2nKRoTLkoRWZweFdVJVCxzOmmCsZc5nG1wZ0jl3S3WyB57AgMBAAEwDQYJKoZI +hvcNAQECBQADfgBl3X7hsuyw4jrg7HFGmhkRuNPHoLQDQCYCPgmc4RKz0Vr2N6W3 +YQO2WxZpO8ZECAyIUwxrl0nHPjXcbLm7qt9cuzovk2C2qUtN8iD3zV9/ZHuO3ABc +1/p3yjkWWW8O6tO1g39NTUJWdrTJXwT4OPjr0l91X817/OWOgHz8UA== +-----END CERTIFICATE----- + +Thawte Personal Basic CA +======================== + +-----BEGIN CERTIFICATE----- +MIIDITCCAoqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFBlcnNvbmFsIEJhc2lj +IENBMSgwJgYJKoZIhvcNAQkBFhlwZXJzb25hbC1iYXNpY0B0aGF3dGUuY29tMB4X +DTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgcsxCzAJBgNVBAYTAlpBMRUw +EwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEaMBgGA1UE +ChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRpb24gU2Vy +dmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQZXJzb25hbCBCYXNpYyBD +QTEoMCYGCSqGSIb3DQEJARYZcGVyc29uYWwtYmFzaWNAdGhhd3RlLmNvbTCBnzAN +BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAvLyTU23AUE+CFeZIlDWmWr5vQvoPR+53 +dXLdjUmbllegeNTKP1GzaQuRdhciB5dqxFGTS+CN7zeVoQxN2jSQHReJl+A1OFdK +wPQIcOk8RHtQfmGakOMj04gRRif1CwcOu93RfyAKiLlWCy4cgNrx454p7xS9CkT7 +G1sY0b8jkyECAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQQF +AAOBgQAt4plrsD16iddZopQBHyvdEktTwq1/qqcAXJFAVyVKOKqEcLnZgA+le1z7 +c8a914phXAPjLSeoF+CEhULcXpvGt7Jtu3Sv5D/Lp7ew4F2+eIMllNLbgQ95B21P +9DkVWlIBe94y1k049hJcBlDfBVu9FEuh3ym6O0GN92NWod8isQ== +-----END CERTIFICATE----- + +Thawte Personal Premium CA +========================== + +-----BEGIN CERTIFICATE----- +MIIDKTCCApKgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBzzELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEjMCEGA1UEAxMaVGhhd3RlIFBlcnNvbmFsIFByZW1p +dW0gQ0ExKjAoBgkqhkiG9w0BCQEWG3BlcnNvbmFsLXByZW1pdW1AdGhhd3RlLmNv +bTAeFw05NjAxMDEwMDAwMDBaFw0yMDEyMzEyMzU5NTlaMIHPMQswCQYDVQQGEwJa +QTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xGjAY +BgNVBAoTEVRoYXd0ZSBDb25zdWx0aW5nMSgwJgYDVQQLEx9DZXJ0aWZpY2F0aW9u +IFNlcnZpY2VzIERpdmlzaW9uMSMwIQYDVQQDExpUaGF3dGUgUGVyc29uYWwgUHJl +bWl1bSBDQTEqMCgGCSqGSIb3DQEJARYbcGVyc29uYWwtcHJlbWl1bUB0aGF3dGUu +Y29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJZtn4B0TPuYwu8KHvE0Vs +Bd/eJxZRNkERbGw77f4QfRKe5ZtCmv5gMcNmt3M6SK5O0DI3lIi1DbbZ8/JE2dWI +Et12TfIa/G8jHnrx2JhFTgcQ7xZC0EN1bUre4qrJMf8fAHB8Zs8QJQi6+u4A6UYD +ZicRFTuqW/KY3TZCstqIdQIDAQABoxMwETAPBgNVHRMBAf8EBTADAQH/MA0GCSqG +SIb3DQEBBAUAA4GBAGk2ifc0KjNyL2071CKyuG+axTZmDhs8obF1Wub9NdP4qPIH +b4Vnjt4rueIXsDqg8A6iAJrf8xQVbrvIhVqYgPn/vnQdPfP+MCXRNzRn+qVxeTBh +KXLA4CxM+1bkOqhv5TJZUtt1KFBZDPgLGeSs2a+WjS9Q2wfD6h+rM+D1KzGJ +-----END CERTIFICATE----- + +Thawte Personal Freemail CA +=========================== + +-----BEGIN CERTIFICATE----- +MIIDLTCCApagAwIBAgIBADANBgkqhkiG9w0BAQQFADCB0TELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMRowGAYD +VQQKExFUaGF3dGUgQ29uc3VsdGluZzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBT +ZXJ2aWNlcyBEaXZpc2lvbjEkMCIGA1UEAxMbVGhhd3RlIFBlcnNvbmFsIEZyZWVt +YWlsIENBMSswKQYJKoZIhvcNAQkBFhxwZXJzb25hbC1mcmVlbWFpbEB0aGF3dGUu +Y29tMB4XDTk2MDEwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgdExCzAJBgNVBAYT +AlpBMRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEa +MBgGA1UEChMRVGhhd3RlIENvbnN1bHRpbmcxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xJDAiBgNVBAMTG1RoYXd0ZSBQZXJzb25hbCBG +cmVlbWFpbCBDQTErMCkGCSqGSIb3DQEJARYccGVyc29uYWwtZnJlZW1haWxAdGhh +d3RlLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA1GnX1LCUZFtx6UfY +DFG26nKRsIRefS0Nj3sS34UldSh0OkIsYyeflXtL734Zhx2G6qPduc6WZBrCFG5E +rHzmj+hND3EfQDimAKOHePb5lIZererAXnbr2RSjXW56fAylS1V/Bhkpf56aJtVq +uzgkCGqYx7Hao5iR/Xnb5VrEHLkCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zAN +BgkqhkiG9w0BAQQFAAOBgQDH7JJ+Tvj1lqVnYiqk8E0RYNBvjWBYYawmu1I1XAjP +MPuoSpaKH2JCI4wXD/S6ZJwXrEcp352YXtJsYHFcoqzceePnbgBHH7UNKOgCneSa +/RP0ptl8sfjcXyMmCZGAc9AUG95DqYMl8uacLxXK/qarigd1iwzdUYRr5PjRznei +gQ== +-----END CERTIFICATE----- + +Thawte Server CA +================ + +-----BEGIN CERTIFICATE----- +MIIDEzCCAnygAwIBAgIBATANBgkqhkiG9w0BAQQFADCBxDELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEm +MCQGCSqGSIb3DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wHhcNOTYwODAx +MDAwMDAwWhcNMjAxMjMxMjM1OTU5WjCBxDELMAkGA1UEBhMCWkExFTATBgNVBAgT +DFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYDVQQKExRUaGF3 +dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNl +cyBEaXZpc2lvbjEZMBcGA1UEAxMQVGhhd3RlIFNlcnZlciBDQTEmMCQGCSqGSIb3 +DQEJARYXc2VydmVyLWNlcnRzQHRoYXd0ZS5jb20wgZ8wDQYJKoZIhvcNAQEBBQAD +gY0AMIGJAoGBANOkUG7I/1Zr5s9dtuoMaHVHoqrC2oQl/Kj0R1HahbUgdJSGHg91 +yekIYfUGbTBuFRkC6VLAYttNmZ7iagxEOM3+vuNkCXDF/rFrKbYvScg71CcEJRCX +L+eQbcAoQpnXTEPew/UhbVSfXcNY4cDk2VuwuNy0e982OsK1ZiIS1ocNAgMBAAGj +EzARMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEEBQADgYEAB/pMaVz7lcxG +7oWDTSEwjsrZqG9JGubaUeNgcGyEYRGhGshIPllDfU+VPaGLtwtimHp1it2ITk6e +QNuozDJ0uW8NxuOzRAvZim+aKZuZGCg70eNAKJpaPNW15yAbi8qkq43pUdniTCxZ +qdq5snUb9kLy78fyGPmJvKP/iiMucEc= +-----END CERTIFICATE----- + +Thawte Premium Server CA +======================== + +-----BEGIN CERTIFICATE----- +MIIDJzCCApCgAwIBAgIBATANBgkqhkiG9w0BAQQFADCBzjELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2FwZSBUb3duMR0wGwYD +VQQKExRUaGF3dGUgQ29uc3VsdGluZyBjYzEoMCYGA1UECxMfQ2VydGlmaWNhdGlv +biBTZXJ2aWNlcyBEaXZpc2lvbjEhMB8GA1UEAxMYVGhhd3RlIFByZW1pdW0gU2Vy +dmVyIENBMSgwJgYJKoZIhvcNAQkBFhlwcmVtaXVtLXNlcnZlckB0aGF3dGUuY29t +MB4XDTk2MDgwMTAwMDAwMFoXDTIwMTIzMTIzNTk1OVowgc4xCzAJBgNVBAYTAlpB +MRUwEwYDVQQIEwxXZXN0ZXJuIENhcGUxEjAQBgNVBAcTCUNhcGUgVG93bjEdMBsG +A1UEChMUVGhhd3RlIENvbnN1bHRpbmcgY2MxKDAmBgNVBAsTH0NlcnRpZmljYXRp +b24gU2VydmljZXMgRGl2aXNpb24xITAfBgNVBAMTGFRoYXd0ZSBQcmVtaXVtIFNl +cnZlciBDQTEoMCYGCSqGSIb3DQEJARYZcHJlbWl1bS1zZXJ2ZXJAdGhhd3RlLmNv +bTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA0jY2aovXwlue2oFBYo847kkE +VdbQ7xwblRZH7xhINTpS9CtqBo87L+pW46+GjZ4X9560ZXUCTe/LCaIhUdib0GfQ +ug2SBhRz1JPLlyoAnFxODLz6FVL88kRu2hFKbgifLy3j+ao6hnO2RlNYyIkFvYMR +uHM/qgeN9EJN50CdHDcCAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG +9w0BAQQFAAOBgQAmSCwWwlj66BZ0DKqqX1Q/8tfJeGBeXm43YyJ3Nn6yF8Q0ufUI +hfzJATj/Tb7yFkJD57taRvvBxhEf8UqwKEbJw8RCfbz6q1lu1bdRiBHjpIUZa4JM +pAwSremkrj/xw0llmozFyD4lt5SZu5IycQfwhl7tUCemDaYj+bvLpgcUQg== +-----END CERTIFICATE----- + +Equifax Secure CA +================= + +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIENd70zzANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV +UzEQMA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2Vy +dGlmaWNhdGUgQXV0aG9yaXR5MB4XDTk4MDgyMjE2NDE1MVoXDTE4MDgyMjE2NDE1 +MVowTjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VxdWlmYXgxLTArBgNVBAsTJEVx +dWlmYXggU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eTCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEAwV2xWGcIYu6gmi0fCG2RFGiYCh7+2gRvE4RiIcPRfM6f +BeC4AfBONOziipUEZKzxa1NfBbPLZ4C/QgKO/t0BCezhABRP/PvwDN1Dulsr4R+A +cJkVV5MW8Q+XarfCaCMczE1ZMKxRHjuvK9buY0V7xdlfUNLjUA86iOe/FP3gx7kC +AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEQ +MA4GA1UEChMHRXF1aWZheDEtMCsGA1UECxMkRXF1aWZheCBTZWN1cmUgQ2VydGlm +aWNhdGUgQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTgw +ODIyMTY0MTUxWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUSOZo+SvSspXXR9gj +IBBPM5iQn9QwHQYDVR0OBBYEFEjmaPkr0rKV10fYIyAQTzOYkJ/UMAwGA1UdEwQF +MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA +A4GBAFjOKer89961zgK5F7WF0bnj4JXMJTENAKaSbn+2kmOeUJXRmm/kEd5jhW6Y +7qj/WsjTVbJmcVfewCHrPSqnI0kBBIZCe/zuf6IWUrVnZ9NA2zsmWLIodz2uFHdh +1voqZiegDfqnc1zqcPGUIWVEX/r87yloqaKHee9570+sB3c4 +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPTCCAaYCEQDNun9W8N/kvFT+IqyzcqpVMA0GCSqGSIb3DQEBAgUAMF8xCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xh +c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05 +NjAxMjkwMDAwMDBaFw0yODA4MDEyMzU5NTlaMF8xCzAJBgNVBAYTAlVTMRcwFQYD +VQQKEw5WZXJpU2lnbiwgSW5jLjE3MDUGA1UECxMuQ2xhc3MgMSBQdWJsaWMgUHJp +bWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCBnzANBgkqhkiG9w0BAQEFAAOB +jQAwgYkCgYEA5Rm/baNWYS2ZSHH2Z965jeu3noaACpEO+jglr0aIguVzqKCbJF0N +H8xlbgyw0FaEGIeaBpsQoXPftFg5a27B9hXVqKg/qhIGjTGsf7A01480Z4gJzRQR +4k5FVmkfeAKA2txHkSm7NsljXMXg1y2He6G3MrB7MLoqLzGq7qNn2tsCAwEAATAN +BgkqhkiG9w0BAQIFAAOBgQBMP7iLxmjf7kMzDl3ppssHhE16M/+SG/Q2rdiVIjZo +EWx8QszznC7EBz8UsA9P/5CSdvnivErpj82ggAr3xSnxgiJduLHdgSOjeyUVRjB5 +FvjqBUuUfx3CHMjjt/QQQDwTw18fU+hI5Ia0e6E1sHslurjTjqs/OJ0ANACY89Fx +lA== +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEC0b/EoXjaOR6+f/9YtFvgswDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAyIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAyIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQC2WoujDWojg4BrzzmH9CETMwZMJaLtVRKXxaeAufqDwSCg+i8VDXyh +YGt+eSz6Bg86rvYbb7HS/y8oUl+DfUvEerf4Zh+AVPy3wo5ZShRXRtGak75BkQO7 +FYCTXOvnzAhsPz6zSvz/S2wj1VCCJkQZjiPDceoZJEcEnnW/yKYAHwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBAIobK/o5wXTXXtgZZKJYSi034DNHD6zt96rbHuSLBlxg +J8pFUs4W7z8GZOeUaHxgMxURaa+dYo2jA1Rrpr7l7gUYYAS/QoD90KioHgE796Nc +r6Pc5iaAIzy4RHT3Cq5Ji2F4zCS/iIqnDupzGUH9TQPwiNHleI2lKk/2lw0Xd8rY +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority +======================================================= + +-----BEGIN CERTIFICATE----- +MIICPDCCAaUCEHC65B0Q2Sk0tjjKewPMur8wDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFz +cyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk2 +MDEyOTAwMDAwMFoXDTI4MDgwMTIzNTk1OVowXzELMAkGA1UEBhMCVVMxFzAVBgNV +BAoTDlZlcmlTaWduLCBJbmMuMTcwNQYDVQQLEy5DbGFzcyAzIFB1YmxpYyBQcmlt +YXJ5IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGfMA0GCSqGSIb3DQEBAQUAA4GN +ADCBiQKBgQDJXFme8huKARS0EN8EQNvjV69qRUCPhAwL0TPZ2RHP7gJYHyX3KqhE +BarsAx94f56TuZoAqiN91qyFomNFx3InzPRMxnVx0jnvT0Lwdd8KkMaOIG+YD/is +I19wKTakyYbnsZogy1Olhec9vn2a/iRFM9x2Fe0PonFkTGUugWhFpwIDAQABMA0G +CSqGSIb3DQEBAgUAA4GBALtMEivPLCYATxQT3ab7/AoRhIzzKBxnki98tsX63/Do +lbwdj2wsqFHMc9ikwFPwTtYmwHYBV4GSXiHx0bH/59AhWM1pF+NEHJwZRDmJXNyc +AA9WjQKZ7aKQRUzkuxCkPfAyAw7xzvjoyVGM5mKf5p/AfbdynMk2OmufTqj/ZA1k +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEEzH6qqYPnHTkxD4PTqJkZIwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMSBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMSBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQCq0Lq+Fi24g9TK0g+8djHKlNgdk4xWArzZbxpvUjZudVYK +VdPfQ4chEWWKfo+9Id5rMj8bhDSVBZ1BNeuS65bdqlk/AVNtmU/t5eIqWpDBucSm +Fc/IReumXY6cPvBkJHalzasab7bYe1FhbqZ/h8jit+U03EGI6glAvnOSPWvndQID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAKlPww3HZ74sy9mozS11534Vnjty637rXC0J +h9ZrbWB85a7FkCMMXErQr7Fd88e2CtvgFZMN3QO8x3aKtd1Pw5sTdbgBwObJW2ul +uIncrKTdcu1OofdPvAbT6shkdHvClUGcZXNY8ZCaPGqxmMnEh7zPRW1F4m4iP/68 +DzFc6PLZ +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAzCCAmwCEQC5L2DMiJ+hekYJuFtwbIqvMA0GCSqGSIb3DQEBBQUAMIHBMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0Ns +YXNzIDIgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBH +MjE6MDgGA1UECxMxKGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9y +aXplZCB1c2Ugb25seTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazAe +Fw05ODA1MTgwMDAwMDBaFw0yODA4MDEyMzU5NTlaMIHBMQswCQYDVQQGEwJVUzEX +MBUGA1UEChMOVmVyaVNpZ24sIEluYy4xPDA6BgNVBAsTM0NsYXNzIDIgUHVibGlj +IFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgLSBHMjE6MDgGA1UECxMx +KGMpIDE5OTggVmVyaVNpZ24sIEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25s +eTEfMB0GA1UECxMWVmVyaVNpZ24gVHJ1c3QgTmV0d29yazCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEAp4gBIXQs5xoD8JjhlzwPIQjxnNuX6Zr8wgQGE75fUsjM +HiwSViy4AWkszJkfrbCWrnkE8hM5wXuYuggs6MKEEyyqaekJ9MepAqRCwiNPStjw +DqL7MWzJ5m+ZJwf15vRMeJ5t60aG+rmGyVTyssSv1EYcWskVMP8NbPUtDm3Of3cC +AwEAATANBgkqhkiG9w0BAQUFAAOBgQByLvl/0fFx+8Se9sVeUYpAmLho+Jscg9ji +nb3/7aHmZuovCfTK1+qlK5X2JGCGTUQug6XELaDTrnhpb3LabK4I8GOSN+a7xDAX +rXfMSTWqz9iP0b63GJZHc2pUIjRkLbYWm1lbtFFZOrMLFPQS32eg9K0yZF6xRnIn +jBJ7xUS0rg== +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEH3Z/gfPqB63EHln+6eJNMYwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgMyBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgMyBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQDMXtERXVxp0KvTuWpMmR9ZmDCOFoUgRm1HP9SFIIThbbP4 +pO0M8RcPO/mn+SXXwc+EY/J8Y8+iR/LGWzOOZEAEaMGAuWQcRXfH2G71lSk8UOg0 +13gfqLptQ5GVj0VXXn7F+8qkBOvqlzdUMG+7AUcyM83cV5tkaWH4mx0ciU9cZwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAFFNzb5cy5gZnBWyATl4Lk0PZ3BwmcYQWpSk +U01UbSuvDV1Ai2TT1+7eVmGSX6bEHRBhNtMsJzzoKQm5EWR0zLVznxxIqbxhAe7i +F6YM40AIOw7n60RzKprxaZLvcRTDOaxxp5EJb+RxBrO6WVcmeQD2+A2iMzAo1KpY +oJ2daZH9 +-----END CERTIFICATE----- + +Verisign Class 4 Public Primary Certification Authority - G2 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIDAjCCAmsCEDKIjprS9esTR/h/xCA3JfgwDQYJKoZIhvcNAQEFBQAwgcExCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xh +c3MgNCBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcy +MTowOAYDVQQLEzEoYykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3Jp +emVkIHVzZSBvbmx5MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMB4X +DTk4MDUxODAwMDAwMFoXDTI4MDgwMTIzNTk1OVowgcExCzAJBgNVBAYTAlVTMRcw +FQYDVQQKEw5WZXJpU2lnbiwgSW5jLjE8MDoGA1UECxMzQ2xhc3MgNCBQdWJsaWMg +UHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEcyMTowOAYDVQQLEzEo +YykgMTk5OCBWZXJpU2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5 +MR8wHQYDVQQLExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMIGfMA0GCSqGSIb3DQEB +AQUAA4GNADCBiQKBgQC68OTP+cSuhVS5B1f5j8V/aBH4xBewRNzjMHPVKmIquNDM +HO0oW369atyzkSTKQWI8/AIBvxwWMZQFl3Zuoq29YRdsTjCG8FE3KlDHqGKB3FtK +qsGgtG7rL+VXxbErQHDbWk2hjh+9Ax/YA9SPTJlxvOKCzFjomDqG04Y48wApHwID +AQABMA0GCSqGSIb3DQEBBQUAA4GBAIWMEsGnuVAVess+rLhDityq3RS6iYF+ATwj +cSGIL4LcY/oCRaxFWdcqWERbt5+BO5JoPeI3JPV7bI92NZYJqFmduc4jq3TWg/0y +cyfYaT5DdPauxYma51N86Xv2S/PBZYPejYqcPIiNOVn8qj8ijaHBZlCBckztImRP +T8qAkbYp +-----END CERTIFICATE----- + +Verisign Class 1 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCLW3VWhFSFCwDPrzhIzrGkMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDEgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAN2E1Lm0+afY8wR4 +nN493GwTFtl63SRRZsDHJlkNrAYIwpTRMx/wgzUfbhvI3qpuFU5UJ+/EbRrsC+MO +8ESlV8dAWB6jRx9x7GD2bZTIGDnt/kIYVt/kTEkQeE4BdjVjEjbdZrwBBDajVWjV +ojYJrKshJlQGrT/KFOCsyq0GHZXi+J3x4GD/wn91K0zM2v6HmSHquv4+VNfSWXjb +PG7PoBMAGrgnoeS+Z5bKoMWznN3JdZ7rMJpfo83ZrngZPyPpXNspva1VyBtUjGP2 +6KbqxzcSXKMpHgLZ2x87tNcPVkeBFQRKr4Mn0cVYiMHd9qqnoxjaaKptEVHhv2Vr +n5Z20T0CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAq2aN17O6x5q25lXQBfGfMY1a +qtmqRiYPce2lrVNWYgFHKkTp/j90CxObufRNG7LRX7K20ohcs5/Ny9Sn2WCVhDr4 +wTcdYcrnsMXlkdpUpqwxga6X3s0IrLjAl4B/bnKk52kTlWUfxJM8/XmPBNQ+T+r3 +ns7NZ3xPZQL/kYVUc8f/NveGLezQXk//EZ9yBta4GvFMDSZl4kSAHsef493oCtrs +pSCAaWihT37ha88HQfqDjrw43bAuEbFrskLMmrz5SCJ5ShkPshw+IHTZasO+8ih4 +E1Z5T21Q6huwtVexN2ZYI/PcD98Kh8TvhgXVOBRgmaNL3gaWcSzy27YfpO8/7g== +-----END CERTIFICATE----- + +Verisign Class 2 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGTCCAwECEGFwy0mMX5hFKeewptlQW3owDQYJKoZIhvcNAQEFBQAwgcoxCzAJ +BgNVBAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVy +aVNpZ24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24s +IEluYy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNp +Z24gQ2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0 +eSAtIEczMB4XDTk5MTAwMTAwMDAwMFoXDTM2MDcxNjIzNTk1OVowgcoxCzAJBgNV +BAYTAlVTMRcwFQYDVQQKEw5WZXJpU2lnbiwgSW5jLjEfMB0GA1UECxMWVmVyaVNp +Z24gVHJ1c3QgTmV0d29yazE6MDgGA1UECxMxKGMpIDE5OTkgVmVyaVNpZ24sIElu +Yy4gLSBGb3IgYXV0aG9yaXplZCB1c2Ugb25seTFFMEMGA1UEAxM8VmVyaVNpZ24g +Q2xhc3MgMiBQdWJsaWMgUHJpbWFyeSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAt +IEczMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArwoNwtUs22e5LeWU +J92lvuCwTY+zYVY81nzD9M0+hsuiiOLh2KRpxbXiv8GmR1BeRjmL1Za6tW8UvxDO +JxOeBUebMXoT2B/Z0wI3i60sR/COgQanDTAM6/c8DyAd3HJG7qUCyFvDyVZpTMUY +wZF7C9UTAJu878NIPkZgIIUq1ZC2zYugzDLdt/1AVbJQHFauzI13TccgTacxdu9o +koqQHgiBVrKtaaNS0MscxCM9H5n+TOgWY47GCI72MfbS+uV23bUckqNJzc0BzWjN +qWm6o+sdDZykIKbBoMXRRkwXbdKsZj+WjOCE1Db/IlnF+RFgqF8EffIa9iVCYQ/E +Srg+iQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQA0JhU8wI1NQ0kdvekhktdmnLfe +xbjQ5F1fdiLAJvmEOjr5jLX77GDx6M4EsMjdpwOPMPOY36TmpDHf0xwLRtxyID+u +7gU8pDM/CzmscHhzS5kr3zDCVLCoO1Wh/hYozUK9dG6A2ydEp85EXdQbkJgNHkKU +sQAsBNB0owIFImNjzYO1+8FtYmtpdf1dcEG59b98377BMnMiIYtYgXsVkXq642RI +sH/7NiXaldDxJBQX3RiAa0YjOVT1jmIJBB2UkKab5iXiQkWquJCtvgiPqQtCGJTP +cjnhsUPgKM+351psE2tJs//jGHyJizNdrDPXp/naOlXJWBD5qu9ats9LS98q +-----END CERTIFICATE----- + +Verisign Class 3 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQCbfgZJoz5iudXukEhxKe9XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDMgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMu6nFL8eB8aHm8b +N3O9+MlrlBIwT/A2R/XQkQr1F8ilYcEWQE37imGQ5XYgwREGfassbqb1EUGO+i2t +KmFZpGcmTNDovFJbcCAEWNF6yaRpvIMXZK0Fi7zQWM6NjPXr8EJJC52XJ2cybuGu +kxUccLwgTS8Y3pKI6GyFVxEa6X7jJhFUokWWVYPKMIno3Nij7SqAP395ZVc+FSBm +CC+Vk7+qRy+oRpfwEuL+wgorUeZ25rdGt+INpsyow0xZVYnm6FNcHOqd8GIWC6fJ +Xwzw3sJ2zq/3avL6QaaiMxTJ5Xpj055iN9WFZZ4O5lMkdBteHRJTW8cs54NJOxWu +imi5V5cCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAERSWwauSCPc/L8my/uRan2Te +2yFPhpk0djZX3dAVL8WtfxUfN2JzPtTnX84XA9s1+ivbrmAJXx5fj267Cz3qWhMe +DGBvtcC1IyIuBwvLqXTLR7sdwdela8wv0kL9Sd2nic9TutoAWii/gt/4uhMdUIaC +/Y4wjylGsB49Ndo4YhYYSq3mtlFs3q9i6wHQHiT+eo8SGhJouPtmmRQURVyu565p +F4ErWjfJXir0xuKhXFSbplQAz/DxwceYMBo7Nhbbo27q/a2ywtrvAkcTisDxszGt +TxzhT5yvDwyd93gN2PQ1VoDat20Xj50egWTh/sVFuq1ruQp6Tk9LhO5L8X3dEQ== +-----END CERTIFICATE----- + +Verisign Class 4 Public Primary Certification Authority - G3 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIEGjCCAwICEQDsoKeLbnVqAc/EfMwvlF7XMA0GCSqGSIb3DQEBBQUAMIHKMQsw +CQYDVQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZl +cmlTaWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWdu +LCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlT +aWduIENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3Jp +dHkgLSBHMzAeFw05OTEwMDEwMDAwMDBaFw0zNjA3MTYyMzU5NTlaMIHKMQswCQYD +VQQGEwJVUzEXMBUGA1UEChMOVmVyaVNpZ24sIEluYy4xHzAdBgNVBAsTFlZlcmlT +aWduIFRydXN0IE5ldHdvcmsxOjA4BgNVBAsTMShjKSAxOTk5IFZlcmlTaWduLCBJ +bmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxRTBDBgNVBAMTPFZlcmlTaWdu +IENsYXNzIDQgUHVibGljIFByaW1hcnkgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkg +LSBHMzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK3LpRFpxlmr8Y+1 +GQ9Wzsy1HyDkniYlS+BzZYlZ3tCD5PUPtbut8XzoIfzk6AzufEUiGXaStBO3IFsJ ++mGuqPKljYXCKtbeZjbSmwL0qJJgfJxptI8kHtCGUvYynEFYHiK9zUVilQhu0Gbd +U6LM8BDcVHOLBKFGMzNcF0C5nk3T875Vg+ixiY5afJqWIpA7iCXy0lOIAgwLePLm +NxdLMEYH5IBtptiWLugs+BGzOA1mppvqySNb247i8xOOGlktqgLw7KSHZtzBP/XY +ufTsgsbSPZUd5cBPhMnZo0QoBmrXRazwa2rvTl/4EYIeOGM0ZlDUPpNz+jDDZq3/ +ky2X7wMCAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAj/ola09b5KROJ1WrIhVZPMq1 +CtRK26vdoV9TxaBXOcLORyu+OshWv8LZJxA6sQU8wHcxuzrTBXttmhwwjIDLk5Mq +g6sFUYICABFna/OIYUdfA5PVWw3g8dShMjWFsjrbsIKr0csKvE+MW8VLADsfKoKm +fjaF3H48ZwC15DtS4KjrXRX5xm3wrR0OhbepmnMUWluPQSjA1egtTaRezarZ7c7c +2NU8Qh0XwRJdRTjDOPP8hS6DRkiy1yBfkjaP53kPmF6Z6PDQpLv1U70qzlmwr25/ +bLvSHgCwIe34QWKCudiyxLtGUPMxxY8BqHTr9Xgn2uf3ZkPznoM+IKrDNWCRzg== +-----END CERTIFICATE----- + +Equifax Secure Global eBusiness CA +================================== + +-----BEGIN CERTIFICATE----- +MIICkDCCAfmgAwIBAgIBATANBgkqhkiG9w0BAQQFADBaMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEtMCsGA1UEAxMkRXF1aWZheCBT +ZWN1cmUgR2xvYmFsIGVCdXNpbmVzcyBDQS0xMB4XDTk5MDYyMTA0MDAwMFoXDTIw +MDYyMTA0MDAwMFowWjELMAkGA1UEBhMCVVMxHDAaBgNVBAoTE0VxdWlmYXggU2Vj +dXJlIEluYy4xLTArBgNVBAMTJEVxdWlmYXggU2VjdXJlIEdsb2JhbCBlQnVzaW5l +c3MgQ0EtMTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAuucXkAJlsTRVPEnC +UdXfp9E3j9HngXNBUmCbnaEXJnitx7HoJpQytd4zjTov2/KaelpzmKNc6fuKcxtc +58O/gGzNqfTWK8D3+ZmqY6KxRwIP1ORROhI8bIpaVIRw28HFkM9yRcuoWcDNM50/ +o5brhTMhHD4ePmBudpxnhcXIw2ECAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgAH +MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUvqigdHJQa0S3ySPY+6j/s1dr +aGwwHQYDVR0OBBYEFL6ooHRyUGtEt8kj2Puo/7NXa2hsMA0GCSqGSIb3DQEBBAUA +A4GBADDiAVGqx+pf2rnQZQ8w1j7aDRRJbpGTJxQx78T3LUX47Me/okENI7SS+RkA +Z70Br83gcfxaz2TE4JaY0KNA4gGK7ycH8WUBikQtBmV1UsCGECAhX2xrD2yuCRyv +8qIYNMR1pHMc8Y3c7635s3a0kr/clRAevsvIO1qEYBlWlKlV +-----END CERTIFICATE----- + +Equifax Secure eBusiness CA 1 +============================= + +-----BEGIN CERTIFICATE----- +MIICgjCCAeugAwIBAgIBBDANBgkqhkiG9w0BAQQFADBTMQswCQYDVQQGEwJVUzEc +MBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5jLjEmMCQGA1UEAxMdRXF1aWZheCBT +ZWN1cmUgZUJ1c2luZXNzIENBLTEwHhcNOTkwNjIxMDQwMDAwWhcNMjAwNjIxMDQw +MDAwWjBTMQswCQYDVQQGEwJVUzEcMBoGA1UEChMTRXF1aWZheCBTZWN1cmUgSW5j +LjEmMCQGA1UEAxMdRXF1aWZheCBTZWN1cmUgZUJ1c2luZXNzIENBLTEwgZ8wDQYJ +KoZIhvcNAQEBBQADgY0AMIGJAoGBAM4vGbwXt3fek6lfWg0XTzQaDJj0ItlZ1MRo +RvC0NcWFAyDGr0WlIVFFQesWWDYyb+JQYmT5/VGcqiTZ9J2DKocKIdMSODRsjQBu +WqDZQu4aIZX5UkxVWsUPOE9G+m34LjXWHXzr4vCwdYDIqROsvojvOm6rXyo4YgKw +Env+j6YDAgMBAAGjZjBkMBEGCWCGSAGG+EIBAQQEAwIABzAPBgNVHRMBAf8EBTAD +AQH/MB8GA1UdIwQYMBaAFEp4MlIR21kWNl7fwRQ2QGpHfEyhMB0GA1UdDgQWBBRK +eDJSEdtZFjZe38EUNkBqR3xMoTANBgkqhkiG9w0BAQQFAAOBgQB1W6ibAxHm6VZM +zfmpTMANmvPMZWnmJXbMWbfWVMMdzZmsGd20hdXgPfxiIKeES1hl8eL5lSE/9dR+ +WB5Hh1Q+WKG1tfgq73HnvMP2sUlG4tega+VWeponmHxGYhTnyfxuAxJ5gDgdSIKN +/Bf+KpYrtWKmpj29f5JZzVoqgrI3eQ== +-----END CERTIFICATE----- + +Equifax Secure eBusiness CA 2 +============================= + +-----BEGIN CERTIFICATE----- +MIIDIDCCAomgAwIBAgIEN3DPtTANBgkqhkiG9w0BAQUFADBOMQswCQYDVQQGEwJV +UzEXMBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2Vj +dXJlIGVCdXNpbmVzcyBDQS0yMB4XDTk5MDYyMzEyMTQ0NVoXDTE5MDYyMzEyMTQ0 +NVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkVxdWlmYXggU2VjdXJlMSYwJAYD +VQQLEx1FcXVpZmF4IFNlY3VyZSBlQnVzaW5lc3MgQ0EtMjCBnzANBgkqhkiG9w0B +AQEFAAOBjQAwgYkCgYEA5Dk5kx5SBhsoNviyoynF7Y6yEb3+6+e0dMKP/wXn2Z0G +vxLIPw7y1tEkshHe0XMJitSxLJgJDR5QRrKDpkWNYmi7hRsgcDKqQM2mll/EcTc/ +BPO3QSQ5BxoeLmFYoBIL5aXfxavqN3HMHMg3OrmXUqesxWoklE6ce8/AatbfIb0C +AwEAAaOCAQkwggEFMHAGA1UdHwRpMGcwZaBjoGGkXzBdMQswCQYDVQQGEwJVUzEX +MBUGA1UEChMORXF1aWZheCBTZWN1cmUxJjAkBgNVBAsTHUVxdWlmYXggU2VjdXJl +IGVCdXNpbmVzcyBDQS0yMQ0wCwYDVQQDEwRDUkwxMBoGA1UdEAQTMBGBDzIwMTkw +NjIzMTIxNDQ1WjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAUUJ4L6q9euSBIplBq +y/3YIHqngnYwHQYDVR0OBBYEFFCeC+qvXrkgSKZQasv92CB6p4J2MAwGA1UdEwQF +MAMBAf8wGgYJKoZIhvZ9B0EABA0wCxsFVjMuMGMDAgbAMA0GCSqGSIb3DQEBBQUA +A4GBAAyGgq3oThr1jokn4jVYPSm0B482UJW/bsGe68SQsoWou7dC4A8HOd/7npCy +0cE+U58DRLB+S/Rv5Hwf5+Kx5Lia78O9zt4LMjTZ3ijtM2vE1Nc9ElirfQkty3D1 +E4qUoSek1nDFbZS1yX2doNLGCEnZZpum0/QL3MUmV+GRMOrN +-----END CERTIFICATE----- + +Thawte Time Stamping CA +======================= + +-----BEGIN CERTIFICATE----- +MIICoTCCAgqgAwIBAgIBADANBgkqhkiG9w0BAQQFADCBizELMAkGA1UEBhMCWkEx +FTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzAN +BgNVBAoTBlRoYXd0ZTEdMBsGA1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAd +BgNVBAMTFlRoYXd0ZSBUaW1lc3RhbXBpbmcgQ0EwHhcNOTcwMTAxMDAwMDAwWhcN +MjAxMjMxMjM1OTU5WjCBizELMAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4g +Q2FwZTEUMBIGA1UEBxMLRHVyYmFudmlsbGUxDzANBgNVBAoTBlRoYXd0ZTEdMBsG +A1UECxMUVGhhd3RlIENlcnRpZmljYXRpb24xHzAdBgNVBAMTFlRoYXd0ZSBUaW1l +c3RhbXBpbmcgQ0EwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANYrWHhhRYZT +6jR7UZztsOYuGA7+4F+oJ9O0yeB8WU4WDnNUYMF/9p8u6TqFJBU820cEY8OexJQa +Wt9MevPZQx08EHp5JduQ/vBR5zDWQQD9nyjfeb6Uu522FOMjhdepQeBMpHmwKxqL +8vg7ij5FrHGSALSQQZj7X+36ty6K+Ig3AgMBAAGjEzARMA8GA1UdEwEB/wQFMAMB +Af8wDQYJKoZIhvcNAQEEBQADgYEAZ9viwuaHPUCDhjc1fR/OmsMMZiCouqoEiYbC +9RAIDb/LogWK0E02PvTX72nGXuSwlG9KuefeW4i2e9vjJ+V2w/A1wcu1J5szedyQ +pgCed/r8zSeUQhac0xxo7L9c3eWpexAKMnRUEzGLhQOEkbdYATAUOK8oyvyxUBkZ +CayJSdM= +-----END CERTIFICATE----- + +thawte Primary Root CA +====================== + +-----BEGIN CERTIFICATE----- +MIIEIDCCAwigAwIBAgIQNE7VVyDV7exJ9C/ON9srbTANBgkqhkiG9w0BAQUFADCB +qTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5jLjEoMCYGA1UECxMf +Q2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYGA1UECxMvKGMpIDIw +MDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxHzAdBgNV +BAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwHhcNMDYxMTE3MDAwMDAwWhcNMzYw +NzE2MjM1OTU5WjCBqTELMAkGA1UEBhMCVVMxFTATBgNVBAoTDHRoYXd0ZSwgSW5j +LjEoMCYGA1UECxMfQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcyBEaXZpc2lvbjE4MDYG +A1UECxMvKGMpIDIwMDYgdGhhd3RlLCBJbmMuIC0gRm9yIGF1dGhvcml6ZWQgdXNl +IG9ubHkxHzAdBgNVBAMTFnRoYXd0ZSBQcmltYXJ5IFJvb3QgQ0EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCsoPD7gFnUnMekz52hWXMJEEUMDSxuaPFs +W0hoSVk3/AszGcJ3f8wQLZU0HObrTQmnHNK4yZc2AreJ1CRfBsDMRJSUjQJib+ta +3RGNKJpchJAQeg29dGYvajig4tVUROsdB58Hum/u6f1OCyn1PoSgAfGcq/gcfomk +6KHYcWUNo1F77rzSImANuVud37r8UVsLr5iy6S7pBOhih94ryNdOwUxkHt3Ph1i6 +Sk/KaAcdHJ1KxtUvkcx8cXIcxcBn6zL9yZJclNqFwJu/U30rCfSMnZEfl2pSy94J +NqR32HuHUETVPm4pafs5SSYeCaWAe0At6+gnhcn+Yf1+5nyXHdWdAgMBAAGjQjBA +MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBR7W0XP +r87Lev0xkhpqtvNG61dIUDANBgkqhkiG9w0BAQUFAAOCAQEAeRHAS7ORtvzw6WfU +DW5FvlXok9LOAz/t2iWwHVfLHjp2oEzsUHboZHIMpKnxuIvW1oeEuzLlQRHAd9mz +YJ3rG9XRbkREqaYB7FViHXe4XI5ISXycO1cRrK1zN44veFyQaEfZYGDm/Ac9IiAX +xPcW6cTYcvnIc3zfFi8VqT79aie2oetaupgf1eNNZAqdE8hhuvU5HIe6uL17In/2 +/qxAeeWsEG89jxt5dovEN7MhGITlNgDrYyCZuen+MwS7QcjBAvlEYyCegc5C09Y/ +LHbTY5xZ3Y+m4Q6gLkH3LpVHz7z9M/P2C2F+fpErgUfCJzDupxBdN49cOSvkBPB7 +jVaMaA== +-----END CERTIFICATE----- + +VeriSign Class 3 Public Primary Certification Authority - G5 +============================================================ + +-----BEGIN CERTIFICATE----- +MIIE0zCCA7ugAwIBAgIQGNrRniZ96LtKIVjNzGs7SjANBgkqhkiG9w0BAQUFADCB +yjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJp +U2lnbiwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxW +ZXJpU2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5IC0gRzUwHhcNMDYxMTA4MDAwMDAwWhcNMzYwNzE2MjM1OTU5WjCByjEL +MAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQLExZW +ZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTowOAYDVQQLEzEoYykgMjAwNiBWZXJpU2ln +biwgSW5jLiAtIEZvciBhdXRob3JpemVkIHVzZSBvbmx5MUUwQwYDVQQDEzxWZXJp +U2lnbiBDbGFzcyAzIFB1YmxpYyBQcmltYXJ5IENlcnRpZmljYXRpb24gQXV0aG9y +aXR5IC0gRzUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvJAgIKXo1 +nmAMqudLO07cfLw8RRy7K+D+KQL5VwijZIUVJ/XxrcgxiV0i6CqqpkKzj/i5Vbex +t0uz/o9+B1fs70PbZmIVYc9gDaTY3vjgw2IIPVQT60nKWVSFJuUrjxuf6/WhkcIz +SdhDY2pSS9KP6HBRTdGJaXvHcPaz3BJ023tdS1bTlr8Vd6Gw9KIl8q8ckmcY5fQG +BO+QueQA5N06tRn/Arr0PO7gi+s3i+z016zy9vA9r911kTMZHRxAy3QkGSGT2RT+ +rCpSx4/VBEnkjWNHiDxpg8v+R70rfk/Fla4OndTRQ8Bnc+MUCH7lP59zuDMKz10/ +NIeWiu5T6CUVAgMBAAGjgbIwga8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8E +BAMCAQYwbQYIKwYBBQUHAQwEYTBfoV2gWzBZMFcwVRYJaW1hZ2UvZ2lmMCEwHzAH +BgUrDgMCGgQUj+XTGoasjY5rw8+AatRIGCx7GS4wJRYjaHR0cDovL2xvZ28udmVy +aXNpZ24uY29tL3ZzbG9nby5naWYwHQYDVR0OBBYEFH/TZafC3ey78DAJ80M5+gKv +MzEzMA0GCSqGSIb3DQEBBQUAA4IBAQCTJEowX2LP2BqYLz3q3JktvXf2pXkiOOzE +p6B4Eq1iDkVwZMXnl2YtmAl+X6/WzChl8gGqCBpH3vn5fJJaCGkgDdk+bW48DW7Y +5gaRQBi5+MHt39tBquCWIMnNZBU4gcmU7qKEKQsTb47bDN0lAtukixlE0kF6BWlK +WE9gyn6CagsCqiUXObXbf+eEZSqVir2G3l6BFoMtEMze/aiCKm0oHw0LxOXnGiYZ +4fQRbxC1lfznQgUy286dUV4otp6F01vvpX1FQHKOtw5rDgb7MzVIcbidJ4vEZV8N +hnacRHr2lVz2XTIIM6RUthg/aFzyQkqFOFSDX9HoLPKsEdao7WNq +-----END CERTIFICATE----- + +Entrust.net Secure Server Certification Authority +================================================= + +-----BEGIN CERTIFICATE----- +MIIE2DCCBEGgAwIBAgIEN0rSQzANBgkqhkiG9w0BAQUFADCBwzELMAkGA1UEBhMC +VVMxFDASBgNVBAoTC0VudHJ1c3QubmV0MTswOQYDVQQLEzJ3d3cuZW50cnVzdC5u +ZXQvQ1BTIGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxpYWIuKTElMCMGA1UECxMc +KGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDE6MDgGA1UEAxMxRW50cnVzdC5u +ZXQgU2VjdXJlIFNlcnZlciBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw05OTA1 +MjUxNjA5NDBaFw0xOTA1MjUxNjM5NDBaMIHDMQswCQYDVQQGEwJVUzEUMBIGA1UE +ChMLRW50cnVzdC5uZXQxOzA5BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5j +b3JwLiBieSByZWYuIChsaW1pdHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBF +bnRydXN0Lm5ldCBMaW1pdGVkMTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUg +U2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGdMA0GCSqGSIb3DQEBAQUA +A4GLADCBhwKBgQDNKIM0VBuJ8w+vN5Ex/68xYMmo6LIQaO2f55M28Qpku0f1BBc/ +I0dNxScZgSYMVHINiC3ZH5oSn7yzcdOAGT9HZnuMNSjSuQrfJNqc1lB5gXpa0zf3 +wkrYKZImZNHkmGw6AIr1NJtl+O3jEP/9uElY3KDegjlrgbEWGWG5VLbmQwIBA6OC +AdcwggHTMBEGCWCGSAGG+EIBAQQEAwIABzCCARkGA1UdHwSCARAwggEMMIHeoIHb +oIHYpIHVMIHSMQswCQYDVQQGEwJVUzEUMBIGA1UEChMLRW50cnVzdC5uZXQxOzA5 +BgNVBAsTMnd3dy5lbnRydXN0Lm5ldC9DUFMgaW5jb3JwLiBieSByZWYuIChsaW1p +dHMgbGlhYi4pMSUwIwYDVQQLExwoYykgMTk5OSBFbnRydXN0Lm5ldCBMaW1pdGVk +MTowOAYDVQQDEzFFbnRydXN0Lm5ldCBTZWN1cmUgU2VydmVyIENlcnRpZmljYXRp +b24gQXV0aG9yaXR5MQ0wCwYDVQQDEwRDUkwxMCmgJ6AlhiNodHRwOi8vd3d3LmVu +dHJ1c3QubmV0L0NSTC9uZXQxLmNybDArBgNVHRAEJDAigA8xOTk5MDUyNTE2MDk0 +MFqBDzIwMTkwNTI1MTYwOTQwWjALBgNVHQ8EBAMCAQYwHwYDVR0jBBgwFoAU8Bdi +E1U9s/8KAGv7UISX8+1i0BowHQYDVR0OBBYEFPAXYhNVPbP/CgBr+1CEl/PtYtAa +MAwGA1UdEwQFMAMBAf8wGQYJKoZIhvZ9B0EABAwwChsEVjQuMAMCBJAwDQYJKoZI +hvcNAQEFBQADgYEAkNwwAvpkdMKnCqV8IY00F6j7Rw7/JXyNEwr75Ji174z4xRAN +95K+8cPV1ZVqBLssziY2ZcgxxufuP+NXdYR6Ee9GTxj005i7qIcyunL2POI9n9cd +2cNgQ4xYDiKWL2KjLB+6rQXvqzJ4h6BUcxm1XAX5Uj5tLUUL9wqT6u0G+bI= +-----END CERTIFICATE----- + +Go Daddy Certification Authority Root Certificate Bundle +======================================================== + +-----BEGIN CERTIFICATE----- +MIIE3jCCA8agAwIBAgICAwEwDQYJKoZIhvcNAQEFBQAwYzELMAkGA1UEBhMCVVMx +ITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28g +RGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0wNjExMTYw +MTU0MzdaFw0yNjExMTYwMTU0MzdaMIHKMQswCQYDVQQGEwJVUzEQMA4GA1UECBMH +QXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5j +b20sIEluYy4xMzAxBgNVBAsTKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5j +b20vcmVwb3NpdG9yeTEwMC4GA1UEAxMnR28gRGFkZHkgU2VjdXJlIENlcnRpZmlj +YXRpb24gQXV0aG9yaXR5MREwDwYDVQQFEwgwNzk2OTI4NzCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMQt1RWMnCZM7DI161+4WQFapmGBWTtwY6vj3D3H +KrjJM9N55DrtPDAjhI6zMBS2sofDPZVUBJ7fmd0LJR4h3mUpfjWoqVTr9vcyOdQm +VZWt7/v+WIbXnvQAjYwqDL1CBM6nPwT27oDyqu9SoWlm2r4arV3aLGbqGmu75RpR +SgAvSMeYddi5Kcju+GZtCpyz8/x4fKL4o/K1w/O5epHBp+YlLpyo7RJlbmr2EkRT +cDCVw5wrWCs9CHRK8r5RsL+H0EwnWGu1NcWdrxcx+AuP7q2BNgWJCJjPOq8lh8BJ +6qf9Z/dFjpfMFDniNoW1fho3/Rb2cRGadDAW/hOUoz+EDU8CAwEAAaOCATIwggEu +MB0GA1UdDgQWBBT9rGEyk2xF1uLuhV+auud2mWjM5zAfBgNVHSMEGDAWgBTSxLDS +kdRMEXGzYcs9of7dqGrU4zASBgNVHRMBAf8ECDAGAQH/AgEAMDMGCCsGAQUFBwEB +BCcwJTAjBggrBgEFBQcwAYYXaHR0cDovL29jc3AuZ29kYWRkeS5jb20wRgYDVR0f +BD8wPTA7oDmgN4Y1aHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNvbS9yZXBv +c2l0b3J5L2dkcm9vdC5jcmwwSwYDVR0gBEQwQjBABgRVHSAAMDgwNgYIKwYBBQUH +AgEWKmh0dHA6Ly9jZXJ0aWZpY2F0ZXMuZ29kYWRkeS5jb20vcmVwb3NpdG9yeTAO +BgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBANKGwOy9+aG2Z+5mC6IG +OgRQjhVyrEp0lVPLN8tESe8HkGsz2ZbwlFalEzAFPIUyIXvJxwqoJKSQ3kbTJSMU +A2fCENZvD117esyfxVgqwcSeIaha86ykRvOe5GPLL5CkKSkB2XIsKd83ASe8T+5o +0yGPwLPk9Qnt0hCqU7S+8MxZC9Y7lhyVJEnfzuz9p0iRFEUOOjZv2kWzRaJBydTX +RE4+uXR21aITVSzGh6O1mawGhId/dQb8vxRMDsxuxN89txJx9OjxUUAiKEngHUuH +qDTMBqLdElrRhjZkAzVvb3du6/KFUJheqwNTrZEjYx8WnM25sgVjOuH0aBsXBTWV +U+4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIE+zCCBGSgAwIBAgICAQ0wDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1Zh +bGlDZXJ0IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIElu +Yy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24g +QXV0aG9yaXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAe +BgkqhkiG9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTA0MDYyOTE3MDYyMFoX +DTI0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBE +YWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3MgMiBDZXJ0 +aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgC +ggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv +2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+q +N1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiO +r18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lN +f4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+YihfukEH +U1jPEX44dMX4/7VpkI+EdOqXG68CAQOjggHhMIIB3TAdBgNVHQ4EFgQU0sSw0pHU +TBFxs2HLPaH+3ahq1OMwgdIGA1UdIwSByjCBx6GBwaSBvjCBuzEkMCIGA1UEBxMb +VmFsaUNlcnQgVmFsaWRhdGlvbiBOZXR3b3JrMRcwFQYDVQQKEw5WYWxpQ2VydCwg +SW5jLjE1MDMGA1UECxMsVmFsaUNlcnQgQ2xhc3MgMiBQb2xpY3kgVmFsaWRhdGlv +biBBdXRob3JpdHkxITAfBgNVBAMTGGh0dHA6Ly93d3cudmFsaWNlcnQuY29tLzEg +MB4GCSqGSIb3DQEJARYRaW5mb0B2YWxpY2VydC5jb22CAQEwDwYDVR0TAQH/BAUw +AwEB/zAzBggrBgEFBQcBAQQnMCUwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3NwLmdv +ZGFkZHkuY29tMEQGA1UdHwQ9MDswOaA3oDWGM2h0dHA6Ly9jZXJ0aWZpY2F0ZXMu +Z29kYWRkeS5jb20vcmVwb3NpdG9yeS9yb290LmNybDBLBgNVHSAERDBCMEAGBFUd +IAAwODA2BggrBgEFBQcCARYqaHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNv +bS9yZXBvc2l0b3J5MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOBgQC1 +QPmnHfbq/qQaQlpE9xXUhUaJwL6e4+PrxeNYiY+Sn1eocSxI0YGyeR+sBjUZsE4O +WBsUs5iB0QQeyAfJg594RAoYC5jcdnplDQ1tgMQLARzLrUc+cb53S8wGd9D0Vmsf +SxOaFIqII6hR8INMqzW/Rn453HWkrugp++85j09VZw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC5zCCAlACAQEwDQYJKoZIhvcNAQEFBQAwgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0 +IFZhbGlkYXRpb24gTmV0d29yazEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAz +BgNVBAsTLFZhbGlDZXJ0IENsYXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9y +aXR5MSEwHwYDVQQDExhodHRwOi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG +9w0BCQEWEWluZm9AdmFsaWNlcnQuY29tMB4XDTk5MDYyNjAwMTk1NFoXDTE5MDYy +NjAwMTk1NFowgbsxJDAiBgNVBAcTG1ZhbGlDZXJ0IFZhbGlkYXRpb24gTmV0d29y +azEXMBUGA1UEChMOVmFsaUNlcnQsIEluYy4xNTAzBgNVBAsTLFZhbGlDZXJ0IENs +YXNzIDIgUG9saWN5IFZhbGlkYXRpb24gQXV0aG9yaXR5MSEwHwYDVQQDExhodHRw +Oi8vd3d3LnZhbGljZXJ0LmNvbS8xIDAeBgkqhkiG9w0BCQEWEWluZm9AdmFsaWNl +cnQuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOOnHK5avIWZJV16vY +dA757tn2VUdZZUcOBVXc65g2PFxTXdMwzzjsvUGJ7SVCCSRrCl6zfN1SLUzm1NZ9 +WlmpZdRJEy0kTRxQb7XBhVQ7/nHk01xC+YDgkRoKWzk2Z/M/VXwbP7RfZHM047QS +v4dk+NoS/zcnwbNDu+97bi5p9wIDAQABMA0GCSqGSIb3DQEBBQUAA4GBADt/UG9v +UJSZSWI4OB9L+KXIPqeCgfYrx+jFzug6EILLGACOTb2oWH+heQC1u+mNr0HZDzTu +IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC +W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd +-----END CERTIFICATE----- + +GeoTrust Global CA +================== + +-----BEGIN CERTIFICATE----- +MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT +MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 +aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAw +WjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UE +AxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9m +OSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIu +T8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6c +JmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmR +Cw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5asz +PeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjm +aPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrM +TjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+g +LaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBO +BgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2Vv +dHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GB +AHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrL +NhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1W +b8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S +-----END CERTIFICATE----- + diff --git a/libs/httplib2/iri2uri.py b/libs/httplib2/iri2uri.py index 70667edf..d88c91fd 100644 --- a/libs/httplib2/iri2uri.py +++ b/libs/httplib2/iri2uri.py @@ -16,7 +16,7 @@ import urlparse # Convert an IRI to a URI following the rules in RFC 3987 -# +# # The characters we need to enocde and escape are defined in the spec: # # iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD @@ -28,28 +28,28 @@ import urlparse # / %xD0000-DFFFD / %xE1000-EFFFD escape_range = [ - (0xA0, 0xD7FF ), - (0xE000, 0xF8FF ), - (0xF900, 0xFDCF ), - (0xFDF0, 0xFFEF), - (0x10000, 0x1FFFD ), - (0x20000, 0x2FFFD ), - (0x30000, 0x3FFFD), - (0x40000, 0x4FFFD ), - (0x50000, 0x5FFFD ), - (0x60000, 0x6FFFD), - (0x70000, 0x7FFFD ), - (0x80000, 0x8FFFD ), - (0x90000, 0x9FFFD), - (0xA0000, 0xAFFFD ), - (0xB0000, 0xBFFFD ), - (0xC0000, 0xCFFFD), - (0xD0000, 0xDFFFD ), - (0xE1000, 0xEFFFD), - (0xF0000, 0xFFFFD ), - (0x100000, 0x10FFFD) + (0xA0, 0xD7FF), + (0xE000, 0xF8FF), + (0xF900, 0xFDCF), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD), + (0x20000, 0x2FFFD), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD), + (0x50000, 0x5FFFD), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD), + (0x80000, 0x8FFFD), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD), + (0xB0000, 0xBFFFD), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD), + (0x100000, 0x10FFFD), ] - + def encode(c): retval = c i = ord(c) @@ -63,19 +63,19 @@ def encode(c): def iri2uri(uri): - """Convert an IRI to a URI. Note that IRIs must be + """Convert an IRI to a URI. Note that IRIs must be passed in a unicode strings. That is, do not utf-8 encode - the IRI before passing it into the function.""" + the IRI before passing it into the function.""" if isinstance(uri ,unicode): (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) authority = authority.encode('idna') # For each character in 'ucschar' or 'iprivate' # 1. encode as utf-8 - # 2. then %-encode each octet of that utf-8 + # 2. then %-encode each octet of that utf-8 uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) uri = "".join([encode(c) for c in uri]) return uri - + if __name__ == "__main__": import unittest @@ -83,7 +83,7 @@ if __name__ == "__main__": def test_uris(self): """Test that URIs are invariant under the transformation.""" - invariant = [ + invariant = [ u"ftp://ftp.is.co.za/rfc/rfc1808.txt", u"http://www.ietf.org/rfc/rfc2396.txt", u"ldap://[2001:db8::7]/c=GB?objectClass?one", @@ -94,7 +94,7 @@ if __name__ == "__main__": u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] for uri in invariant: self.assertEqual(uri, iri2uri(uri)) - + def test_iri(self): """ Test that the right type of escaping is done for each part of the URI.""" self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) @@ -107,4 +107,4 @@ if __name__ == "__main__": unittest.main() - + diff --git a/libs/httplib2/socks.py b/libs/httplib2/socks.py new file mode 100644 index 00000000..0991f4cf --- /dev/null +++ b/libs/httplib2/socks.py @@ -0,0 +1,438 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +""" + +import base64 +import socket +import struct +import sys + +if getattr(socket, 'socket', None) is None: + raise ImportError('socket.socket missing, proxy support unusable') + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 +PROXY_TYPE_HTTP_NO_TUNNEL = 4 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): pass +class GeneralProxyError(ProxyError): pass +class Socks5AuthError(ProxyError): pass +class Socks5Error(ProxyError): pass +class Socks4Error(ProxyError): pass +class HTTPError(ProxyError): pass + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + +def wrapmodule(module): + """wrapmodule(module) + Attempts to replace a module's socket library with a SOCKS socket. Must set + a default proxy using setdefaultproxy(...) first. + This will only work on modules that import socket directly into the namespace; + most of the Python Standard Library falls into this category. + """ + if _defaultproxy != None: + module.socket.socket = socksocket + else: + raise GeneralProxyError((4, "no proxy specified")) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + self.__httptunnel = True + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = self.recv(count) + while len(data) < count: + d = self.recv(count-len(data)) + if not d: raise GeneralProxyError((0, "connection closed unexpectedly")) + data = data + d + return data + + def sendall(self, content, *args): + """ override socket.socket.sendall method to rewrite the header + for non-tunneling proxies if needed + """ + if not self.__httptunnel: + content = self.__rewriteproxy(content) + return super(socksocket, self).sendall(content, *args) + + def __rewriteproxy(self, header): + """ rewrite HTTP request headers to support non-tunneling proxies + (i.e. those which do not support the CONNECT method). + This only works for HTTP (not HTTPS) since HTTPS requires tunneling. + """ + host, endpt = None, None + hdrs = header.split("\r\n") + for hdr in hdrs: + if hdr.lower().startswith("host:"): + host = hdr + elif hdr.lower().startswith("get") or hdr.lower().startswith("post"): + endpt = hdr + if host and endpt: + hdrs.remove(host) + hdrs.remove(endpt) + host = host.split(" ")[1] + endpt = endpt.split(" ") + if (self.__proxy[4] != None and self.__proxy[5] != None): + hdrs.insert(0, self.__getauthheader()) + hdrs.insert(0, "Host: %s" % host) + hdrs.insert(0, "%s http://%s%s %s" % (endpt[0], host, endpt[1], endpt[2])) + return "\r\n".join(hdrs) + + def __getauthheader(self): + auth = self.__proxy[4] + ":" + self.__proxy[5] + return "Proxy-Authorization: Basic " + base64.b64encode(auth) + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall(struct.pack('BBBB', 0x05, 0x02, 0x00, 0x02)) + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall(struct.pack('BBB', 0x05, 0x01, 0x00)) + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1:2] == chr(0x00).encode(): + # No authentication is required + pass + elif chosenauth[1:2] == chr(0x02).encode(): + # Okay, we need to perform a basic username/password + # authentication. + self.sendall(chr(0x01).encode() + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0:1] != chr(0x01).encode(): + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1:2] != chr(0x00).encode(): + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == chr(0xFF).encode(): + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = struct.pack('BBB', 0x05, 0x01, 0x00) + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + chr(0x01).encode() + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]: + # Resolve remotely + ipaddr = None + req = req + chr(0x03).encode() + chr(len(destaddr)).encode() + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + chr(0x01).encode() + ipaddr + req = req + struct.pack(">H", destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0:1] != chr(0x05).encode(): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1:2] != chr(0x00).encode(): + # Connection failed + self.close() + if ord(resp[1:2])<=8: + raise Socks5Error((ord(resp[1:2]), _socks5errors[ord(resp[1:2])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3:4] == chr(0x01).encode(): + boundaddr = self.__recvall(4) + elif resp[3:4] == chr(0x03).encode(): + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4:5])) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H", self.__recvall(2))[0] + self.__proxysockname = (boundaddr, boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]: + ipaddr = struct.pack("BBBB", 0x00, 0x00, 0x00, 0x01) + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = struct.pack(">BBH", 0x04, 0x01, destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + chr(0x00).encode() + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv: + req = req + destaddr + chr(0x00).encode() + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0:1] != chr(0x00).encode(): + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1:2] != chr(0x5A).encode(): + # Server returned an error + self.close() + if ord(resp[1:2]) in (91, 92, 93): + self.close() + raise Socks4Error((ord(resp[1:2]), _socks4errors[ord(resp[1:2]) - 90])) + else: + raise Socks4Error((94, _socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]), struct.unpack(">H", resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if not self.__proxy[3]: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + headers = ["CONNECT ", addr, ":", str(destport), " HTTP/1.1\r\n"] + headers += ["Host: ", destaddr, "\r\n"] + if (self.__proxy[4] != None and self.__proxy[5] != None): + headers += [self.__getauthheader(), "\r\n"] + headers.append("\r\n") + self.sendall("".join(headers).encode()) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n".encode()) == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ".encode(), 2) + if statusline[0] not in ("HTTP/1.0".encode(), "HTTP/1.1".encode()): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self, despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (not type(destpair) in (list,tuple)) or (len(destpair) < 2) or (not isinstance(destpair[0], basestring)) or (type(destpair[1]) != int): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP_NO_TUNNEL: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + if destpair[1] == 443: + self.__negotiatehttp(destpair[0],destpair[1]) + else: + self.__httptunnel = False + elif self.__proxy[0] == None: + _orgsocket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) diff --git a/libs/pyasn1/__init__.py b/libs/pyasn1/__init__.py index 7de39fe5..88aff79c 100644 --- a/libs/pyasn1/__init__.py +++ b/libs/pyasn1/__init__.py @@ -1 +1,8 @@ -majorVersionId = '1' +import sys + +# http://www.python.org/dev/peps/pep-0396/ +__version__ = '0.1.7' + +if sys.version_info[:2] < (2, 4): + raise RuntimeError('PyASN1 requires Python 2.4 or later') + diff --git a/libs/pyasn1/codec/__init__.py b/libs/pyasn1/codec/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/__init__.py +++ b/libs/pyasn1/codec/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/ber/__init__.py b/libs/pyasn1/codec/ber/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/ber/__init__.py +++ b/libs/pyasn1/codec/ber/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/ber/decoder.py b/libs/pyasn1/codec/ber/decoder.py index ae9311cb..be0cf490 100644 --- a/libs/pyasn1/codec/ber/decoder.py +++ b/libs/pyasn1/codec/ber/decoder.py @@ -1,21 +1,24 @@ # BER decoder from pyasn1.type import tag, base, univ, char, useful, tagmap from pyasn1.codec.ber import eoo -from pyasn1.compat.octets import oct2int, octs2ints -from pyasn1 import error +from pyasn1.compat.octets import oct2int, octs2ints, isOctetsType +from pyasn1 import debug, error class AbstractDecoder: protoComponent = None def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - raise error.PyAsn1Error('Decoder not implemented for %s' % tagSet) + length, state, decodeFun, substrateFun): + raise error.PyAsn1Error('Decoder not implemented for %s' % (tagSet,)) def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % tagSet) + length, state, decodeFun, substrateFun): + raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,)) class AbstractSimpleDecoder(AbstractDecoder): + tagFormats = (tag.tagFormatSimple,) def _createComponent(self, asn1Spec, tagSet, value=None): + if tagSet[0][1] not in self.tagFormats: + raise error.PyAsn1Error('Invalid tag format %r for %r' % (tagSet[0], self.protoComponent,)) if asn1Spec is None: return self.protoComponent.clone(value, tagSet) elif value is None: @@ -24,7 +27,10 @@ class AbstractSimpleDecoder(AbstractDecoder): return asn1Spec.clone(value) class AbstractConstructedDecoder(AbstractDecoder): + tagFormats = (tag.tagFormatConstructed,) def _createComponent(self, asn1Spec, tagSet, value=None): + if tagSet[0][1] not in self.tagFormats: + raise error.PyAsn1Error('Invalid tag format %r for %r' % (tagSet[0], self.protoComponent,)) if asn1Spec is None: return self.protoComponent.clone(tagSet) else: @@ -32,19 +38,34 @@ class AbstractConstructedDecoder(AbstractDecoder): class EndOfOctetsDecoder(AbstractSimpleDecoder): def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - return eoo.endOfOctets, substrate[:length] + length, state, decodeFun, substrateFun): + return eoo.endOfOctets, substrate[length:] class ExplicitTagDecoder(AbstractSimpleDecoder): + protoComponent = univ.Any('') + tagFormats = (tag.tagFormatConstructed,) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - return decodeFun(substrate[:length], asn1Spec, tagSet, length) + length, state, decodeFun, substrateFun): + if substrateFun: + return substrateFun( + self._createComponent(asn1Spec, tagSet, ''), + substrate, length + ) + head, tail = substrate[:length], substrate[length:] + value, _ = decodeFun(head, asn1Spec, tagSet, length) + return value, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): + if substrateFun: + return substrateFun( + self._createComponent(asn1Spec, tagSet, ''), + substrate, length + ) value, substrate = decodeFun(substrate, asn1Spec, tagSet, length) terminator, substrate = decodeFun(substrate) - if terminator == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(terminator) and \ + terminator == eoo.endOfOctets: return value, substrate else: raise error.PyAsn1Error('Missing end-of-octets terminator') @@ -71,79 +92,71 @@ class IntegerDecoder(AbstractSimpleDecoder): '\xfb': -5 } - def _valueFilter(self, value): - try: - return int(value) - except OverflowError: - return value - def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: - raise error.PyAsn1Error('Empty substrate') - if substrate in self.precomputedValues: - value = self.precomputedValues[substrate] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: + return self._createComponent(asn1Spec, tagSet, 0), tail + if head in self.precomputedValues: + value = self.precomputedValues[head] else: - firstOctet = oct2int(substrate[0]) + firstOctet = oct2int(head[0]) if firstOctet & 0x80: value = -1 else: value = 0 - for octet in substrate: + for octet in head: value = value << 8 | oct2int(octet) - value = self._valueFilter(value) - return self._createComponent(asn1Spec, tagSet, value), substrate + return self._createComponent(asn1Spec, tagSet, value), tail class BooleanDecoder(IntegerDecoder): protoComponent = univ.Boolean(0) - def _valueFilter(self, value): - if value: - return 1 - else: - return 0 + def _createComponent(self, asn1Spec, tagSet, value=None): + return IntegerDecoder._createComponent(self, asn1Spec, tagSet, value and 1 or 0) class BitStringDecoder(AbstractSimpleDecoder): protoComponent = univ.BitString(()) + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] if tagSet[0][1] == tag.tagFormatSimple: # XXX what tag to check? - if not substrate: - raise error.PyAsn1Error('Missing initial octet') - trailingBits = oct2int(substrate[0]) + if not head: + raise error.PyAsn1Error('Empty substrate') + trailingBits = oct2int(head[0]) if trailingBits > 7: raise error.PyAsn1Error( 'Trailing bits overflow %s' % trailingBits ) - substrate = substrate[1:] - lsb = p = 0; l = len(substrate)-1; b = () + head = head[1:] + lsb = p = 0; l = len(head)-1; b = () while p <= l: if p == l: lsb = trailingBits j = 7 - o = oct2int(substrate[p]) + o = oct2int(head[p]) while j >= lsb: b = b + ((o>>j)&0x01,) j = j - 1 p = p + 1 - return self._createComponent(asn1Spec, tagSet, b), '' + return self._createComponent(asn1Spec, tagSet, b), tail r = self._createComponent(asn1Spec, tagSet, ()) - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun(substrate) + if substrateFun: + return substrateFun(r, substrate, length) + while head: + component, head = decodeFun(head) r = r + component - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -154,27 +167,29 @@ class BitStringDecoder(AbstractSimpleDecoder): class OctetStringDecoder(AbstractSimpleDecoder): protoComponent = univ.OctetString('') + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] if tagSet[0][1] == tag.tagFormatSimple: # XXX what tag to check? - return self._createComponent(asn1Spec, tagSet, substrate), '' + return self._createComponent(asn1Spec, tagSet, head), tail r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun(substrate) + if substrateFun: + return substrateFun(r, substrate, length) + while head: + component, head = decodeFun(head) r = r + component - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet, '') - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -186,93 +201,89 @@ class OctetStringDecoder(AbstractSimpleDecoder): class NullDecoder(AbstractSimpleDecoder): protoComponent = univ.Null('') def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) - if substrate: - raise error.PyAsn1Error('Unexpected substrate for Null') - return r, substrate + if head: + raise error.PyAsn1Error('Unexpected %d-octet substrate for Null' % length) + return r, tail class ObjectIdentifierDecoder(AbstractSimpleDecoder): protoComponent = univ.ObjectIdentifier(()) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: raise error.PyAsn1Error('Empty substrate') - oid = (); index = 0 - # Get the first subid - subId = oct2int(substrate[index]) - oid = oid + divmod(subId, 40) - index = index + 1 - substrateLen = len(substrate) - + # Get the first subid + subId = oct2int(head[0]) + oid = divmod(subId, 40) + + index = 1 + substrateLen = len(head) while index < substrateLen: - subId = oct2int(substrate[index]) - if subId < 128: - oid = oid + (subId,) - index = index + 1 - else: + subId = oct2int(head[index]) + index = index + 1 + if subId == 128: + # ASN.1 spec forbids leading zeros (0x80) in sub-ID OID + # encoding, tolerating it opens a vulnerability. + # See http://www.cosic.esat.kuleuven.be/publications/article-1432.pdf page 7 + raise error.PyAsn1Error('Invalid leading 0x80 in sub-OID') + elif subId > 128: # Construct subid from a number of octets nextSubId = subId subId = 0 - while nextSubId >= 128 and index < substrateLen: + while nextSubId >= 128: subId = (subId << 7) + (nextSubId & 0x7F) + if index >= substrateLen: + raise error.SubstrateUnderrunError( + 'Short substrate for sub-OID past %s' % (oid,) + ) + nextSubId = oct2int(head[index]) index = index + 1 - nextSubId = oct2int(substrate[index]) - if index == substrateLen: - raise error.SubstrateUnderrunError( - 'Short substrate for OID %s' % oid - ) subId = (subId << 7) + nextSubId - oid = oid + (subId,) - index = index + 1 - return self._createComponent(asn1Spec, tagSet, oid), substrate[index:] + oid = oid + (subId,) + return self._createComponent(asn1Spec, tagSet, oid), tail class RealDecoder(AbstractSimpleDecoder): protoComponent = univ.Real() def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] - if not length: - raise error.SubstrateUnderrunError('Short substrate for Real') - fo = oct2int(substrate[0]); substrate = substrate[1:] - if fo & 0x40: # infinite value - value = fo & 0x01 and '-inf' or 'inf' - elif fo & 0x80: # binary enoding - if fo & 0x11 == 0: - n = 1 - elif fo & 0x01: - n = 2 - elif fo & 0x02: - n = 3 - else: - n = oct2int(substrate[0]) - eo, substrate = substrate[:n], substrate[n:] - if not eo or not substrate: + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: + return self._createComponent(asn1Spec, tagSet, 0.0), tail + fo = oct2int(head[0]); head = head[1:] + if fo & 0x80: # binary enoding + n = (fo & 0x03) + 1 + if n == 4: + n = oct2int(head[0]) + eo, head = head[:n], head[n:] + if not eo or not head: raise error.PyAsn1Error('Real exponent screwed') - e = 0 + e = oct2int(eo[0]) & 0x80 and -1 or 0 while eo: # exponent e <<= 8 e |= oct2int(eo[0]) eo = eo[1:] p = 0 - while substrate: # value + while head: # value p <<= 8 - p |= oct2int(substrate[0]) - substrate = substrate[1:] + p |= oct2int(head[0]) + head = head[1:] if fo & 0x40: # sign bit p = -p value = (p, 2, e) + elif fo & 0x40: # infinite value + value = fo & 0x01 and '-inf' or 'inf' elif fo & 0xc0 == 0: # character encoding try: if fo & 0x3 == 0x1: # NR1 - value = (int(substrate), 10, 0) + value = (int(head), 10, 0) elif fo & 0x3 == 0x2: # NR2 - value = float(substrate) + value = float(head) elif fo & 0x3 == 0x3: # NR3 - value = float(substrate) + value = float(head) else: raise error.SubstrateUnderrunError( 'Unknown NR (tag %s)' % fo @@ -281,13 +292,11 @@ class RealDecoder(AbstractSimpleDecoder): raise error.SubstrateUnderrunError( 'Bad character Real syntax' ) - elif fo & 0xc0 == 0x40: # special real value - pass else: raise error.SubstrateUnderrunError( 'Unknown encoding (tag %s)' % fo ) - return self._createComponent(asn1Spec, tagSet, value), substrate + return self._createComponent(asn1Spec, tagSet, value), tail class SequenceDecoder(AbstractConstructedDecoder): protoComponent = univ.Sequence() @@ -301,17 +310,15 @@ class SequenceDecoder(AbstractConstructedDecoder): return r.getComponentPositionNearType(t, idx) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) idx = 0 - if not decodeFun: - return r, substrate - while substrate: + if substrateFun: + return substrateFun(r, substrate, length) + while head: asn1Spec = self._getComponentTagMap(r, idx) - component, substrate = decodeFun( - substrate, asn1Spec - ) + component, head = decodeFun(head, asn1Spec) idx = self._getComponentPositionByType( r, component.getEffectiveTagSet(), idx ) @@ -319,18 +326,19 @@ class SequenceDecoder(AbstractConstructedDecoder): idx = idx + 1 r.setDefaultComponents() r.verifySizeSpec() - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) idx = 0 while substrate: asn1Spec = self._getComponentTagMap(r, idx) - if not decodeFun: - return r, substrate component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break idx = self._getComponentPositionByType( r, component.getEffectiveTagSet(), idx @@ -348,32 +356,31 @@ class SequenceDecoder(AbstractConstructedDecoder): class SequenceOfDecoder(AbstractConstructedDecoder): protoComponent = univ.SequenceOf() def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) asn1Spec = r.getComponentType() idx = 0 - if not decodeFun: - return r, substrate - while substrate: - component, substrate = decodeFun( - substrate, asn1Spec - ) + while head: + component, head = decodeFun(head, asn1Spec) r.setComponentByPosition(idx, component, asn1Spec is None) idx = idx + 1 r.verifySizeSpec() - return r, substrate + return r, tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) asn1Spec = r.getComponentType() idx = 0 - if not decodeFun: - return r, substrate while substrate: component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r.setComponentByPosition(idx, component, asn1Spec is None) idx = idx + 1 @@ -401,43 +408,68 @@ class SetOfDecoder(SequenceOfDecoder): class ChoiceDecoder(AbstractConstructedDecoder): protoComponent = univ.Choice() + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): - substrate = substrate[:length] + length, state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] r = self._createComponent(asn1Spec, tagSet) - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) if r.getTagSet() == tagSet: # explicitly tagged Choice - component, substrate = decodeFun( - substrate, r.getComponentTagMap() + component, head = decodeFun( + head, r.getComponentTagMap() ) else: - component, substrate = decodeFun( - substrate, r.getComponentTagMap(), tagSet, length, state + component, head = decodeFun( + head, r.getComponentTagMap(), tagSet, length, state ) if isinstance(component, univ.Choice): effectiveTagSet = component.getEffectiveTagSet() else: effectiveTagSet = component.getTagSet() r.setComponentByType(effectiveTagSet, component, 0, asn1Spec is None) + return r, tail + + def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, + length, state, decodeFun, substrateFun): + r = self._createComponent(asn1Spec, tagSet) + if substrateFun: + return substrateFun(r, substrate, length) + if r.getTagSet() == tagSet: # explicitly tagged Choice + component, substrate = decodeFun(substrate, r.getComponentTagMap()) + eooMarker, substrate = decodeFun(substrate) # eat up EOO marker + if not eoo.endOfOctets.isSameTypeWith(eooMarker) or \ + eooMarker != eoo.endOfOctets: + raise error.PyAsn1Error('No EOO seen before substrate ends') + else: + component, substrate= decodeFun( + substrate, r.getComponentTagMap(), tagSet, length, state + ) + if isinstance(component, univ.Choice): + effectiveTagSet = component.getEffectiveTagSet() + else: + effectiveTagSet = component.getTagSet() + r.setComponentByType(effectiveTagSet, component, 0, asn1Spec is None) return r, substrate - indefLenValueDecoder = valueDecoder - class AnyDecoder(AbstractSimpleDecoder): protoComponent = univ.Any() + tagFormats = (tag.tagFormatSimple, tag.tagFormatConstructed) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): if asn1Spec is None or \ asn1Spec is not None and tagSet != asn1Spec.getTagSet(): # untagged Any container, recover inner header substrate length = length + len(fullSubstrate) - len(substrate) substrate = fullSubstrate - substrate = substrate[:length] - return self._createComponent(asn1Spec, tagSet, value=substrate), '' + if substrateFun: + return substrateFun(self._createComponent(asn1Spec, tagSet), + substrate, length) + head, tail = substrate[:length], substrate[length:] + return self._createComponent(asn1Spec, tagSet, value=head), tail def indefLenValueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, - length, state, decodeFun): + length, state, decodeFun, substrateFun): if asn1Spec is not None and tagSet == asn1Spec.getTagSet(): # tagged Any type -- consume header substrate header = '' @@ -450,11 +482,12 @@ class AnyDecoder(AbstractSimpleDecoder): # Any components do not inherit initial tag asn1Spec = self.protoComponent - if not decodeFun: - return r, substrate + if substrateFun: + return substrateFun(r, substrate, length) while substrate: component, substrate = decodeFun(substrate, asn1Spec) - if component == eoo.endOfOctets: + if eoo.endOfOctets.isSameTypeWith(component) and \ + component == eoo.endOfOctets: break r = r + component else: @@ -550,7 +583,10 @@ class Decoder: self.__tagSetCache = {} def __call__(self, substrate, asn1Spec=None, tagSet=None, - length=None, state=stDecodeTag, recursiveFlag=1): + length=None, state=stDecodeTag, recursiveFlag=1, + substrateFun=None): + if debug.logger & debug.flagDecoder: + debug.logger('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate))) fullSubstrate = substrate while state != stStop: if state == stDecodeTag: @@ -559,6 +595,9 @@ class Decoder: raise error.SubstrateUnderrunError( 'Short octet stream on tag decoding' ) + if not isOctetsType(substrate) and \ + not isinstance(substrate, univ.OctetString): + raise error.PyAsn1Error('Bad octet stream type') firstOctet = substrate[0] substrate = substrate[1:] @@ -598,6 +637,7 @@ class Decoder: else: tagSet = lastTag + tagSet state = stDecodeLength + debug.logger and debug.logger & debug.flagDecoder and debug.logger('tag decoded into %r, decoding length' % tagSet) if state == stDecodeLength: # Decode length if not substrate: @@ -625,12 +665,13 @@ class Decoder: for char in lengthString: length = (length << 8) | oct2int(char) size = size + 1 - state = stGetValueDecoder substrate = substrate[size:] if length != -1 and len(substrate) < length: raise error.SubstrateUnderrunError( '%d-octet short' % (length - len(substrate)) ) + state = stGetValueDecoder + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length]))) if state == stGetValueDecoder: if asn1Spec is None: state = stGetValueDecoderByTag @@ -669,14 +710,27 @@ class Decoder: state = stDecodeValue else: state = stTryAsExplicitTag + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('codec %s chosen by a built-in type, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as explicit tag')) + debug.scope.push(concreteDecoder is None and '?' or concreteDecoder.protoComponent.__class__.__name__) if state == stGetValueDecoderByAsn1Spec: if isinstance(asn1Spec, (dict, tagmap.TagMap)): if tagSet in asn1Spec: __chosenSpec = asn1Spec[tagSet] else: __chosenSpec = None + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('candidate ASN.1 spec is a map of:') + for t, v in asn1Spec.getPosMap().items(): + debug.logger(' %r -> %s' % (t, v.__class__.__name__)) + if asn1Spec.getNegMap(): + debug.logger('but neither of: ') + for i in asn1Spec.getNegMap().items(): + debug.logger(' %r -> %s' % (t, v.__class__.__name__)) + debug.logger('new candidate ASN.1 spec is %s, chosen by %r' % (__chosenSpec is None and '<none>' or __chosenSpec.__class__.__name__, tagSet)) else: __chosenSpec = asn1Spec + debug.logger and debug.logger & debug.flagDecoder and debug.logger('candidate ASN.1 spec is %s' % asn1Spec.__class__.__name__) if __chosenSpec is not None and ( tagSet == __chosenSpec.getTagSet() or \ tagSet in __chosenSpec.getTagMap() @@ -687,9 +741,11 @@ class Decoder: __chosenSpec.typeId in self.__typeMap: # ambiguous type concreteDecoder = self.__typeMap[__chosenSpec.typeId] + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value decoder chosen for an ambiguous type by type ID %s' % (__chosenSpec.typeId,)) elif baseTagSet in self.__tagMap: # base type or tagged subtype concreteDecoder = self.__tagMap[baseTagSet] + debug.logger and debug.logger & debug.flagDecoder and debug.logger('value decoder chosen by base %r' % (baseTagSet,)) else: concreteDecoder = None if concreteDecoder: @@ -700,8 +756,13 @@ class Decoder: elif tagSet == self.__endOfOctetsTagSet: concreteDecoder = self.__tagMap[tagSet] state = stDecodeValue + debug.logger and debug.logger & debug.flagDecoder and debug.logger('end-of-octets found') else: + concreteDecoder = None state = stTryAsExplicitTag + if debug.logger and debug.logger & debug.flagDecoder: + debug.logger('codec %s chosen by ASN.1 spec, decoding %s' % (state == stDecodeValue and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as explicit tag')) + debug.scope.push(__chosenSpec is None and '?' or __chosenSpec.__class__.__name__) if state == stTryAsExplicitTag: if tagSet and \ tagSet[0][1] == tag.tagFormatConstructed and \ @@ -710,34 +771,35 @@ class Decoder: concreteDecoder = explicitTagDecoder state = stDecodeValue else: + concreteDecoder = None state = self.defaultErrorState + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s chosen, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "<none>", state == stDecodeValue and 'value' or 'as failure')) if state == stDumpRawValue: concreteDecoder = self.defaultRawDecoder + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s chosen, decoding value' % concreteDecoder.__class__.__name__) state = stDecodeValue if state == stDecodeValue: - if recursiveFlag: - decodeFun = self - else: - decodeFun = None + if recursiveFlag == 0 and not substrateFun: # legacy + substrateFun = lambda a,b,c: (a,b[:c]) if length == -1: # indef length value, substrate = concreteDecoder.indefLenValueDecoder( fullSubstrate, substrate, asn1Spec, tagSet, length, - stGetValueDecoder, decodeFun + stGetValueDecoder, self, substrateFun ) else: - value, _substrate = concreteDecoder.valueDecoder( + value, substrate = concreteDecoder.valueDecoder( fullSubstrate, substrate, asn1Spec, tagSet, length, - stGetValueDecoder, decodeFun + stGetValueDecoder, self, substrateFun ) - if recursiveFlag: - substrate = substrate[length:] - else: - substrate = _substrate state = stStop + debug.logger and debug.logger & debug.flagDecoder and debug.logger('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, value.prettyPrint(), substrate and debug.hexdump(substrate) or '<none>')) if state == stErrorCondition: raise error.PyAsn1Error( '%r not in asn1Spec: %r' % (tagSet, asn1Spec) ) + if debug.logger and debug.logger & debug.flagDecoder: + debug.scope.pop() + debug.logger('decoder left scope %s, call completed' % debug.scope) return value, substrate decode = Decoder(tagMap, typeMap) diff --git a/libs/pyasn1/codec/ber/encoder.py b/libs/pyasn1/codec/ber/encoder.py index 2149b0ba..173949d0 100644 --- a/libs/pyasn1/codec/ber/encoder.py +++ b/libs/pyasn1/codec/ber/encoder.py @@ -1,8 +1,8 @@ # BER encoder from pyasn1.type import base, tag, univ, char, useful from pyasn1.codec.ber import eoo -from pyasn1.compat.octets import int2oct, ints2octs, null, str2octs -from pyasn1 import error +from pyasn1.compat.octets import int2oct, oct2int, ints2octs, null, str2octs +from pyasn1 import debug, error class Error(Exception): pass @@ -78,9 +78,24 @@ class ExplicitlyTaggedItemEncoder(AbstractItemEncoder): explicitlyTaggedItemEncoder = ExplicitlyTaggedItemEncoder() +class BooleanEncoder(AbstractItemEncoder): + supportIndefLenMode = 0 + _true = ints2octs((1,)) + _false = ints2octs((0,)) + def encodeValue(self, encodeFun, value, defMode, maxChunkSize): + return value and self._true or self._false, 0 + class IntegerEncoder(AbstractItemEncoder): supportIndefLenMode = 0 + supportCompactZero = False def encodeValue(self, encodeFun, value, defMode, maxChunkSize): + if value == 0: # shortcut for zero value + if self.supportCompactZero: + # this seems to be a correct way for encoding zeros + return null, 0 + else: + # this seems to be a widespread way for encoding zeros + return ints2octs((0,)), 0 octets = [] value = int(value) # to save on ops on asn1 type while 1: @@ -149,18 +164,15 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): index = 5 else: if len(oid) < 2: - raise error.PyAsn1Error('Short OID %s' % value) + raise error.PyAsn1Error('Short OID %s' % (value,)) # Build the first twos - index = 0 - subid = oid[index] * 40 - subid = subid + oid[index+1] - if subid < 0 or subid > 0xff: + if oid[0] > 6 or oid[1] > 39 or oid[0] == 6 and oid[1] > 15: raise error.PyAsn1Error( - 'Initial sub-ID overflow %s in OID %s' % (oid[index:], value) + 'Initial sub-ID overflow %s in OID %s' % (oid[:2], value) ) - octets = (subid,) - index = index + 2 + octets = (oid[0] * 40 + oid[1],) + index = 2 # Cycle through subids for subid in oid[index:]: @@ -184,6 +196,7 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): return ints2octs(octets), 0 class RealEncoder(AbstractItemEncoder): + supportIndefLenMode = 0 def encodeValue(self, encodeFun, value, defMode, maxChunkSize): if value.isPlusInfinity(): return int2oct(0x40), 0 @@ -206,9 +219,11 @@ class RealEncoder(AbstractItemEncoder): m >>= 1 e += 1 eo = null - while e: + while e not in (0, -1): eo = int2oct(e&0xff) + eo e >>= 8 + if e == 0 and eo and oct2int(eo[0]) & 0x80: + eo = int2oct(0) + eo n = len(eo) if n > 0xff: raise error.PyAsn1Error('Real exponent overflow') @@ -268,7 +283,7 @@ class AnyEncoder(OctetStringEncoder): tagMap = { eoo.endOfOctets.tagSet: EndOfOctetsEncoder(), - univ.Boolean.tagSet: IntegerEncoder(), + univ.Boolean.tagSet: BooleanEncoder(), univ.Integer.tagSet: IntegerEncoder(), univ.BitString.tagSet: BitStringEncoder(), univ.OctetString.tagSet: OctetStringEncoder(), @@ -313,6 +328,7 @@ class Encoder: self.__typeMap = typeMap def __call__(self, value, defMode=1, maxChunkSize=0): + debug.logger & debug.flagEncoder and debug.logger('encoder called in %sdef mode, chunk size %s for type %s, value:\n%s' % (not defMode and 'in' or '', maxChunkSize, value.__class__.__name__, value.prettyPrint())) tagSet = value.getTagSet() if len(tagSet) > 1: concreteEncoder = explicitlyTaggedItemEncoder @@ -322,13 +338,16 @@ class Encoder: elif tagSet in self.__tagMap: concreteEncoder = self.__tagMap[tagSet] else: - baseTagSet = value.baseTagSet - if baseTagSet in self.__tagMap: - concreteEncoder = self.__tagMap[baseTagSet] + tagSet = value.baseTagSet + if tagSet in self.__tagMap: + concreteEncoder = self.__tagMap[tagSet] else: - raise Error('No encoder for %s' % value) - return concreteEncoder.encode( + raise Error('No encoder for %s' % (value,)) + debug.logger & debug.flagEncoder and debug.logger('using value codec %s chosen by %r' % (concreteEncoder.__class__.__name__, tagSet)) + substrate = concreteEncoder.encode( self, value, defMode, maxChunkSize ) + debug.logger & debug.flagEncoder and debug.logger('built %s octets of substrate: %s\nencoder completed' % (len(substrate), debug.hexdump(substrate))) + return substrate encode = Encoder(tagMap, typeMap) diff --git a/libs/pyasn1/codec/cer/__init__.py b/libs/pyasn1/codec/cer/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/cer/__init__.py +++ b/libs/pyasn1/codec/cer/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/cer/decoder.py b/libs/pyasn1/codec/cer/decoder.py index 71395d22..9fd37c13 100644 --- a/libs/pyasn1/codec/cer/decoder.py +++ b/libs/pyasn1/codec/cer/decoder.py @@ -7,22 +7,25 @@ from pyasn1 import error class BooleanDecoder(decoder.AbstractSimpleDecoder): protoComponent = univ.Boolean(0) def valueDecoder(self, fullSubstrate, substrate, asn1Spec, tagSet, length, - state, decodeFun): - substrate = substrate[:length] - if not substrate: + state, decodeFun, substrateFun): + head, tail = substrate[:length], substrate[length:] + if not head: raise error.PyAsn1Error('Empty substrate') - byte = oct2int(substrate[0]) + byte = oct2int(head[0]) + # CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while + # BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1 + # in http://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf if byte == 0xff: value = 1 elif byte == 0x00: value = 0 else: raise error.PyAsn1Error('Boolean CER violation: %s' % byte) - return self._createComponent(asn1Spec, tagSet, value), substrate[1:] + return self._createComponent(asn1Spec, tagSet, value), tail tagMap = decoder.tagMap.copy() tagMap.update({ - univ.Boolean.tagSet: BooleanDecoder(), + univ.Boolean.tagSet: BooleanDecoder() }) typeMap = decoder.typeMap diff --git a/libs/pyasn1/codec/der/__init__.py b/libs/pyasn1/codec/der/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/codec/der/__init__.py +++ b/libs/pyasn1/codec/der/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/codec/der/decoder.py b/libs/pyasn1/codec/der/decoder.py index 0f5a24ca..604abec2 100644 --- a/libs/pyasn1/codec/der/decoder.py +++ b/libs/pyasn1/codec/der/decoder.py @@ -2,4 +2,8 @@ from pyasn1.type import univ from pyasn1.codec.cer import decoder -decode = decoder.Decoder(decoder.tagMap, decoder.typeMap) +tagMap = decoder.tagMap +typeMap = decoder.typeMap +Decoder = decoder.Decoder + +decode = Decoder(tagMap, typeMap) diff --git a/libs/pyasn1/compat/__init__.py b/libs/pyasn1/compat/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/compat/__init__.py +++ b/libs/pyasn1/compat/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/compat/octets.py b/libs/pyasn1/compat/octets.py index d0303eaa..f7f2a29b 100644 --- a/libs/pyasn1/compat/octets.py +++ b/libs/pyasn1/compat/octets.py @@ -8,6 +8,7 @@ if version_info[0] <= 2: octs2ints = lambda s: [ oct2int(x) for x in s ] str2octs = lambda x: x octs2str = lambda x: x + isOctetsType = lambda s: isinstance(s, str) else: ints2octs = bytes int2oct = lambda x: ints2octs((x,)) @@ -16,3 +17,4 @@ else: octs2ints = lambda s: [ x for x in s ] str2octs = lambda x: x.encode() octs2str = lambda x: x.decode() + isOctetsType = lambda s: isinstance(s, bytes) diff --git a/libs/pyasn1/debug.py b/libs/pyasn1/debug.py new file mode 100644 index 00000000..c27cb1d4 --- /dev/null +++ b/libs/pyasn1/debug.py @@ -0,0 +1,65 @@ +import sys +from pyasn1.compat.octets import octs2ints +from pyasn1 import error +from pyasn1 import __version__ + +flagNone = 0x0000 +flagEncoder = 0x0001 +flagDecoder = 0x0002 +flagAll = 0xffff + +flagMap = { + 'encoder': flagEncoder, + 'decoder': flagDecoder, + 'all': flagAll + } + +class Debug: + defaultPrinter = sys.stderr.write + def __init__(self, *flags): + self._flags = flagNone + self._printer = self.defaultPrinter + self('running pyasn1 version %s' % __version__) + for f in flags: + if f not in flagMap: + raise error.PyAsn1Error('bad debug flag %s' % (f,)) + self._flags = self._flags | flagMap[f] + self('debug category \'%s\' enabled' % f) + + def __str__(self): + return 'logger %s, flags %x' % (self._printer, self._flags) + + def __call__(self, msg): + self._printer('DBG: %s\n' % msg) + + def __and__(self, flag): + return self._flags & flag + + def __rand__(self, flag): + return flag & self._flags + +logger = 0 + +def setLogger(l): + global logger + logger = l + +def hexdump(octets): + return ' '.join( + [ '%s%.2X' % (n%16 == 0 and ('\n%.5d: ' % n) or '', x) + for n,x in zip(range(len(octets)), octs2ints(octets)) ] + ) + +class Scope: + def __init__(self): + self._list = [] + + def __str__(self): return '.'.join(self._list) + + def push(self, token): + self._list.append(token) + + def pop(self): + return self._list.pop() + +scope = Scope() diff --git a/libs/pyasn1/type/__init__.py b/libs/pyasn1/type/__init__.py index e69de29b..8c3066b2 100644 --- a/libs/pyasn1/type/__init__.py +++ b/libs/pyasn1/type/__init__.py @@ -0,0 +1 @@ +# This file is necessary to make this directory a package. diff --git a/libs/pyasn1/type/base.py b/libs/pyasn1/type/base.py index db31671e..40873719 100644 --- a/libs/pyasn1/type/base.py +++ b/libs/pyasn1/type/base.py @@ -120,7 +120,12 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): def prettyIn(self, value): return value def prettyOut(self, value): return str(value) - def prettyPrint(self, scope=0): return self.prettyOut(self._value) + def prettyPrint(self, scope=0): + if self._value is noValue: + return '<no value>' + else: + return self.prettyOut(self._value) + # XXX Compatibility stub def prettyPrinter(self, scope=0): return self.prettyPrint(scope) diff --git a/libs/pyasn1/type/namedtype.py b/libs/pyasn1/type/namedtype.py index aa9c5678..48967a5f 100644 --- a/libs/pyasn1/type/namedtype.py +++ b/libs/pyasn1/type/namedtype.py @@ -60,12 +60,12 @@ class NamedTypes: tagMap = self.__namedTypes[idx].getType().getTagMap() for t in tagMap.getPosMap(): if t in self.__tagToPosIdx: - raise error.PyAsn1Error('Duplicate type %s' % t) + raise error.PyAsn1Error('Duplicate type %s' % (t,)) self.__tagToPosIdx[t] = idx try: return self.__tagToPosIdx[tagSet] except KeyError: - raise error.PyAsn1Error('Type %s not found' % tagSet) + raise error.PyAsn1Error('Type %s not found' % (tagSet,)) def getNameByPosition(self, idx): try: @@ -79,12 +79,12 @@ class NamedTypes: idx = idx - 1 n = self.__namedTypes[idx].getName() if n in self.__nameToPosIdx: - raise error.PyAsn1Error('Duplicate name %s' % n) + raise error.PyAsn1Error('Duplicate name %s' % (n,)) self.__nameToPosIdx[n] = idx try: return self.__nameToPosIdx[name] except KeyError: - raise error.PyAsn1Error('Name %s not found' % name) + raise error.PyAsn1Error('Name %s not found' % (name,)) def __buildAmbigiousTagMap(self): ambigiousTypes = () diff --git a/libs/pyasn1/type/namedval.py b/libs/pyasn1/type/namedval.py index 815e2d42..d0fea7cc 100644 --- a/libs/pyasn1/type/namedval.py +++ b/libs/pyasn1/type/namedval.py @@ -15,10 +15,10 @@ class NamedValues: name = namedValue val = automaticVal if name in self.nameToValIdx: - raise error.PyAsn1Error('Duplicate name %s' % name) + raise error.PyAsn1Error('Duplicate name %s' % (name,)) self.nameToValIdx[name] = val if val in self.valToNameIdx: - raise error.PyAsn1Error('Duplicate value %s' % name) + raise error.PyAsn1Error('Duplicate value %s=%s' % (name, val)) self.valToNameIdx[val] = name self.namedValues = self.namedValues + ((name, val),) automaticVal = automaticVal + 1 diff --git a/libs/pyasn1/type/tag.py b/libs/pyasn1/type/tag.py index 0cf67ebd..1144907f 100644 --- a/libs/pyasn1/type/tag.py +++ b/libs/pyasn1/type/tag.py @@ -18,7 +18,7 @@ class Tag: def __init__(self, tagClass, tagFormat, tagId): if tagId < 0: raise error.PyAsn1Error( - 'Negative tag ID (%s) not allowed' % tagId + 'Negative tag ID (%s) not allowed' % (tagId,) ) self.__tag = (tagClass, tagFormat, tagId) self.uniq = (tagClass, tagId) diff --git a/libs/pyasn1/type/tagmap.py b/libs/pyasn1/type/tagmap.py index 53e1791a..7cec3a10 100644 --- a/libs/pyasn1/type/tagmap.py +++ b/libs/pyasn1/type/tagmap.py @@ -28,7 +28,7 @@ class TagMap: def clone(self, parentType, tagMap, uniq=False): if self.__defType is not None and tagMap.getDef() is not None: - raise error.PyAsn1Error('Duplicate default value at %s' % self) + raise error.PyAsn1Error('Duplicate default value at %s' % (self,)) if tagMap.getDef() is not None: defType = tagMap.getDef() else: @@ -37,7 +37,7 @@ class TagMap: posMap = self.__posMap.copy() for k in tagMap.getPosMap(): if uniq and k in posMap: - raise error.PyAsn1Error('Duplicate positive key %s' % k) + raise error.PyAsn1Error('Duplicate positive key %s' % (k,)) posMap[k] = parentType negMap = self.__negMap.copy() diff --git a/libs/pyasn1/type/univ.py b/libs/pyasn1/type/univ.py index cb4f49b7..9cd16f8a 100644 --- a/libs/pyasn1/type/univ.py +++ b/libs/pyasn1/type/univ.py @@ -69,13 +69,18 @@ class Integer(base.AbstractSimpleAsn1Item): def prettyIn(self, value): if not isinstance(value, str): - return int(value) + try: + return int(value) + except: + raise error.PyAsn1Error( + 'Can\'t coerce %s into integer: %s' % (value, sys.exc_info()[1]) + ) r = self.__namedValues.getValue(value) if r is not None: return r try: return int(value) - except ValueError: + except: raise error.PyAsn1Error( 'Can\'t coerce %s into integer: %s' % (value, sys.exc_info()[1]) ) @@ -224,14 +229,14 @@ class BitString(base.AbstractSimpleAsn1Item): return tuple(r) else: raise error.PyAsn1Error( - 'Bad BIT STRING value notation %s' % value + 'Bad BIT STRING value notation %s' % (value,) ) else: for i in value.split(','): j = self.__namedValues.getValue(i) if j is None: raise error.PyAsn1Error( - 'Unknown bit identifier \'%s\'' % i + 'Unknown bit identifier \'%s\'' % (i,) ) if j >= len(r): r.extend([0]*(j-len(r)+1)) @@ -528,7 +533,7 @@ class Real(base.AbstractSimpleAsn1Item): ) if value[1] not in (2, 10): raise error.PyAsn1Error( - 'Prohibited base for Real value: %s' % value[1] + 'Prohibited base for Real value: %s' % (value[1],) ) if value[1] == 10: value = self.__normalizeBase10(value) @@ -648,7 +653,7 @@ class SetOf(base.AbstractConstructedAsn1Item): def _verifyComponent(self, idx, value): if self._componentType is not None and \ not self._componentType.isSuperTypeOf(value): - raise error.PyAsn1Error('Component type error %s' % value) + raise error.PyAsn1Error('Component type error %s' % (value,)) def getComponentByPosition(self, idx): return self._componentValues[idx] def setComponentByPosition(self, idx, value=None, verifyConstraints=True): @@ -924,9 +929,9 @@ class Choice(Set): return self._componentValues[self._currentIdx] >= other return NotImplemented if sys.version_info[0] <= 2: - def __nonzero__(self, other): return bool(self._componentValues) + def __nonzero__(self): return bool(self._componentValues) else: - def __bool__(self, other): return bool(self._componentValues) + def __bool__(self): return bool(self._componentValues) def __len__(self): return self._currentIdx is not None and 1 or 0 diff --git a/libs/pyutil/_version.py b/libs/pyutil/_version.py index 617d2205..376b2b9b 100644 --- a/libs/pyutil/_version.py +++ b/libs/pyutil/_version.py @@ -6,7 +6,7 @@ # pyutil.version_class for a description of what the different fields mean. __pkgname__ = "pyutil" -verstr = "1.9.3" +verstr = "1.9.7" try: from pyutil.version_class import Version as pyutil_Version __version__ = pyutil_Version(verstr) diff --git a/libs/pyutil/benchutil.py b/libs/pyutil/benchutil.py index 3e773a63..6c286346 100644 --- a/libs/pyutil/benchutil.py +++ b/libs/pyutil/benchutil.py @@ -1,4 +1,4 @@ -# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn +# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn # This file is part of pyutil; see README.rst for licensing terms. """ @@ -21,10 +21,10 @@ the second, e.g.: >>> rep_bench(fib, 25, UNITS_PER_SECOND=1000) best: 1.968e+00, 3th-best: 1.987e+00, mean: 2.118e+00, 3th-worst: 2.175e+00, worst: 2.503e+00 (of 10) -The output is reporting the number of milliseconds that executing the function -took, divided by N, from ten different invocations of fib(). It reports the -best, worst, M-th best, M-th worst, and mean, where "M" is the natural log of -the number of invocations (in this case 10). +The output is reporting the number of milliseconds that executing the +function took, divided by N, from ten different invocations of +fib(). It reports the best, worst, M-th best, M-th worst, and mean, +where "M" is 1/4 of the number of invocations (in this case 10). 2. Now run it with different values of N and look for patterns: @@ -74,10 +74,12 @@ and the main function is to make them be methods of the same object, e.g.: 4. Things to fix: - a. I used to have it hooked up to use the "hotshot" profiler on the code being - measured. I recently tried to change it to use the newer cProfile profiler - instead, but I don't understand the interface to cProfiler so it just gives an - exception if you pass profile=True. Please fix this and send me a patch. + a. I used to have it hooked up to use the "hotshot" profiler on the + code being measured. I recently tried to change it to use the newer + cProfile profiler instead, but I don't understand the interface to + cProfiler so it just gives an exception if you pass + profile=True. Please fix this and send me a patch. xxx change it to + statprof b. Wouldn't it be great if this script emitted results in a json format that was understood by a tool to make pretty interactive explorable graphs? The @@ -122,7 +124,7 @@ def mult(a, b): except TypeError: return to_decimal(a) * to_decimal(b) -def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): +def rep_bench(func, n, runtime=1.0, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ Will run the func up to MAXREPS times, but won't start a new run if MAXTIME (wall-clock time) has already elapsed (unless MAXTIME is None). @@ -130,33 +132,43 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p @param quiet Don't print anything--just return the results dict. """ assert isinstance(n, int), (n, type(n)) + global worstemptymeasure + emsta = clock() + do_nothing(2**32) + emstop = clock() + empty = emstop - emsta + if empty > worstemptymeasure: + worstemptymeasure = empty + if (worstemptymeasure*2) >= runtime: + raise BadMeasure("Apparently simply invoking an empty Python function can take as long as %0.10f seconds, and we were running iterations for only about %0.10f seconds. So the measurement of the runtime of the code under benchmark is not reliable. Please pass a higher number for the 'runtime' argument to bench_it().") + startwallclocktime = time.time() - tls = [] # elapsed time in seconds + tls = [] # (elapsed time per iter in seconds, iters) bmes = [] while ((len(tls) < MAXREPS) or (MAXREPS is None)) and ((MAXTIME is None) or ((time.time() - startwallclocktime) < MAXTIME)): if initfunc: initfunc(n) try: - tl = bench_it(func, n, profile=profile, profresults=profresults) + tl, iters = bench_it(func, n, runtime=runtime, profile=profile, profresults=profresults) except BadMeasure, bme: bmes.append(bme) else: - tls.append(tl) + tls.append((tl, iters)) if len(tls) == 0: raise Exception("Couldn't get any measurements within time limits or number-of-attempts limits. Maybe something is wrong with your clock? %s" % (bmes,)) - sumtls = reduce(operator.__add__, tls) + sumtls = sum([tl for (tl, iters) in tls]) mean = sumtls / len(tls) tls.sort() - worst = tls[-1] - best = tls[0] - _assert(best > worstemptymeasure*MARGINOFERROR, "%s(n=%s) took %0.10f seconds, but we cannot measure times much less than about %0.10f seconds. Try a more time-consuming variant (such as higher n)." % (func, n, best, worstemptymeasure*MARGINOFERROR,)) + worst = tls[-1][0] + best = tls[0][0] + m = len(tls)/4 if m > 0: - mthbest = tls[m-1] - mthworst = tls[-m] + mthbest = tls[m-1][0] + mthworst = tls[-m][0] else: - mthbest = tls[0] - mthworst = tls[-1] + mthbest = tls[0][0] + mthworst = tls[-1][0] # The +/-0 index is the best/worst, the +/-1 index is the 2nd-best/worst, # etc, so we use mp1 to name it. @@ -196,26 +208,22 @@ class BadMeasure(Exception): def do_nothing(n): pass -def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"): +def bench_it(func, n, runtime=1.0, profile=False, profresults="pyutil-benchutil.prof"): if profile: - st = clock() - cProfile.run('func(n)', profresults) - sto = clock() + raise NotImplementedException() else: + iters = 0 st = clock() - func(n) + deadline = st + runtime sto = clock() + while sto < deadline: + func(n) + iters += 1 + sto = clock() timeelapsed = sto - st - if timeelapsed <= 0: - raise BadMeasure(timeelapsed) - global worstemptymeasure - emsta = clock() - do_nothing(2**32) - emstop = clock() - empty = emstop - emsta - if empty > worstemptymeasure: - worstemptymeasure = empty - return timeelapsed + if (timeelapsed <= 0) or (iters == 0): + raise BadMeasure((timeelapsed, iters)) + return (timeelapsed / iters, iters) def bench(func, initfunc=None, TOPXP=21, MAXREPS=5, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", outputjson=False, jsonresultsfname="pyutil-benchutil-results.json", UNITS_PER_SECOND=1): BSIZES = [] diff --git a/libs/pyutil/benchutil.py~ b/libs/pyutil/benchutil.py~ index 3ec323eb..a33111e8 100644 --- a/libs/pyutil/benchutil.py~ +++ b/libs/pyutil/benchutil.py~ @@ -1,4 +1,4 @@ -# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn +# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn # This file is part of pyutil; see README.rst for licensing terms. """ @@ -104,6 +104,24 @@ def makeg(func): func() return blah +def to_decimal(x): + """ + See if D(x) returns something. If instead it raises TypeError, x must have been a float, so convert it to Decimal by way of string. (In Python >= 2.7, D(x) does this automatically. + """ + try: + return D(x) + except TypeError: + return D("%0.54f" % (x,)) + +def mult(a, b): + """ + If we get TypeError from * (possibly because one is float and the other is Decimal), then promote them both to Decimal. + """ + try: + return a * b + except TypeError: + return to_decimal(a) * to_decimal(b) + def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ Will run the func up to MAXREPS times, but won't start a new run if MAXTIME @@ -144,12 +162,12 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p # etc, so we use mp1 to name it. mp1 = m+1 res = { - 'worst': (worst*UNITS_PER_SECOND)/n, - 'best': (best*UNITS_PER_SECOND)/n, + 'worst': mult(worst, UNITS_PER_SECOND)/n, + 'best': mult(best, UNITS_PER_SECOND)/n, 'mp1': mp1, - 'mth-best': (mthbest*UNITS_PER_SECOND)/n, - 'mth-worst': (mthworst*UNITS_PER_SECOND)/n, - 'mean': (mean*UNITS_PER_SECOND)/n, + 'mth-best': mult(mthbest, UNITS_PER_SECOND)/n, + 'mth-worst': mult(mthworst, UNITS_PER_SECOND)/n, + 'mean': mult(mean, UNITS_PER_SECOND)/n, 'num': len(tls), } @@ -178,7 +196,10 @@ class BadMeasure(Exception): def do_nothing(n): pass -def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"): +def bench_it(func, n, runtime=0.1, profile=False, profresults="pyutil-benchutil.prof"): + """ + runtime is how many seconds to + """ if profile: st = clock() cProfile.run('func(n)', profresults) diff --git a/libs/pyutil/data/wordlist.txt b/libs/pyutil/data/wordlist.txt new file mode 100644 index 00000000..e1048b99 --- /dev/null +++ b/libs/pyutil/data/wordlist.txt @@ -0,0 +1,7248 @@ +fawn +yellow +four +prefix +payoff +scold +outwit +lore +lord +swivel +deli +pigment +foul +fur +disturb +prize +broiler +wooden +satchel +crotch +fritter +charter +tired +miller +bacon +second +tether +ruthless +thunder +fossil +succumb +cull +specialist +hero +avert +herb +splinter +here +herd +china +dogwood +cult +shriek +chink +pancreas +robin +neurologist +climber +diplomat +golden +gridiron +lengthen +summons +remnant +stern +unit +spoke +exhort +statesmanship +music +bedrock +passport +strike +teaspoon +relay +relax +hurt +meteorologist +glass +hurl +hole +hold +unpack +sweeten +blade +locker +locket +plunger +wand +wane +unjust +household +digit +malign +caution +want +rayon +hog +hoe +travel +copious +cutback +revisit +how +hot +hop +cheetah +diagram +possum +modest +antonym +pigtail +revolt +alias +decoy +wing +squint +wine +feedback +misdemeanor +kickoff +foodstuff +butcher +dreamer +fir +bowlder +fix +fib +fig +fin +undercut +enrich +slate +interrupt +sixteen +silver +scholar +thyme +seamstress +debut +arrow +debug +volcano +burial +whim +concord +knockout +garment +allah +spider +crocus +turnip +yiddish +fortnight +allay +whir +whip +diction +smirk +mason +semiconductor +re +adapt +outburst +knit +scruff +silicon +miaow +thumbtack +shopper +wasp +wash +instruct +rhododendron +tango +master +architect +bitter +listen +wisdom +swish +sulphur +crawl +trek +peril +outlay +coward +tree +shower +pneumonia +sheen +acclaim +entail +girder +runner +spectrum +headland +increment +quay +dozen +kidnap +gripe +hum +greenback +tipi +matriarch +stirrup +object +toil +microsecond +mouth +addict +letter +fluster +drought +thriller +expound +singer +upend +grove +professor +camp +detriment +nineteenth +scream +marvel +bomb +reactor +heckler +ulcer +caper +layout +menu +bust +cougar +bush +bliss +rich +mend +rice +plate +pocket +cushion +fetish +relish +jaguar +boarder +pretzel +patch +hasten +respond +fair +heirloom +radium +radius +result +fail +crouch +clef +best +irk +yogurt +ire +wage +extend +vestment +souvenir +extent +wheelbarrow +carbon +debt +roller +accident +trickster +veer +disdain +cup +logic +genus +rehash +gopher +canyon +bewilder +chrome +onomatopoeia +advert +grapefruit +stadium +jackass +counterattack +life +retrospect +spit +worker +wish +lift +toboggan +chile +child +chili +spin +wildcat +dissect +employ +calcium +delicatessen +locksmith +letdown +player +elicit +eighteen +violin +doorman +specter +hone +toaster +honk +rebellion +split +bid +european +typhoid +boiler +ownership +supper +tuna +tune +furlough +noblewoman +unhook +abound +bellow +beset +plight +brandish +previous +ham +hag +hay +prison +falter +east +hat +quirk +birth +shadow +gangplank +remind +pavement +battlefield +attorney +right +old +creek +crowd +creed +crown +glove +billboard +creep +chorus +okra +bottom +circumvent +inhuman +fox +foe +fog +binder +yoke +slither +recollect +despair +rebut +eightieth +sob +sod +overshadow +honeymoon +overgrow +sop +sow +wrap +fabric +panorama +support +tame +avail +width +hothead +call +overhand +overhang +telegraph +offer +thesaurus +beech +squalid +safeguard +otter +duel +misinform +paprika +vanguard +pest +duet +proud +tournament +proven +exist +quintuplet +dealer +leer +floor +glacier +actor +flood +role +entomologist +sunset +smell +leek +intend +glutton +ointment +asterisk +taurus +intent +cleaver +entrust +windscreen +puss +lowdown +time +push +gown +chain +viaduct +skate +chair +midst +millisecond +ballet +uneven +vex +crater +oversight +jerk +ameba +embark +flora +mourn +knapsack +southpaw +exact +epic +judaism +tear +teas +teat +crustacean +subway +team +skewer +prevent +meadow +gremlin +attic +sigh +milligram +heavyweight +crescent +playpen +crackpot +melt +current +boost +abscond +gnaw +splice +address +brilliant +endow +queue +influx +love +radish +prefer +piranha +fake +instal +forefront +sky +homesick +turret +wicker +wicket +scope +prosecutor +wicked +afford +refrain +visual +appendix +behalf +mascot +lumberjack +pretend +descriptor +dispossess +stole +winter +savor +sputter +meddler +slush +spot +textual +date +suck +dove +pulley +stress +conscious +bluster +wheelchair +quadrant +mango +so +skirmish +truce +drunken +archeologist +footstep +yearn +jig +disconnect +thumb +accordion +nearsighted +councillor +hubbub +suspicion +thump +apron +civilian +insomnia +nation +amulet +twilight +ketchup +handiwork +revert +fisherman +quarter +quartet +receipt +fireproof +breakthrough +sponsor +troll +naked +canvas +onrush +trauma +formula +dumbfound +million +envelop +vicious +disrespect +mime +plea +byte +workmanship +punk +wrong +ostrich +punt +footwear +neglect +gunshot +potter +one +reopen +chide +conifer +vote +paleontologist +languish +boulevard +wrath +convent +bite +extortion +shiver +draft +cite +starfish +shawl +artifact +snatch +antic +boyfriend +iceberg +rival +stammer +counselor +janitor +prospect +sac +greyhound +argument +alley +sad +say +borough +saw +handicraft +tulip +general +knead +zoo +note +take +destroy +printer +buffer +squalor +compress +buffet +crochet +knee +byway +lawn +enamel +blockhead +sale +cocoanut +wind +axe +salt +cobra +homespun +lotus +friction +bright +slot +slow +slop +unkind +gourd +transact +cloak +debunk +slog +hockey +slob +robe +clank +dissimilar +psychiatrist +clang +outlet +prime +artist +saliva +borrow +soloist +carrion +handcuff +primp +landlord +tortilla +where +xmas +vision +gout +gangster +cheesecloth +diver +bugler +mutton +plummet +bootleg +teacup +bureau +mope +vender +jumper +spars +screen +dome +supermarket +adept +jovial +spare +spark +quack +oust +fit +madcap +mane +flipper +backpack +twin +boar +supervisor +extinct +twig +boat +companionship +stretch +west +breath +reflex +gist +thousand +photon +cloudburst +turtleneck +former +jute +scarlet +straighten +spotlight +girth +brow +canon +dubious +monk +blab +fame +spunk +breakdown +hideaway +deft +barber +disown +booster +driftwood +veal +pewter +dimension +scholarship +summer +manifold +poach +disconcert +slime +rest +invalid +alarmist +mandolin +instrument +overthrow +stopwatch +haystack +joyful +sportsmanship +rejoin +dart +dark +brazier +snarl +traffic +cranium +vacuum +world +snare +dare +clan +clam +stranger +shutter +glamor +clay +claw +inter +kennel +clap +auditorium +obstruct +grub +potion +lobster +racial +endeavor +tote +tube +moslem +tuba +nook +exit +refer +zest +ration +leadership +standpoint +stone +ace +slender +meal +tumor +neighbor +act +mean +invert +braggart +homeless +wade +hypnotist +racquet +hew +burglar +her +gleam +glean +mindless +harpsichord +italic +hem +hen +defrost +epilog +pull +regimen +darken +wafer +rage +hooray +tripe +ruse +flirt +reprimand +whiz +torso +pulp +rust +ohm +gong +ad +fright +certain +epaulet +catchup +hoodlum +ay +ax +tranquil +jargon +slobber +cream +yoga +collector +abolish +tight +backgammon +congress +annex +slant +midget +brotherhood +slang +rostrum +neuter +thorn +groom +mask +kilogram +mash +mimic +mast +mass +ringworm +waiver +retch +gingham +influenza +consider +neigh +upkeep +taxicab +tinsel +to +tail +smile +norm +debit +baton +candid +salesperson +cobalt +strand +laud +pedant +sand +adjust +small +mammal +peon +ninetieth +plaid +past +burnish +gossip +canvass +healer +hick +offbeat +clock +section +succinct +method +contrast +full +hash +lobbyist +saleswoman +dramatist +backlash +brutal +prior +hamster +skyrocket +social +action +welder +raze +depart +vie +sherbet +regiment +captor +coercion +entrap +select +casket +enliven +petroleum +maltreat +pearl +sitter +morn +ballad +more +teen +teem +door +tester +signpost +nomad +doom +cunning +fatal +malt +chisel +patriarch +knocker +midstream +mall +learn +grope +male +stewardess +prompt +taunt +gallop +scab +accept +autumn +gallon +scar +rustler +condemn +huge +speedboat +fruition +cling +clink +plant +anoint +blotter +variant +unsound +plane +waver +flutter +pucker +wrench +trellis +patio +pant +instep +trade +paper +pang +brim +mislay +hearsay +buttercup +epoch +coarsen +bypass +motley +sucker +gadget +consign +imperil +skipper +harrow +nugget +fount +found +lantern +status +eyelash +clockwork +scribe +penicillin +lipstick +research +highway +bungler +belief +porcelain +bedlam +cockpit +loafer +suntan +acorn +riser +reproach +prefab +drivel +sicken +bumper +testament +clump +major +purport +limerick +number +feeder +slipper +footprint +florist +glitter +guess +guest +jet +swipe +vocalist +saint +gnash +relationship +tightwad +typhoon +mural +consult +grace +frock +getaway +vocal +video +defect +waft +pedestrian +graffito +caress +blond +gasket +sell +ballerina +ragamuffin +tarnish +spaghetti +self +trowel +poplar +brace +bobbin +kneecap +hypochondriac +blackboard +nasal +twine +raucous +virus +plan +wive +foyer +oyster +unequal +arson +covet +cover +barren +barrel +bulletin +chowder +golf +cruiser +affix +session +freight +impact +condor +writer +peculiar +condom +factor +downpour +dandelion +streamer +resent +actress +compass +banner +tumult +sojourn +caramel +enema +weaver +river +outlaw +prospectus +manger +set +creator +overwhelm +jade +sex +see +sea +contour +analog +project +urchin +fission +crossword +pickup +crosswalk +kneel +candor +mildew +hardship +disallow +incident +dividend +pagan +scatterbrain +lass +last +thou +opal +feminist +amoeba +lash +whole +load +loaf +electrician +pendulum +bell +loam +loan +hollow +scallop +church +psychoanalyst +underlay +napalm +airfield +devil +filth +imbed +proprietor +veneer +firm +sweetheart +champion +fire +infect +upstart +fund +deport +hostess +straight +budget +error +outskirt +real +pound +moth +vow +chasm +vanish +chase +starlight +seek +shorten +wasteland +specimen +commune +snail +teeter +cigar +epithet +alert +opinion +stack +recent +expend +clime +person +sixtieth +crayfish +telegram +aroma +belabor +amp +demerit +sandal +goblet +chest +eager +horseradish +homeland +wrongdoer +input +limp +cordon +format +bureaucrat +quest +cataclysm +blackjack +falcon +abduct +flannel +spine +consensus +crescendo +spring +beckon +palm +pall +sight +curious +sprint +battalion +pale +gruel +benefit +religion +be +odor +agreement +carol +by +scepter +coexist +hatchet +sacrament +ambush +biennial +repair +contributor +next +span +sock +submit +custom +spay +suit +spar +spat +blueprint +perplex +poster +lint +slump +pastor +overbear +link +atom +line +up +slander +foist +hornet +insignia +genial +aerial +nationalist +haunt +char +chap +chat +parsec +breather +phantom +paradox +tuft +uranium +scrape +parakeet +swirl +freighter +tart +tedium +scapegoat +trouser +scrub +gardenia +hackney +lane +land +fighter +algorithm +scotch +age +feud +summit +walker +fresh +crowbar +rescuer +hello +essay +code +partial +serviceman +scratch +broaden +totem +soften +leggin +renown +prim +flashback +young +send +moor +tremor +garret +armament +garden +quadruplet +llama +precinct +wipe +magic +harbor +eve +anxious +race +rack +mishap +crook +croon +odd +ode +victor +index +yock +sauerkraut +apparatus +indian +proffer +bird +inspect +leg +punch +acquit +let +fifteen +vinegar +great +casino +screech +scatter +survey +insulin +grandchild +buss +popcorn +mussel +maker +grower +sire +disobey +causal +zip +archbishop +theme +aeon +eleven +doubt +yardstick +midday +pencil +babe +shipwreck +patrol +rubbish +central +hoard +pout +pour +thin +drill +coffin +cherubim +bent +pawn +process +lock +slim +high +slit +bend +slip +pelvis +martyr +trumpet +weaken +rhubarb +delay +blackhead +luster +stow +halter +singular +await +wristwatch +notebook +tier +marrow +hawk +autograph +tomato +counter +robot +element +writ +allot +allow +alloy +thigh +mute +insight +spatula +comma +mutt +warren +perfect +decay +shudder +garnet +derelict +prosper +python +belch +bat +launder +dock +snake +kiss +bar +cage +wrangler +truth +scorch +subset +bump +static +thirteen +mete +jagged +disco +tenth +wander +matrix +bag +fatherland +venom +czar +oblong +lob +shut +perish +tempo +graze +tempt +shun +embarrass +minstrel +chilli +mainland +spill +length +stickler +scare +scarf +manuscript +scene +cobweb +owner +scent +prank +lop +opossum +sergeant +spaceship +painful +stomach +chagrin +vouch +rotunda +haven +steel +wet +bother +aggressor +psalm +disband +unman +steep +torrent +misunderstand +beggar +viewer +partnership +correspond +tonight +receptionist +fourteenth +mischief +depict +soak +bacterium +bassoon +hammer +adjunct +lilt +soap +soar +calculus +manor +raindrop +cipher +vise +segment +fervent +instil +locust +enlist +soprano +fiasco +brew +fact +bring +brine +bedroom +rough +asylum +trivial +brink +redirect +disillusion +planter +jay +jaw +jar +jam +tape +bourbon +flinch +hope +jackpot +move +familiar +scorn +sinus +wring +antagonist +smash +shaver +summon +stuff +rein +withstand +pronoun +packer +frame +packet +bellhop +airmail +dungeon +wire +mien +partisan +unravel +piston +pistol +email +browbeat +fetich +physicist +courtyard +lawsuit +tantrum +drum +quitter +ramp +drug +doorway +puff +roughen +medallion +revamp +migrant +distil +javelin +indict +chromium +lectern +mailman +gondola +quaver +blatant +feather +ballast +sheepish +crisscross +federalist +mannequin +altruism +banish +laser +runway +bathtub +maul +groin +ripe +lush +site +lust +mockingbird +tenor +passbook +ransom +tattoo +inquest +terrorist +buffoon +outbreak +android +balm +ball +balk +dusk +fiesta +bale +bald +dust +broccoli +mosaic +audit +off +shotgun +polyp +command +diphtheria +audio +maggot +compel +glut +glue +rambler +web +generous +clergyman +wee +wed +arrest +crack +scoundrel +government +chancellor +crux +haul +cedar +desk +password +recurs +placenta +crisp +onion +resin +alkali +stagger +imprison +nymph +sprain +overcast +foray +habitat +thief +daylight +flush +wisecrack +ballot +transport +henchman +disbelief +hoarder +avoid +disk +doer +passion +saucepan +stairway +putt +drift +ornithologist +stage +iris +sister +adverb +peal +ingest +union +artefact +parsley +assess +lung +mere +muck +commission +caviar +watchman +stamina +much +function +funnel +cosmopolitan +frisk +shyness +grate +rectum +count +congresswoman +smooth +monument +problem +baptism +cordial +kowtow +sirloin +retina +inn +replica +ink +anesthesia +furl +sexual +saviour +behold +reckless +chum +monday +repeal +veil +vein +ghost +eon +rule +dynamo +torrid +pension +tryout +abhor +buoy +inning +dote +rapid +mansion +defraud +voter +spew +bludgeon +bike +daze +regal +chill +regalia +whack +ale +compassion +blanket +distort +mania +chauvinist +chapel +whisk +daydream +pinch +scalar +handout +roadblock +unblock +math +triumph +chew +paperback +phoney +speck +heliport +sabbath +horn +chef +aristocrat +panda +stardom +lizard +walkout +toll +crunch +dustpan +pursuit +paraffin +sorceress +hairdo +daughter +envoy +adopt +tankard +smoke +loincloth +bunker +anarchist +envious +sanatorium +infield +spigot +thrust +hindsight +total +bra +plot +plow +plop +sweater +gloss +ploy +insult +plod +knoll +beeswax +solicit +award +yard +tariff +overrun +word +err +crest +work +grovel +tinder +era +elbow +spendthrift +quiver +serpent +flunk +impair +liter +chameleon +sever +moron +disappoint +beach +pizza +fever +lad +ladder +lag +lab +lay +law +arch +cosmonaut +retort +greet +greek +green +south +worst +order +greed +salon +gumption +devote +muffin +misconduct +mayor +sheaf +avocado +valor +carton +shear +then +fragment +safe +break +band +bang +coffer +overprint +tzar +bank +bread +crock +gallows +lisp +iguana +schemer +transient +prawn +sled +flock +slew +hostel +burlap +network +diesel +fellowship +amethyst +marigold +barrier +veto +standard +stencil +lollipop +morass +drench +ticket +maniac +raisin +flawless +renew +sprig +regress +vanquish +kin +render +system +hamstring +chopper +disembark +comic +overs +neck +upshot +tourniquet +kiwi +emblem +luncheon +cereal +rebuff +minibus +guild +target +tavern +hike +medley +iota +guilt +iron +minus +pessimist +lull +ponder +strength +realm +widen +silversmith +latter +hamper +transmit +curfew +maiden +boxcar +sue +negro +phase +proverb +grave +deacon +swamp +bracket +aunt +rickshaw +oppress +mitten +crust +boyhood +nephew +toast +geyser +layman +geologist +predecessor +do +ardor +ecologist +roundabout +slack +rebirth +runt +rune +rung +crucifixion +steak +steal +steam +ghoul +reread +misdirect +christian +goulash +pastel +gentleman +cellist +contraband +drawl +accord +unfold +kitchen +cop +cot +cow +brat +excrement +ill +cob +brag +cod +cog +bran +coo +con +emporium +eyesight +tone +spear +royal +trunk +nonconformist +infirm +speak +charisma +scarecrow +warmth +leech +baud +hacksaw +millionth +hoist +spellbind +gracious +physician +inhibit +gnu +launcher +air +aim +ail +abrupt +thrash +aid +stink +have +sticker +sting +throat +brake +cone +hebrew +uplift +stint +descent +perform +descend +decibel +wheel +raid +fuss +nil +swell +hang +evil +hand +fuse +nip +nit +scenario +drip +ragged +client +mamma +kinship +indigo +photo +victim +extol +thyroid +exalt +shout +cognac +board +zillion +righteous +plasma +intercom +fusion +boxer +cape +retreat +cooler +night +flatter +born +rile +flatten +bore +orchid +cede +humor +peek +peel +pose +confer +peer +peep +chafe +foreskin +chaff +diner +coral +visa +banker +horizon +cherish +gingerbread +octopus +croak +faint +dilemma +tetanus +float +profession +bound +curios +sedan +loin +beet +piggyback +wag +bookend +wad +frill +sovereign +fight +gybe +way +wax +burro +war +fizz +peninsula +holdup +boon +true +reset +absent +nursemaid +smidgeon +maximum +crystal +veterinarian +emir +emit +aorta +flat +abstract +molt +flaw +postscript +subsist +prayer +cacao +face +mold +mole +stake +shrine +test +upholster +unwilling +frolic +shrink +heyday +chairperson +hemophilia +faze +affidavit +loyal +longshoreman +igloo +concept +matron +consul +fulcrum +datum +horseback +supplement +toothpick +varnish +grape +zone +mallet +flask +graph +hump +flash +manicurist +glad +rhythm +tusk +terror +idealist +southwest +brown +congest +kitten +blast +brows +ophthalmologist +gun +gum +gut +guy +diarrhoea +upper +brave +regret +bravo +thinker +cost +helpless +tempest +cargo +appear +economist +menthol +medal +havoc +uniform +tarantula +appeal +caveat +genes +gawk +jester +disclaim +goldfish +teacher +buck +merriment +fogey +precursor +plotter +eke +disavow +trial +convertor +pillow +bolt +extra +paragon +keeper +marker +firearm +market +streetcar +prove +subvert +live +matador +club +cluck +clue +logarithm +prepay +graphic +slogan +car +cap +caw +cat +meow +can +cab +heart +hears +chip +sake +bridesmaid +abort +chin +chic +serum +bankrupt +freezer +write +lobe +storeroom +criterion +entrench +afternoon +product +dive +southern +bawl +motorway +pave +drastic +flourish +crepe +grandson +explicit +offend +barnyard +forfeit +haircut +ledger +brain +nitrogen +cold +braid +ethic +willow +theorem +window +artisan +factual +tiara +gizzard +nought +halt +fling +nod +rake +overcrowd +dishearten +hale +half +recap +courtship +taillight +provision +discuss +halo +wont +concerto +servant +drop +domain +supplant +year +operand +wavelength +happen +album +accomplish +space +thirst +rational +thong +carp +cart +virtuoso +quart +rebel +marina +prospector +card +care +fungus +tomahawk +british +honest +nonprofit +profess +blind +madam +blink +rink +rind +ring +drove +tomorrow +size +sheep +sheer +sheet +silent +bookmark +breed +callous +traction +checker +tragic +heartburn +friend +pomp +courier +that +peck +scalpel +rugged +recruit +magnesium +optimist +extinguish +angel +slay +slat +premier +slap +racetrack +slam +anger +breakfast +recover +slab +upbeat +veteran +shore +snout +siesta +begin +prick +halibut +price +foothold +dream +tooth +aerosol +washcloth +fifth +ground +gnat +snack +ratio +stair +proportion +jolt +stain +juror +shrill +pumpkin +cannon +loath +stroll +leather +thermal +husband +druggist +concert +burst +spore +whitewash +unfit +staunch +sport +incisor +concern +crawfish +glaze +complexion +import +clench +pluck +blame +broil +impromptu +whisker +guffaw +pertain +priestess +temper +aura +comet +evict +adroit +dispatch +exploit +semicolon +lioness +harmless +rebuild +toss +sisterhood +textbook +bloodhound +crumb +these +trick +scum +cherub +fool +marksman +zenith +mucus +soil +agnostic +laggard +bias +eras +bestial +beaver +waterway +petunia +helium +develop +media +pester +poetic +document +sweeper +finish +thunderbolt +foal +foam +cymbal +fruit +volley +trawler +smelt +quartz +theater +framework +patchwork +demean +taxi +livestock +fester +battleship +typhus +neon +touch +speed +death +refurbish +treatment +baloney +momentum +lade +ream +hover +frown +spectacular +larva +read +ruler +swig +leapfrog +earnest +detract +stunt +execution +reap +hovel +rear +postcard +incest +roll +engross +oblivion +output +downward +falsehood +laugh +verbal +landslid +squirm +garland +putter +cleanser +deficit +squirt +wheat +sadden +throb +sixth +tuition +strainer +bazaar +throw +comparison +placard +hiatus +chop +fell +wolf +parson +chow +ruff +assassin +processor +heater +outlook +earring +watchdog +your +restless +stare +grater +log +area +start +stealth +low +lot +wigwam +groan +pitcher +rump +posterior +recoil +omelet +lymph +thaw +corduroy +wholes +hire +fraud +default +bucket +draftsman +cornmeal +gibberish +charlatan +scanner +pickax +sheath +mover +antenna +housework +valid +colt +you +houseboat +poor +polar +poop +peat +pear +peas +overreact +podium +peak +pool +fiscal +assert +moonlight +repay +forswear +mourner +skeleton +breadth +groundwork +angler +month +unrest +bequeath +carpet +corps +gymnast +foster +spearhead +fervor +fountain +washroom +horror +verb +minded +heaven +ceaseless +saboteur +tendril +blackout +smock +homonym +resound +exodus +casual +bomber +protractor +lurch +milk +turmoil +vet +excess +strong +arena +divisor +noticeboard +outgrowth +vegetarian +whine +soldier +amount +base +trainer +put +haemoglobin +seventeenth +taker +helper +pup +titbit +assessor +chestnut +yuck +suction +grill +nine +parasol +transcend +pusher +boycott +archipelago +tract +phrase +magenta +frostbit +sheikh +warhead +spreadsheet +lyre +snippet +reject +gash +circuit +rude +sneak +denial +gasp +reelect +undergo +figment +overnight +ego +dread +egg +lynch +earthworm +help +reservoir +slouch +auburn +reclaim +preempt +soot +helm +hell +clarinet +limelight +prowler +lateral +heron +astonish +forbear +dim +food +musket +terrain +vomit +foot +stopper +holiday +payer +twelfth +bless +radial +trailer +pamper +thirteenth +talisman +event +magnet +vertigo +wedlock +teak +publish +eardrum +sustain +shrivel +outrun +ass +pun +drink +bass +dirt +dung +dune +cornstarch +reason +heroin +ask +ash +turnout +bask +bash +pus +launch +curtsey +round +bridegroom +caption +liaison +heartbeat +blush +assign +arsenal +demagog +elder +effortless +mist +miss +blossom +minion +station +expand +fingernail +scheme +banana +merciless +slosh +behind +bowel +trapper +boredom +sign +leotard +tunic +lament +ouster +hyaena +bride +bobsled +currant +toward +weapon +phobia +chipmunk +yowl +null +sensual +lid +lie +koala +cave +lip +useless +honeycomb +popular +quota +plunder +mace +watt +smoulder +clear +cleat +succor +clean +skein +blend +humid +phenomenon +cowgirl +flicker +sheik +crayon +copyright +paranoid +scooter +less +ikon +ampersand +waterproof +custodian +outlast +strut +strum +basement +chimney +monogram +fluff +chasten +geld +courteous +cramp +backtrack +grey +close +despatch +bandwagon +haddock +aqueduct +wow +grocer +won +woodchuck +wok +woe +stalk +bettor +wreath +philanthropist +spray +distinguish +zipper +garrison +delimit +eggplant +buzz +vault +protector +mausoleum +onward +oversleep +liken +proton +header +badminton +vessel +catwalk +stamp +damp +nape +damn +threaten +dame +alto +liven +exempt +deter +liver +hobnob +furrow +pact +loom +utmost +look +socialist +governor +rope +pace +while +smart +fleet +loot +loop +pack +petal +hoax +grant +belong +makeshift +discredit +grand +conflict +sham +hallway +optic +dime +bonus +banter +overweight +user +boa +grind +auditor +five +ambassador +chore +abstain +bearer +morsel +tick +botch +pier +carat +march +albino +game +jibe +banjo +optician +signal +manifest +eel +sleigh +sketch +creation +undress +yolk +urgent +impoverish +mustang +clothespin +fundamentalist +gild +simmer +slash +slapstick +run +rum +rub +booklet +benefactor +rug +stem +step +stew +taboo +subtract +rut +discus +shine +faith +pigpen +letup +portico +reappear +aye +congressman +block +foreswear +misbehavior +dude +within +gilt +pentagon +connector +syllabus +palomino +harem +frost +reed +womankind +reef +reek +reel +dull +skulk +swagger +chiropractor +ringlet +foresight +similar +psychopath +kidney +straitjacket +nab +sullen +nag +objection +obelisk +nap +department +nay +draw +resign +drag +tundra +drab +formal +horseplay +outing +orbit +depth +bribe +pinion +underbrush +cheerful +go +emboss +dunk +ammonia +compact +aquarium +baron +aria +stave +ameer +shack +geranium +warpath +epitaph +velour +schooner +virtuous +fond +wave +trough +cellular +tenet +nausea +stiff +asparagus +gender +button +hive +verdict +cloister +pilfer +picket +blitz +jump +booth +ardent +languor +cartel +click +poke +wallet +colonel +valet +cell +rotten +experiment +stanch +brooch +fifteenth +quell +weirdo +convert +chant +gent +repel +behead +ricksha +wig +daybreak +danger +win +clout +wit +ligament +infest +gimmick +wiz +cloud +metaphor +snoop +copperhead +crag +crab +cram +expressway +compatriot +mismatch +starter +salad +consort +ride +donut +archer +meet +control +wharf +halloween +glade +skirt +bandana +filament +circular +fare +farm +thunderstorm +canker +foment +corral +scoop +encyclopedia +scoot +agenda +american +cadet +sperm +gunman +hock +brood +broom +brook +walnut +youngster +skater +frond +auto +dike +relentless +snorkel +placid +stout +hands +front +refuel +muff +perfectionist +mode +upward +commonwealth +unwind +chunk +mollusc +seesaw +apartheid +mollusk +special +gallant +armor +confess +fathom +remiss +activist +playground +wick +obsess +umbrella +hopscotch +watermark +jilt +undo +advisor +sneer +princess +shrew +timer +keep +counterpart +elector +keel +mad +blogger +seventeen +bitch +drinker +equinox +dump +wrapper +chintz +attach +attack +jellyfish +final +beard +introvert +punish +feint +noun +plough +piecework +waist +photograph +spurn +cartoonist +beg +bed +bee +discolor +swindler +firework +spurt +bet +are +exhibit +fume +tabu +torment +sundown +portrait +need +border +rotor +bastard +sprinkler +gunner +jaunt +tactic +truck +detector +visor +brand +african +camper +rigor +awe +plumber +eject +spleen +urn +upset +snapshot +businessman +constrain +skunk +affair +indoor +crate +molest +cavort +sheriff +fiberglass +winner +wreak +rash +earner +rasp +gradual +fuel +sulfur +joint +fallout +buyer +endless +gray +tobacco +gust +ordain +topaz +nutmeg +she +contain +recoup +grab +conduct +widow +hardwood +shake +orphan +portend +computer +driveway +equip +portent +unearth +southward +tend +state +lug +tens +antler +tent +bleed +castoff +blinker +keg +bemoan +key +overhaul +thank +sniff +career +admit +spatter +plankton +jersey +christen +tuesday +poem +sari +tread +shaikh +yap +cent +quiz +yam +treat +yak +whisper +poet +fibber +spaniel +nuptial +brunch +debtor +novel +ripen +pandemonium +harden +neuron +hearten +steer +generic +balloon +speaker +northwest +blight +fireman +flesh +absorb +powwow +inbreed +spree +magician +rift +weld +surfboard +lunch +glower +well +drone +welt +underdog +discord +mistaken +dose +distant +laurel +skill +cinder +jackal +dais +ovum +snicker +stratum +possess +warrant +homework +canter +rafter +fate +burden +loss +clown +tablespoons +lose +divest +satirist +rote +page +backlog +shed +glare +twitter +hush +redress +home +peter +competitor +pinpoint +overlay +broad +overlap +hinder +individualist +journal +offset +instinct +smidgen +refuge +freedom +cleans +nightclub +rodeo +dominion +wallop +buzzard +cocoa +pointless +gerbil +snowplow +mastermind +museum +poinsettia +drumstick +mohair +jinx +backhand +cricket +north +gait +admonish +neutral +ho +technician +overflow +ear +eat +he +leper +limit +cello +display +wringer +twist +entreat +contest +meteor +finch +chemist +fodder +star +stay +stag +foil +stab +phosphorus +appoint +sphinx +shunt +broncho +atheist +atheism +portion +pardon +mackerel +demand +unfurl +protest +asian +captain +gamut +swab +swan +sinew +swat +swap +anticlimax +sway +loon +appal +void +vase +smack +govern +affect +hitch +vast +pilgrim +naturalist +vector +washout +whirlwind +quilt +crave +yack +cactus +quill +pander +wreck +orchestra +bikini +spokesman +haze +new +net +maverick +seventh +mew +onus +cardboard +interpret +taper +credit +harass +jamb +permit +prolog +menial +hunch +campaign +bayonet +moral +handlebar +overhear +ore +overheat +calk +overhead +calm +intrust +recommend +type +tell +calf +demon +wart +warp +warn +dogma +warm +pecan +adult +qualm +flotilla +ward +blindfold +confound +rook +room +candlestick +worth +bungalow +headway +root +defer +vodka +give +climax +assent +honey +surveyor +quail +freshen +polio +rib +stockyard +answer +abdomen +plank +coup +fracas +passageway +waterfront +lesbian +guerilla +attempt +third +maintain +capitalist +fetus +deck +keyboard +windshield +furlong +harmonica +crew +better +persist +pass +workout +microfilm +caterpillar +grammar +meat +mistrust +roast +side +bone +luck +caustic +aids +dawn +extract +jell +contend +velvet +gradient +open +crucial +content +reader +linear +whiff +bestow +mistress +needlework +steward +fleck +loud +skinflint +playwright +grade +hoop +hoot +buttock +hook +ditch +hoof +hood +hydrant +acquaint +spinach +historian +enthral +woodwind +brainwash +dwell +inferno +twister +gym +somewhat +gambler +symptom +preacher +affront +keyword +matter +loiter +mink +seep +quench +modern +mind +mine +ginger +seed +seem +churn +mint +unfasten +alibi +desist +chess +sleeper +quarterback +phonograph +chatterbox +regular +condominium +blacklist +don +alarm +impostor +dog +doe +solemn +digress +constrict +consumer +dot +hunger +visitor +probe +syntax +chord +sundial +northeast +explain +jailer +sugar +folder +inventor +edict +patter +smut +stop +coast +pincushion +watermelon +smug +earl +earn +peacock +bay +reload +bad +troop +cower +ban +stinger +linguist +enchant +attest +subject +snuff +scrap +sail +causeway +scram +baboon +warrior +triplet +vitamin +lotion +orthodontist +beautician +cousin +motto +sprinter +pate +typist +height +gusher +aftermath +arsenic +ether +accustom +tint +recur +three +erect +ting +chrysanthemum +trigger +interest +basic +basil +basin +idol +chug +mushroom +suppress +dismiss +deepen +encyclopaedia +unpopular +tank +affirm +tang +near +moratorium +neat +motorist +anchor +spawn +seven +cane +diaphragm +it +shame +jest +in +id +disappear +if +abscess +growl +sap +parish +make +kit +delight +squat +garlic +warden +unicorn +jealous +overt +bequest +kid +butter +romp +smoker +inherit +bedspread +diphthong +left +protocol +just +yen +unfair +psych +human +yes +cretin +yew +legion +character +wretch +save +opt +discreet +background +shoulder +nude +manual +pillar +dean +squander +deal +deaf +maxim +dead +revel +intern +dear +strife +sprawl +pail +collect +normal +councilor +flounder +bullfight +tartar +bold +statistician +burn +blackmail +cottontail +sift +protagonist +burp +burr +tartan +super +innuendo +crucifix +craftsman +commit +marshal +unsay +paunch +chimp +down +lieu +chime +initial +lampoon +editor +fraction +unseat +fork +undergrowth +form +fore +ford +diaper +overburden +analyst +fort +pavilion +whiskey +boomerang +cosmos +propaganda +shin +disciplinarian +classic +covert +sidestep +drive +gland +scrawl +fatherhood +ship +graft +vista +marxist +smidgin +excel +handed +venison +congeal +marxism +sling +faction +handicap +slink +felt +diet +parenthood +journey +reign +stoke +weekend +derail +billion +potato +jacket +gorilla +almanac +teeth +meander +befriend +proletariat +woodwork +skip +skit +invent +adjourn +mild +mile +skim +skin +mill +primer +proletarian +skid +surplus +seasick +misread +depend +swoon +father +countdown +deject +swoop +regatta +unburden +string +yeast +pathologist +merit +join +jettison +stiffen +hoorah +din +stapler +nectar +die +dig +democrat +noiseless +item +dip +blur +shave +thresher +villa +worm +slake +sunup +talker +fillet +suspect +drunkard +shoo +dwarf +dweller +wail +guardian +clerk +makeup +stallion +waif +detest +tangent +deceit +rue +wait +box +boy +cuckoo +shift +bow +dither +boo +raccoon +cyst +bob +nylon +bog +elect +plumb +kayak +surmount +transplant +saki +wealth +perk +visit +vineyard +somersault +sharpen +yoghurt +aspirin +labyrinth +curriculum +downtown +tandem +rigid +savior +effort +gnome +demolish +pageant +moccasin +melodrama +flu +soul +impel +soup +sour +claim +plaza +reflector +predict +agent +drawer +council +craze +pink +purr +arbor +tilt +clever +parch +pine +till +sunday +sword +tile +pathway +pint +map +mar +mat +may +gelatin +membership +mankind +tablecloth +grow +man +relinquish +aimless +hemlock +omen +tale +switch +jail +deposit +talc +unleash +basket +longhand +talk +shield +rabbi +moralist +lyric +pitch +solder +pointer +group +monitor +bedbug +maid +drummer +maim +mail +main +tonic +killer +shatter +minuet +safari +teller +rattler +outweigh +feverish +peasant +careless +rock +hijack +eyelid +latin +bookshop +poker +gavel +unlock +manifesto +girl +stitch +monolog +priest +dutch +blubber +sensor +correct +monster +zombi +vinyl +jaywalk +cough +underwear +waiter +buzzer +thing +registrar +blacksmith +think +frequent +first +lone +crib +long +extrovert +thoughtless +lap +autocrat +escort +daunt +mermaid +anus +yoghourt +memo +broadcast +butt +proofread +tractor +coconut +lick +piccolo +marijuana +dash +comedian +sulk +nazi +sherbert +stopgap +daredevil +acumen +squad +interior +channel +pain +trace +roster +track +acrid +zigzag +whizz +assault +billow +pair +synonym +napkin +typeset +scowl +voodoo +toucan +amir +shop +lexicon +shot +show +cornea +veranda +therapist +shoe +threshold +corner +label +cornet +enthusiast +fend +objector +flapjack +dice +plume +enough +syphon +black +consent +enthusiasm +fiendish +plump +get +straggler +stomp +midriff +slyness +gee +gibber +neckerchief +gem +disinherit +beseech +skull +businesswoman +yield +stupid +nostril +tallow +kernel +sear +eighteenth +seat +seam +seal +stigma +calendar +wonder +puma +parent +limber +ornament +forecast +gage +pump +august +foreword +slingshot +tacit +wednesday +gauntlet +childbirth +tug +tuck +trader +tour +tout +delirium +stretcher +cancer +spank +cancel +tub +mare +underworld +imp +undershirt +mark +mart +workshop +rancher +fiftieth +chalet +graveyard +squash +wake +sound +litterbug +epidermis +slumber +cock +strait +strain +sudden +protein +par +pat +harsh +paw +pay +woodland +same +heartbreak +pad +cotton +pal +pan +exhaust +oil +chloroform +munch +companion +foghorn +polygon +drain +vertebra +soundproof +outdoor +suitor +money +imprint +leeway +aspect +flavor +asthma +godchild +comradeship +forgo +pile +pill +grip +grit +mop +mow +moo +mom +mob +railroad +grim +grin +oxygen +server +chamber +nose +hallelujah +fulfil +sneaker +afflict +witchcraft +ascend +dole +ascent +spasm +gross +confirm +pioneer +inject +gladden +highbrow +linoleum +intravenous +knife +raincoat +broker +squall +bravado +racoon +opium +contagion +roar +island +insect +mixer +thrive +partizan +road +checkup +dagger +coupon +splint +empress +whiten +strip +uptown +skillet +paraphernalia +jigsaw +totalitarian +madden +tycoon +tripod +striker +shroud +hiccup +gore +spice +ember +magnolia +grouch +conqueror +embed +deadlock +affection +deer +deep +fellow +planetarium +deem +file +girlfriend +deed +hound +film +fill +tedious +selfish +personnel +hybrid +repent +drouth +field +prism +astronaut +fruitless +lapel +shelter +gander +unload +burrow +god +gangway +oral +motel +represent +forger +pheasant +forget +founder +suburban +dollar +rebind +zinc +implement +crimson +hideous +premium +parcel +straightforward +scout +scour +fall +bottleneck +pueblo +hinterland +dampen +dictatorship +flyover +neighborhood +clinch +gnarl +burger +zero +cottonwood +lawyer +further +misrepresent +ribbon +dial +skeptic +stool +trinket +stoop +plush +movement +girlhood +malaria +intrench +twang +mule +ranger +beacon +bigamist +capacitor +search +stupor +margin +airport +chipper +chieftain +narrow +fatten +quotient +wizard +caravan +transit +sadist +sadism +establish +dachshund +hobgoblin +eye +score +distinct +two +splash +libel +furor +wiper +diamond +brisk +opportunist +particular +disfavor +nineteen +town +hour +cluster +fast +dew +remain +paragraph +den +abandon +stubborn +shark +buttress +onslaught +share +sphere +minimum +rainstorm +attain +junket +sharp +botanist +siren +awkward +comfort +rapport +stir +bleat +whopper +blacken +blood +bloom +chute +coax +orchard +coat +doctor +spiteful +electron +blunder +mislead +coal +sect +infant +setback +radar +dough +lava +suffer +hundredth +sodium +bosom +late +speech +clamor +lath +lookout +goof +good +goon +detour +frigid +compound +detach +complain +bombard +headroom +countersign +token +monsoon +clamp +harm +hark +mental +hare +hard +beret +banquet +connect +fist +callus +hart +orient +harp +flower +creditor +trooper +pigeon +seaport +granola +print +foreground +assist +cockroach +pleasant +gig +faucet +prophet +omit +wither +pure +corkscrew +copper +perturb +barbarian +shoal +cups +jabber +razor +construct +paint +leash +statement +mama +hummingbird +catapult +pare +park +selector +glycerin +dentist +part +pars +youth +totter +plead +hangout +cistern +blanch +mountain +cardigan +couch +onset +build +zucchini +flute +salmon +chart +most +charm +moss +eskimo +organist +humanitarian +mammoth +pennant +squelch +weigh +standoff +sector +sparrow +fine +find +giant +merger +nervous +ruin +fiend +boulder +prowess +paperweight +cholera +express +ferret +cheapen +batter +breast +theft +silk +pellet +restart +silo +huff +common +archaeologist +printout +vine +lion +overeat +tender +expert +burner +myriad +stowaway +subscript +hypochondria +premiss +egotist +complement +figurehead +mailbox +pagoda +aircraft +sultan +archway +annual +foreign +point +smother +newborn +pamphlet +dancer +esophagus +platinum +pocketbook +secret +amnesia +reformat +finalist +ram +gay +gas +gap +holler +gal +understand +gag +chatter +gab +bile +politician +metro +solid +bill +holocaust +crutch +fun +lingo +manner +mystic +astound +rancor +eczema +ingrain +anaesthesia +sociologist +dishonor +ewe +seminar +corridor +neutron +itch +leopard +yesterday +moment +stripe +unveil +timid +task +werewolf +withdraw +landmark +grid +recant +spend +howl +darn +shape +snot +timber +rundown +impetus +cut +cur +pollster +snag +forbid +cue +punter +cub +snap +bridal +easter +brainstorm +bin +squawk +rebound +bib +judgement +redeem +bit +knock +disrepair +blemish +flue +fagot +flux +bamboo +foolish +walrus +sequin +transgress +often +back +impeach +extremist +mirror +lightning +scald +scale +culprit +pet +pelt +pew +pep +pen +scalp +lard +lark +peg +pea +patient +fed +megaton +constraint +oatmeal +drama +catnip +pediatrician +offshoot +obstetrician +gambit +maelstrom +tiff +clack +lesson +jockey +few +doll +errand +camera +handbook +forward +nougat +sideshow +showman +switchboard +calico +lifeguard +planet +jumbo +azalea +constant +flow +possessor +lye +curd +cure +curb +curl +prevail +stagecoach +leaflet +crypt +underweight +cellar +lend +tablespoon +papa +lens +cater +desert +statesman +mantel +notion +uterus +anguish +caribou +stroller +seaman +golfer +strew +parrakeet +peanut +welter +mower +rudder +compost +blaze +atlas +gravel +queen +dessert +rhyme +claustrophobia +surgeon +molar +verandah +knight +shock +crow +queer +crop +append +power +junior +anthem +access +clipboard +bachelor +intercept +sink +sing +roof +bode +implicit +remark +talent +conceit +resurrect +weekday +climb +honor +blizzard +liqueur +talon +oval +scandal +gateway +sermon +lime +patron +asteroid +butler +charcoal +trait +kiosk +thatch +trail +train +armadillo +harvest +fan +account +tunnel +carrot +obvious +smear +parkway +unread +fetch +employe +truism +sanitarium +teamster +boney +spruce +serial +contempt +hangar +lamb +lame +lamp +forest +goner +stock +roam +leukemia +bluff +terrier +fray +drape +bind +guru +liner +linen +chief +poacher +furious +furnish +disarm +meter +bunch +marshmallow +decorum +labor +kindergarten +heroism +willing +marsh +dad +junction +dab +dam +spell +swordfish +mention +courtroom +sonata +day +strive +flail +snowdrift +thrill +slacken +cider +memoir +sawdust +disregard +flair +thwart +jailor +jugular +pivot +cupboard +lentil +salesman +hippopotamus +matt +defend +rev +repress +stub +mate +barley +stud +smog +stun +red +franc +frank +hanker +fourteen +salami +likelihood +afterward +squadron +indent +mortar +skateboard +yarn +mortal +workbook +retain +retail +waitress +suffix +overshoot +ethnic +sack +brute +whoop +puppet +guidebook +vandal +pauper +ancient +monkey +bologna +laps +vulgar +vagina +hexagon +scant +liquor +cabin +sixteenth +gear +eavesdrop +bulldog +smolder +forethought +springboard +nun +bodyguard +prune +shrapnel +shampoo +linchpin +lover +anthropologist +tide +cavern +pedlar +countryman +waken +optimum +mix +parka +spinster +meek +dryness +hazel +eight +clamber +handbag +hoodwink +transcript +payment +gullet +gulley +gather +request +absurd +rendezvous +occasion +thicken +recess +kite +text +hamlet +traitor +industrialist +sidetrack +portfolio +floodlight +thicket +staff +communism +scorpion +madman +prolong +resubmit +satan +oppressor +communist +inferior +equilibrium +gumdrop +starch +beat +rush +bear +beam +bean +october +beak +bead +organ +ashtray +nutriment +eyebrow +motherhood +mascara +conform +showdown +infidel +racket +interview +reform +pattern +nebula +brunt +hammock +progress +tailspin +sorrow +stratagem +deliver +blackbird +boloney +exclaim +instant +joke +equal +kosher +swim +swallow +highland +guerrilla +glorious +wear +comment +vent +denim +overcoat +commend +vend +harpoon +manhood +citizenship +copier +newscast +gaze +teamwork +gulch +curtain +curtail +hyacinth +juggler +censor +goddess +bulk +bull +bulb +skew +carburetor +cypher +plain +homey +bray +kinfolk +bicker +dissent +squid +blimp +creak +prose +partner +inspector +lynchpin +portray +whirl +grinder +matchbook +defiant +anorak +tumbler +infer +whirr +tighten +pockmark +sauna +ion +grandstand +sunburn +judgment +retard +center +builder +pickpocket +thought +starboard +usual +coaster +humdrum +fingerprint +storey +clinic +interim +surpass +tough +earshot +flashlight +tong +flee +lupin +lake +bench +add +citizen +ado +crossroad +ravel +match +raven +cantaloup +punctual +newsstand +dryer +insert +flamingo +like +success +sofa +journalist +heed +arraign +chick +soft +heel +outfield +propel +fuze +hail +hair +convey +proper +paddock +novelist +shrug +shrub +slide +tureen +regain +pepper +hose +slight +host +panel +beaker +actual +socket +flake +preen +toadstool +pickaback +discard +tomb +tome +snitch +chronic +guard +esteem +custard +underpass +glimmer +gene +maze +globe +buy +bus +coke +sequel +but +bun +bum +bug +bud +embargo +woodsman +wise +ecosystem +debrief +flip +wisp +wist +trapezoid +condiment +plutonium +pin +garter +domino +circus +pie +pig +pit +campus +gush +oaf +cashew +oak +detail +virtual +detain +sewer +oar +redden +dresser +wallow +nutrient +godsend +yelp +baker +jab +hiker +pupil +yell +cookbook +vermin +sleek +sleet +sleep +liar +hate +trolley +sallow +tweet +glider +under +tweed +pride +merchant +lure +risk +rise +lurk +jack +confetti +anemia +school +parrot +enjoy +overdo +cracker +almond +direct +nail +street +monorail +ransack +blue +hide +worsen +poison +beater +supplier +dashboard +wink +even +pontoon +studio +path +crossbow +enrol +connoisseur +forum +ravish +auction +settler +mentor +midway +blowtorch +stray +straw +strap +cassino +would +phlegm +bayou +asset +spike +preview +musk +mush +saber +muse +grief +phone +muss +pouch +must +shoot +hutch +ma +ms +mr +machinist +fortress +quarrel +loosen +joyous +hemoglobin +dolphin +mayhem +attract +end +trill +keen +bunk +vagrant +rhinoceros +shred +toxin +gate +ancestor +dialect +moisten +kilowatt +mess +lump +mesh +sparkler +parallel +stronghold +splendid +spout +patent +enter +vapor +hedgehog +fetter +deform +clapper +sprout +over +bleach +mallard +oven +caster +digest +forehead +theologian +womanhood +comprehend +tramp +drawback +fade +croquet +tourist +plaster +roost +knighthood +monarch +rental +gloom +chuck +choir +prohibit +hanger +unscrew +gymnasium +poncho +truant +saturday +depress +goo +lair +dinosaur +nitwit +psychic +tonsil +gob +emphysema +nite +washer +resistor +carcass +rail +free +rain +acrobat +fret +harpist +ritual +filter +hopeless +soda +rang +accent +puck +rank +restrict +rant +sober +toy +their +sarcasm +top +tow +tot +fiction +ton +duress +toe +urban +murder +overdraw +tool +hearth +embellish +solicitor +toot +incur +western +nonpartisan +lather +prong +flame +mirth +countess +rag +donkey +fashion +handkerchief +ruckus +taint +raw +rat +rap +protract +spade +ray +snow +thorough +contact +hatch +snob +cleft +extravert +quicken +rider +evangelist +shallow +milkman +coil +coin +glow +interject +flop +metal +freeway +policewoman +flog +yank +chariot +bait +endear +saga +alight +random +sage +dupe +radio +rector +earth +bail +shellfish +spite +stanza +disgust +axiom +waltz +gees +watch +fluid +ultimatum +report +reconstruct +noon +spokesperson +egoism +public +erupt +pacifist +pummel +habit +wrest +nut +resist +corrupt +hourglass +mull +mud +mug +finger +mum +approach +wean +weak +contort +boss +toothbrush +southeast +larynx +devour +devout +censorship +newt +protect +irregular +fault +papyrus +facet +elf +smuggler +trust +bingo +bathroom +beef +legend +beer +spread +communion +loft +bladder +uncommon +craft +spearmint +catch +snipe +teapot +misfit +lessen +thousandth +referendum +pyramid +handrail +broth +lollypop +exterior +suggest +wound +overstep +utilitarian +complex +papaya +screw +pick +deflect +suburb +portal +postmark +tassel +ocean +mother +bough +bugger +rodent +shorthand +enlighten +elk +elm +moonbeam +flutist +kelp +misprint +teetotal +upturn +ramrod +dismount +quicksand +spanner +authorship +roach +befit +rumor +apart +ditto +gift +zeal +contradict +hunt +dishonest +zoom +mongrel +hunk +mosquito +hunchback +sanction +excerpt +curio +accost +usher +indirect +intellect +doorstep +nobleman +cooper +combat +letterhead +ice +rhino +newsprint +skylight +convict +christmas +splendor +cord +core +khaki +brawl +corn +brawn +cork +discount +shuck +plug +census +cowboy +plum +choke +surround +caulk +dinner +plus +alga +duke +abet +civic +civil +bath +engulf +cafeteria +art +scamper +transform +sunlight +forbad +virgin +gin +head +medium +amateur +heal +stereo +heat +hear +heap +raft +counsel +muster +bargain +bide +latch +adorn +trim +trio +forearm +cobbler +trip +no +tit +when +junta +tin +whet +tie +implant +depot +pseudonym +evergreen +cleric +toad +geneticist +bullet +navel +yacht +withhold +fasten +backward +coach +impression +rob +rod +focus +livelihood +snip +yokel +rot +discern +environment +aplomb +melon +prop +coop +impend +plantain +cook +cool +looney +level +brother +encroach +quick +lever +pork +drier +trend +bullfrog +pore +inland +voucher +takeoff +bake +port +colic +hymn +choral +postman +spire +theist +thresh +tormentor +humorist +water +fluke +entertain +witch +twentieth +tire +boast +catnap +blotch +cinnamon +prude +weird +tweak +brighten +touchdown +post +panacea +concoct +scan +handler +prey +today +chapter +conductor +altar +cashier +drown +dismal +inhabit +judo +conceal +flagship +hullabaloo +fauna +laughter +streak +overpass +sandbag +trump +stream +despot +stroke +cube +hydrogen +bigot +dress +vital +fourth +dope +ballroom +fascist +clone +scoff +fascism +birthday +apprenticeship +eighth +repeat +classroom +twitch +liquid +inform +reaper +lagoon +superscript +refund +rye +midnight +blare +worship +thermostat +apex +platform +farmer +meridian +cutter +underneath +conquer +fern +rescind +wagon +term +name +realist +opera +bunion +bullion +realism +ailment +torch +zebra +distributor +hysteria +hacker +concur +profit +middleman +gram +clover +hull +hulk +flyer +tuner +flare +highjack +motion +turn +place +swine +swing +turf +preach +childhood +origin +pelican +feign +suspend +insist +scollop +bobcat +array +peddler +given +afterthought +district +opus +trillion +plastic +assort +white +hue +hug +hub +cope +season +hut +enigma +naught +grunt +holder +wide +bewitch +spokeswoman +oath +powder +rend +froth +pro +ani +ant +rent +dragon +stolid +marathon +ideal +blunt +surf +sure +aspen +equestrian +tornado +freshman +librarian +bluebird +icon +latex +tendon +annul +seafood +later +koran +readjust +senior +slope +perch +convoy +cheat +cheap +trespass +hack +broach +hustler +trot +woolen +gulf +genius +gull +shimmer +crime +gulp +woof +wood +deign +wool +entrant +viewpoint +lighten +jazz +festoon +tailor +lighter +dye +homestead +reveal +aluminum +workman +joker +dumfound +bison +picnic +pane +vizor +prowl +optometrist +detect +crooked +review +spoons +hiss +smokestack +caucus +fearless +guitar +coma +comb +come +zodiac +isthmus +reaction +superstar +region +quiet +contract +railway +penal +adjoin +color +armchair +pot +period +pop +pole +colon +polo +pod +poll +runaway +turkey +hobo +schoolboy +tiger +padlock +hernia +careful +spirit +robber +pilot +case +shaft +amend +mount +cash +cask +cast +mound +ventriloquist +vest +exult +clutter +helmet +projector +author +alphabet +fender +bowl +check +macaroni +catfish +bellboy +hermit +week +sang +applaud +nest +driver +weed +director +petticoat +lute +puke +vowel +muffler +weep +cartoon +ranch +relief +model +reward +sinner +clod +clog +tip +kilt +ninth +clot +lavish +violent +kill +kiln +kilo +polish +satin +blow +blot +hint +rose +regent +except +blog +bloc +blob +hind +scrapbook +disrupt +impound +kingdom +blowout +sandman +mugger +towel +bracelet +snort +friar +tower +node +deduct +wombat +interlock +canteen +slice +mood +slick +legal +moon +moos +teardrop +moot +heir +porter +metropolis +quit +unmask +slaughter +quip +ok +oh +of +jeer +shrimp +pistachio +karat +stand +ox +doze +accredit +amber +tribe +vicar +polka +garb +spinal +forewarn +feudal +whaler +there +racism +strict +racist +valley +fish +gibe +relic +jug +regard +cabinet +castaway +strenuous +jut +terminus +feeler +grasp +grass +toilet +ruffian +cinema +frighten +lichen +encompass +bishop +incorrect +abyss +fiddler +heather +idiot +diarrhea +rubber +idiom +heathen +trash +stalwart +championship +symbol +cove +nucleus +serious +brass +wife +invest +derrick +treason +apparel +platter +all +lace +duplex +lack +spacecraft +disc +dish +follow +settlement +titter +wanton +thursday +program +neglig +woman +song +fat +roe +psychologist +retract +ultraviolet +awful +dapper +fad +induct +stimulus +list +trench +align +flick +ten +tea +tee +rate +design +chalk +what +sub +sun +sum +whimper +rascal +brief +overload +crush +version +pulpit +intersect +row +womb +lacquer +pumpernickel +backer +goodby +thrift +misinterpret +heifer +jogger +cataract +haemophilia +murmur +snug +snub +herring +proceed +tarpaulin +wield +hurray +rustic +quash +inlay +garnish +hurrah +minor +ladybug +wretched +flap +mire +protestor +stutter +flit +flag +stick +mellow +chaplain +berth +wrestler +plagiarist +searchlight +sunscreen +pond +court +goal +goad +goat +sandwich +okay +algebra +headrest +embalm +reflect +catalog +numb +short +ricochet +tsar +shade +waylay +mission +scientist +flaunt +reconnect +pretext +stride +islam +thirtieth +style +glide +pray +inward +wilder +abbey +mattress +resort +airstrip +bout +soccer +might +alter +return +hunter +underground +abacus +mathematician +liniment +policeman +refresh +tactless +friendship +weight +needless +duchess +falsetto +expect +inflict +wager +alcohol +disquiet +hilt +dugout +loll +health +hill +shipment +fiber +solvent +friday +differ +effect +disinfect +octagon +physic +teach +sidewalk +jew +blister +thread +threat +bushel +feed +dine +feel +sailor +revolution +least +blank +cigaret +idea +moan +script +gourmet +interact +grime +stork +swarm +storm +moat +syrup +store +mainstay +imperfect +option +hotel +fidget +king +kind +vial +kink +stall +cuff +foreleg +stale +restful +amass +cleaner +exert +strengthen +shrewd +bookworm +gale +gala +gall +remodel +smallpox +toughen +bacteria +chairman +donor +pianist +buff +gill +foreman +rapist +reckon +english +reach +react +nothing +quorum +hyena +amphibian +saloon +notch +scaffold +asphalt +memorandum +felon +font +anvil +firewood +betray +hip +shepherd +hit +deaden +reprint +him +adulthood +snowstorm +forego +stump +martyrdom +arc +bare +bard +bark +ark +arm +barn +blurt +parchment +various +plywood +nincompoop +solo +muslim +sole +outfit +succeed +inertia +orangutang +blazer +bandit +context +bond +cynic +sloth +flier +distress +chaperon +sweet +wastebasket +sweep +weasel +rave +shaykh +bolster +dub +overlook +dud +due +buttermilk +pa +watchword +brick +pi +flight +quintet +dropout +marihuana +cinch +temperament +instructor +heighten +toga +shove +batch +pitchfork +kick +behavior +incognito +lodger +bluegrass +sirup +rip +shamrock +rim +frantic +rig +rid +reprogram +chauffeur +shirt +kimono +viola +shirk +sliver +straightjacket +restraint +painless +throwback +cement +birch +robust +knack +lower +earmark +cheek +cheep +cheer +pollen +facial +vigilant +cabaret +continent +tablet +contractor +plateau +tuxedo +complaint +vendor +foreshadow +awaken +confront +uproar +distrust +breeder +hallmark +play +global +litter +wonderland +butterscotch +saucer +prow +seller +prom +prod +sag +perpendicular +tinker +raider +vivid +cautious +undertow +yawn +ordeal +militia +dialog +tomboy +conquest +momma +piteous +holster +vagabond +stench +canal +pundit +question +swill +parsnip +etch +filet +potassium +glamour +cloth +crank +usurp +delta +upright +crane +outpost +penguin +patriot +consist +apricot +caller +peppermint +husk +cartwheel +highlight +dill +freak +dismay +sublet +sagebrush +rainbow +lemon +riot +peach +grouper +saffron +nick +parlor +ferment +bandstand +mock +nice +mustard +chirp +meaning +vigil +vice +ocular +remit +pyre +buffalo +scroll +pervert +lean +alien +dispel +gang +theorist +gold +uphold +floss +breach +sniper +materialist +toenail +spool +spoon +spook +spoof +harlot +outdo +pleas +pleat +trawl +procession +fold +reunion +acid +folk +sandstorm +outsmart +acronym +relent +kangaroo +gloat +miser +cyclist +barb +survivor +guarantor +orangutan +armpit +shovel +duct +ensign +apt +volt +motor +duck +thick +redo +ape +use +fee +fen +frog +germ +modicum +fez +sort +parliament +porch +musician +impress +sore +rabbit +recount +penis +sculptor +annoy +topic +augment +critic +lumber +executor +proof +bittersweet +tap +tar +tax +villain +tag +condescend +tab +spa +silt +tan +rape +counterfeit +sir +sip +scuff +sit +tamper +six +outclass +occur +sic +carrier +goldsmith +toddler +panic +sin +defeat +tension +lesion +attend +tact +hazard +discomfort +tack +wrist +taco +footpath +aftereffect +light +arduous +schoolchild +sailboat +stamped +minnow +damsel +accompanist +hemp +tyrant +badger +glen +superior +inlet +sill +glee +nostalgia +flank +restrain +glisten +turban +redhead +bye +flex +crash +citrus +flour +flout +emerald +flea +republican +investor +successor +easel +footstool +profound +edit +feast +fuzz +trap +blacktop +cocoon +tray +lilac +mincemeat +interplay +our +proclaim +out +semen +tabloid +cocktail +sentiment +frontier +vehement +disarray +clatter +impart +plural +proviso +planner +utensil +tenement +pendant +gospel +tenant +greenhorn +tanker +zoologist +rivet +uproot +embryo +sew +bouquet +echo +bonnet +eleventh +synagog +salient +droop +unknown +galley +snore +anaemia +drool +boil +tidbit +shell +shelf +transistor +woo +diminish +persecutor +goblin +institution +kickback +frugal +brazen +yodel +laughingstock +clip +fowl +splatter +flunkey +blip +footwork +outstrip +disjoint +pallor +catholic +clove +rout +outward +bagel +lope +divert +trivia +pharmacist +divers +clash +petrol +siphon +filch +fortieth +class +clasp +fang +dens +dent +pipe +vernacular +gain +son +stove +sonnet +utter +chicken +feat +winch +dandruff +rioter +herald +piano +local +counsellor +vigor +sued +skimp +plaintiff +spud +watercolor +barter +bronco +spur +rite +ghetto +bisect +compliment +ascertain +sediment +view +unison +workbench +ebb +expel +hymnal +distract +violet +still +closet +superb +favor +viper +crude +torpedo +avow +jot +exam +amen +joy +foetus +job +spoil +jog +swift +memento +lifeboat +april +grain +commando +wall +hyphen +walk +respect +unclean +decent +trademark +tutor +reindeer +mike +nickel +cypress +penmanship +dearth +overturn +present +kerchief +corset +wilt +vanilla +priesthood +will +fingertip +wild +whirlpool +layer +mutant +motif +apprehend +rooster +lightweight +thug +thud +whore +headlight +cross +member +pediatrist +inch +grandeur +slave +diploma +outcast +beast +student +pedal +whale +collar +gutter +masochist +overwork +scissor +twirl +flint +outgrow +bandanna +rocker +cameo +rocket +camel +boot +wren +obtain +replenish +biologist +daub +distend +smite +now +panther +drunk +smith +hall +book +ski +enact +knob +sick +myth +know +knot +press +redesign +doughnut +loser +cutlet +vortex +clutch +exceed +setter +flagrant +birthmark +demeanor +growth +export +leaf +lead +leak +miner +leap +belt +leader +trout +obey +slur +mitt +slut +slum +pasta +mite +slug +throne +pike +throng +rare +linger +column +biscuit +fear +swear +sweat +udder +emperor +owl +outset +own +owe +weather +champ +brush +billfold +gape +rowboat +van +platoon +transfer +spiral +grotto +cliff +vat +nourish +catsup +unwrap +saunter +mutter +brassier +assail +tomcat +daffodil +nightgown +record +cake +faggot +maroon +boardwalk +abbot +counteract +limb +squirrel +mutual +glint +boor +percent +other +boom +branch +cutthroat +junk +mulch +june +squeak +squeal +extort +jewel +gynecologist +vane +sash diff --git a/libs/pyutil/fileutil.py~ b/libs/pyutil/fileutil.py~ deleted file mode 100644 index e37eb792..00000000 --- a/libs/pyutil/fileutil.py~ +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn -# This file is part of pyutil; see README.rst for licensing terms. - -""" -Futz with files like a pro. -""" - -import errno, exceptions, os, stat, tempfile - -try: - import bsddb -except ImportError: - DBNoSuchFileError = None -else: - DBNoSuchFileError = bsddb.db.DBNoSuchFileError - -# read_file() and write_file() copied from Mark Seaborn's blog post. Please -# read it for complete rationale: -# http://lackingrhoticity.blogspot.com/2009/12/readfile-and-writefile-in-python.html - -def read_file(filename, mode='rb'): - """ Read the contents of the file named filename and return it in - a string. This function closes the file handle before it returns - (even if the underlying Python implementation's garbage collector - doesn't). """ - fh = open(filename, mode) - try: - return fh.read() - finally: - fh.close() - -def write_file(filename, data, mode='wb'): - """ Write the string data into a file named filename. This - function closes the file handle (ensuring that the written data is - flushed from the perspective of the Python implementation) before - it returns (even if the underlying Python implementation's garbage - collector doesn't).""" - fh = open(filename, mode) - try: - fh.write(data) - finally: - fh.close() - -# For backwards-compatibility in case someone is using these names. We used to -# have a superkludge in fileutil.py under these names. -def rename(src, dst, tries=4, basedelay=0.1): - return os.rename(src, dst) - -def remove(f, tries=4, basedelay=0.1): - return os.remove(f) - -def rmdir(f, tries=4, basedelay=0.1): - return os.rmdir(f) - -class _Dir(object): - """ - Hold a set of files and subdirs and clean them all up when asked to. - """ - def __init__(self, name, cleanup=True): - self.name = name - self.cleanup = cleanup - self.files = [] - self.subdirs = set() - - def file(self, fname, mode=None): - """ - Create a file in the tempdir and remember it so as to close() it - before attempting to cleanup the temp dir. - - @rtype: file - """ - ffn = os.path.join(self.name, fname) - if mode is not None: - fo = open(ffn, mode) - else: - fo = open(ffn) - self.register_file(fo) - return fo - - def subdir(self, dirname): - """ - Create a subdirectory in the tempdir and remember it so as to call - shutdown() on it before attempting to clean up. - - @rtype: _Dir instance - """ - ffn = os.path.join(self.name, dirname) - sd = _Dir(ffn, self.cleanup) - self.register_subdir(sd) - make_dirs(sd.name) - return sd - - def register_file(self, fileobj): - """ - Remember the file object and call close() on it before attempting to - clean up. - """ - self.files.append(fileobj) - - def register_subdir(self, dirobj): - """ - Remember the _Dir object and call shutdown() on it before attempting - to clean up. - """ - self.subdirs.add(dirobj) - - def shutdown(self): - if self.cleanup: - for subdir in hasattr(self, 'subdirs') and self.subdirs or []: - subdir.shutdown() - for fileobj in hasattr(self, 'files') and self.files or []: - if DBNoSuchFileError is None: - fileobj.close() # "close()" is idempotent so we don't need to catch exceptions here - else: - try: - fileobj.close() - except DBNoSuchFileError: - # Ah, except that the bsddb module's file-like object (a DB object) has a non-idempotent close... - pass - - if hasattr(self, 'name'): - rm_dir(self.name) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - try: - self.shutdown() - except: - import traceback - traceback.print_exc() - -class NamedTemporaryDirectory(_Dir): - """ - Call tempfile.mkdtemp(), store the name of the dir in self.name, and - rm_dir() when it gets garbage collected or "shutdown()". - - Also keep track of file objects for files within the tempdir and call - close() on them before rm_dir(). This is a convenient way to open temp - files within the directory, and it is very helpful on Windows because you - can't delete a directory which contains a file which is currently open. - """ - - def __init__(self, cleanup=True, *args, **kwargs): - """ If cleanup, then the directory will be rmrf'ed when the object is shutdown. """ - name = tempfile.mkdtemp(*args, **kwargs) - _Dir.__init__(self, name, cleanup) - -class ReopenableNamedTemporaryFile: - """ - This uses tempfile.mkstemp() to generate a secure temp file. It then closes - the file, leaving a zero-length file as a placeholder. You can get the - filename with ReopenableNamedTemporaryFile.name. When the - ReopenableNamedTemporaryFile instance is garbage collected or its shutdown() - method is called, it deletes the file. - """ - def __init__(self, *args, **kwargs): - fd, self.name = tempfile.mkstemp(*args, **kwargs) - os.close(fd) - - def __repr__(self): - return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) - - def __str__(self): - return self.__repr__() - - def __del__(self): - self.shutdown() - - def shutdown(self): - remove(self.name) - -def make_dirs(dirname, mode=0777): - """ - An idempotent version of os.makedirs(). If the dir already exists, do - nothing and return without raising an exception. If this call creates the - dir, return without raising an exception. If there is an error that - prevents creation or if the directory gets deleted after make_dirs() creates - it and before make_dirs() checks that it exists, raise an exception. - """ - tx = None - try: - os.makedirs(dirname, mode) - except OSError, x: - tx = x - - if not os.path.isdir(dirname): - if tx: - raise tx - raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning... - -def rmtree(dirname): - """ - A threadsafe and idempotent version of shutil.rmtree(). If the dir is - already gone, do nothing and return without raising an exception. If this - call removes the dir, return without raising an exception. If there is an - error that prevents deletion or if the directory gets created again after - rm_dir() deletes it and before rm_dir() checks that it is gone, raise an - exception. - """ - excs = [] - try: - os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - for f in os.listdir(dirname): - fullname = os.path.join(dirname, f) - if os.path.isdir(fullname): - rm_dir(fullname) - else: - remove(fullname) - os.rmdir(dirname) - except EnvironmentError, le: - # Ignore "No such file or directory", collect any other exception. - if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): - excs.append(le) - except Exception, le: - excs.append(le) - - # Okay, now we've recursively removed everything, ignoring any "No - # such file or directory" errors, and collecting any other errors. - - if os.path.exists(dirname): - if len(excs) == 1: - raise excs[0] - if len(excs) == 0: - raise OSError, "Failed to remove dir for unknown reason." - raise OSError, excs - -def rm_dir(dirname): - # Renamed to be like shutil.rmtree and unlike rmdir. - return rmtree(dirname) - -def remove_if_possible(f): - try: - remove(f) - except EnvironmentError: - pass - -def remove_if_present(f): - try: - remove(f) - except EnvironmentError, le: - # Ignore "No such file or directory", re-raise any other exception. - if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): - raise - -def rmdir_if_possible(f): - try: - rmdir(f) - except EnvironmentError: - pass - -def open_or_create(fname, binarymode=True): - try: - f = open(fname, binarymode and "r+b" or "r+") - except EnvironmentError: - f = open(fname, binarymode and "w+b" or "w+") - return f - -def du(basedir): - size = 0 - - for root, dirs, files in os.walk(basedir): - for f in files: - fn = os.path.join(root, f) - size += os.path.getsize(fn) - - return size diff --git a/libs/pyutil/iputil.py b/libs/pyutil/iputil.py index b8b96362..cb3e7c02 100644 --- a/libs/pyutil/iputil.py +++ b/libs/pyutil/iputil.py @@ -1,22 +1,12 @@ -# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith) -# portions adapted from nattraverso.ipdiscover -# portions authored by Brian Warner, working for Allmydata -# most recent version authored by Zooko O'Whielacronx, working for Allmydata - # from the Python Standard Library -import os, re, socket, sys +import os, re, socket, sys, subprocess # from Twisted -from twisted.internet import defer, reactor -from twisted.python import failure +from twisted.internet import defer, threads, reactor from twisted.internet.protocol import DatagramProtocol -from twisted.internet.utils import getProcessOutput from twisted.python.procutils import which from twisted.python import log -# from pyutil -import observer - try: import resource def increase_rlimits(): @@ -77,6 +67,7 @@ except ImportError: # since one might be shadowing the other. This hack appeases pyflakes. increase_rlimits = _increase_rlimits + def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad @@ -121,14 +112,16 @@ def get_local_ip_for(target): except socket.gaierror: # DNS isn't running, or somehow we encountered an error - # note: if an interface is configured and up, but nothing is connected to it, - # gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror - # . This is synchronous and occurs for each node being started, so users of certain unit - # tests will see something like 120s of delay, which may be enough to hit the default - # trial timeouts. For that reason, get_local_addresses_async() was changed to default to - # the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also - # makes node startup a tad faster. - + # note: if an interface is configured and up, but nothing is + # connected to it, gethostbyname("A.ROOT-SERVERS.NET") will take 20 + # seconds to raise socket.gaierror . This is synchronous and occurs + # for each node being started, so users of + # test.common.SystemTestMixin (like test_system) will see something + # like 120s of delay, which may be enough to hit the default trial + # timeouts. For that reason, get_local_addresses_async() was changed + # to default to the numerical ip address for A.ROOT-SERVERS.NET, to + # avoid this DNS lookup. This also makes node startup fractionally + # faster. return None udpprot = DatagramProtocol() port = reactor.listenUDP(0, udpprot) @@ -146,16 +139,29 @@ _platform_map = { "linux-i386": "linux", # redhat "linux-ppc": "linux", # redhat "linux2": "linux", # debian + "linux3": "linux", # debian "win32": "win32", "irix6-n32": "irix", "irix6-n64": "irix", "irix6": "irix", "openbsd2": "bsd", + "openbsd3": "bsd", + "openbsd4": "bsd", + "openbsd5": "bsd", "darwin": "bsd", # Mac OS X "freebsd4": "bsd", "freebsd5": "bsd", "freebsd6": "bsd", + "freebsd7": "bsd", + "freebsd8": "bsd", + "freebsd9": "bsd", "netbsd1": "bsd", + "netbsd2": "bsd", + "netbsd3": "bsd", + "netbsd4": "bsd", + "netbsd5": "bsd", + "netbsd6": "bsd", + "dragonfly2": "bsd", "sunos5": "sunos", "cygwin": "cygwin", } @@ -173,12 +179,12 @@ _win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\ # These work in Redhat 6.x and Debian 2.2 potato _linux_path = '/sbin/ifconfig' -_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) +_linux_re = re.compile('^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) -# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD -_bsd_path = '/sbin/ifconfig' -_bsd_args = ('-a',) -_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) +# NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X +_netbsd_path = '/sbin/ifconfig' +_netbsd_args = ('-a',) +_netbsd_re = re.compile('^\s+inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) # Irix 6.5 _irix_path = '/usr/etc/ifconfig' @@ -186,39 +192,6 @@ _irix_path = '/usr/etc/ifconfig' # Solaris 2.x _sunos_path = '/usr/sbin/ifconfig' -class SequentialTrier(object): - """ I hold a list of executables to try and try each one in turn - until one gives me a list of IP addresses.""" - - def __init__(self, exebasename, args, regex): - assert not os.path.isabs(exebasename) - self.exes_left_to_try = which(exebasename) - self.exes_left_to_try.reverse() - self.args = args - self.regex = regex - self.o = observer.OneShotObserverList() - self._try_next() - - def _try_next(self): - if not self.exes_left_to_try: - self.o.fire(None) - else: - exe = self.exes_left_to_try.pop() - d2 = _query(exe, self.args, self.regex) - - def cb(res): - if res: - self.o.fire(res) - else: - self._try_next() - - def eb(why): - self._try_next() - - d2.addCallbacks(cb, eb) - - def when_tried(self): - return self.o.when_fired() # k: platform string as provided in the value of _platform_map # v: tuple of (path_to_tool, args, regex,) @@ -226,19 +199,22 @@ _tool_map = { "linux": (_linux_path, (), _linux_re,), "win32": (_win32_path, _win32_args, _win32_re,), "cygwin": (_win32_path, _win32_args, _win32_re,), - "bsd": (_bsd_path, _bsd_args, _bsd_re,), - "irix": (_irix_path, _bsd_args, _bsd_re,), - "sunos": (_sunos_path, _bsd_args, _bsd_re,), + "bsd": (_netbsd_path, _netbsd_args, _netbsd_re,), + "irix": (_irix_path, _netbsd_args, _netbsd_re,), + "sunos": (_sunos_path, _netbsd_args, _netbsd_re,), } + def _find_addresses_via_config(): - # originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API. + return threads.deferToThread(_synchronously_find_addresses_via_config) + +def _synchronously_find_addresses_via_config(): + # originally by Greg Smith, hacked by Zooko to conform to Brian's API platform = _platform_map.get(sys.platform) - (pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,)) + if not platform: + raise UnsupportedPlatformError(sys.platform) - # If the platform isn't known then we attempt BSD-style ifconfig. If it - # turns out that we don't get anything resembling a dotted quad IPv4 address - # out of it, then we'll raise UnsupportedPlatformError. + (pathtotool, args, regex,) = _tool_map[platform] # If pathtotool is a fully qualified path then we just try that. # If it is merely an executable name then we use Twisted's @@ -246,34 +222,33 @@ def _find_addresses_via_config(): # gives us something that resembles a dotted-quad IPv4 address. if os.path.isabs(pathtotool): - d = _query(pathtotool, args, regex) + return _query(pathtotool, args, regex) else: - d = SequentialTrier(pathtotool, args, regex).when_tried() - - d.addCallback(_check_result) - return d - -def _check_result(result): - if not result and not _platform_map.has_key(sys.platform): - return failure.Failure(UnsupportedPlatformError(sys.platform)) - else: - return result + exes_to_try = which(pathtotool) + for exe in exes_to_try: + try: + addresses = _query(exe, args, regex) + except Exception: + addresses = [] + if addresses: + return addresses + return [] def _query(path, args, regex): - d = getProcessOutput(path, args) - def _parse(output): - addresses = [] - outputsplit = output.split('\n') - for outline in outputsplit: - m = regex.match(outline) - if m: - addr = m.groupdict()['address'] - if addr not in addresses: - addresses.append(addr) + env = {'LANG': 'en_US.UTF-8'} + p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + (output, err) = p.communicate() - return addresses - d.addCallback(_parse) - return d + addresses = [] + outputsplit = output.split('\n') + for outline in outputsplit: + m = regex.match(outline) + if m: + addr = m.groupdict()['address'] + if addr not in addresses: + addresses.append(addr) + + return addresses def _cygwin_hack_find_addresses(target): addresses = [] diff --git a/libs/pyutil/iputil.py~ b/libs/pyutil/iputil.py~ new file mode 100644 index 00000000..b8b96362 --- /dev/null +++ b/libs/pyutil/iputil.py~ @@ -0,0 +1,288 @@ +# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith) +# portions adapted from nattraverso.ipdiscover +# portions authored by Brian Warner, working for Allmydata +# most recent version authored by Zooko O'Whielacronx, working for Allmydata + +# from the Python Standard Library +import os, re, socket, sys + +# from Twisted +from twisted.internet import defer, reactor +from twisted.python import failure +from twisted.internet.protocol import DatagramProtocol +from twisted.internet.utils import getProcessOutput +from twisted.python.procutils import which +from twisted.python import log + +# from pyutil +import observer + +try: + import resource + def increase_rlimits(): + # We'd like to raise our soft resource.RLIMIT_NOFILE, since certain + # systems (OS-X, probably solaris) start with a relatively low limit + # (256), and some unit tests want to open up more sockets than this. + # Most linux systems start with both hard and soft limits at 1024, + # which is plenty. + + # unfortunately the values to pass to setrlimit() vary widely from + # one system to another. OS-X reports (256, HUGE), but the real hard + # limit is 10240, and accepts (-1,-1) to mean raise it to the + # maximum. Cygwin reports (256, -1), then ignores a request of + # (-1,-1): instead you have to guess at the hard limit (it appears to + # be 3200), so using (3200,-1) seems to work. Linux reports a + # sensible (1024,1024), then rejects (-1,-1) as trying to raise the + # maximum limit, so you could set it to (1024,1024) but you might as + # well leave it alone. + + try: + current = resource.getrlimit(resource.RLIMIT_NOFILE) + except AttributeError: + # we're probably missing RLIMIT_NOFILE + return + + if current[0] >= 1024: + # good enough, leave it alone + return + + try: + if current[1] > 0 and current[1] < 1000000: + # solaris reports (256, 65536) + resource.setrlimit(resource.RLIMIT_NOFILE, + (current[1], current[1])) + else: + # this one works on OS-X (bsd), and gives us 10240, but + # it doesn't work on linux (on which both the hard and + # soft limits are set to 1024 by default). + resource.setrlimit(resource.RLIMIT_NOFILE, (-1,-1)) + new = resource.getrlimit(resource.RLIMIT_NOFILE) + if new[0] == current[0]: + # probably cygwin, which ignores -1. Use a real value. + resource.setrlimit(resource.RLIMIT_NOFILE, (3200,-1)) + + except ValueError: + log.msg("unable to set RLIMIT_NOFILE: current value %s" + % (resource.getrlimit(resource.RLIMIT_NOFILE),)) + except: + # who knows what. It isn't very important, so log it and continue + log.err() +except ImportError: + def _increase_rlimits(): + # TODO: implement this for Windows. Although I suspect the + # solution might be "be running under the iocp reactor and + # make this function be a no-op". + pass + # pyflakes complains about two 'def FOO' statements in the same time, + # since one might be shadowing the other. This hack appeases pyflakes. + increase_rlimits = _increase_rlimits + +def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET + """ + Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad + strings) that are currently configured on this host, sorted in descending + order of how likely we think they are to work. + + @param target: we want to learn an IP address they could try using to + connect to us; The default value is fine, but it might help if you + pass the address of a host that you are actually trying to be + reachable to. + """ + addresses = [] + local_ip = get_local_ip_for(target) + if local_ip: + addresses.append(local_ip) + + if sys.platform == "cygwin": + d = _cygwin_hack_find_addresses(target) + else: + d = _find_addresses_via_config() + + def _collect(res): + for addr in res: + if addr != "0.0.0.0" and not addr in addresses: + addresses.append(addr) + return addresses + d.addCallback(_collect) + + return d + +def get_local_ip_for(target): + """Find out what our IP address is for use by a given target. + + @return: the IP address as a dotted-quad string which could be used by + to connect to us. It might work for them, it might not. If + there is no suitable address (perhaps we don't currently have an + externally-visible interface), this will return None. + """ + + try: + target_ipaddr = socket.gethostbyname(target) + except socket.gaierror: + # DNS isn't running, or somehow we encountered an error + + # note: if an interface is configured and up, but nothing is connected to it, + # gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror + # . This is synchronous and occurs for each node being started, so users of certain unit + # tests will see something like 120s of delay, which may be enough to hit the default + # trial timeouts. For that reason, get_local_addresses_async() was changed to default to + # the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also + # makes node startup a tad faster. + + return None + udpprot = DatagramProtocol() + port = reactor.listenUDP(0, udpprot) + try: + udpprot.transport.connect(target_ipaddr, 7) + localip = udpprot.transport.getHost().host + except socket.error: + # no route to that host + localip = None + port.stopListening() # note, this returns a Deferred + return localip + +# k: result of sys.platform, v: which kind of IP configuration reader we use +_platform_map = { + "linux-i386": "linux", # redhat + "linux-ppc": "linux", # redhat + "linux2": "linux", # debian + "win32": "win32", + "irix6-n32": "irix", + "irix6-n64": "irix", + "irix6": "irix", + "openbsd2": "bsd", + "darwin": "bsd", # Mac OS X + "freebsd4": "bsd", + "freebsd5": "bsd", + "freebsd6": "bsd", + "netbsd1": "bsd", + "sunos5": "sunos", + "cygwin": "cygwin", + } + +class UnsupportedPlatformError(Exception): + pass + +# Wow, I'm really amazed at home much mileage we've gotten out of calling +# the external route.exe program on windows... It appears to work on all +# versions so far. Still, the real system calls would much be preferred... +# ... thus wrote Greg Smith in time immemorial... +_win32_path = 'route.exe' +_win32_args = ('print',) +_win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S) + +# These work in Redhat 6.x and Debian 2.2 potato +_linux_path = '/sbin/ifconfig' +_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) + +# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD +_bsd_path = '/sbin/ifconfig' +_bsd_args = ('-a',) +_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) + +# Irix 6.5 +_irix_path = '/usr/etc/ifconfig' + +# Solaris 2.x +_sunos_path = '/usr/sbin/ifconfig' + +class SequentialTrier(object): + """ I hold a list of executables to try and try each one in turn + until one gives me a list of IP addresses.""" + + def __init__(self, exebasename, args, regex): + assert not os.path.isabs(exebasename) + self.exes_left_to_try = which(exebasename) + self.exes_left_to_try.reverse() + self.args = args + self.regex = regex + self.o = observer.OneShotObserverList() + self._try_next() + + def _try_next(self): + if not self.exes_left_to_try: + self.o.fire(None) + else: + exe = self.exes_left_to_try.pop() + d2 = _query(exe, self.args, self.regex) + + def cb(res): + if res: + self.o.fire(res) + else: + self._try_next() + + def eb(why): + self._try_next() + + d2.addCallbacks(cb, eb) + + def when_tried(self): + return self.o.when_fired() + +# k: platform string as provided in the value of _platform_map +# v: tuple of (path_to_tool, args, regex,) +_tool_map = { + "linux": (_linux_path, (), _linux_re,), + "win32": (_win32_path, _win32_args, _win32_re,), + "cygwin": (_win32_path, _win32_args, _win32_re,), + "bsd": (_bsd_path, _bsd_args, _bsd_re,), + "irix": (_irix_path, _bsd_args, _bsd_re,), + "sunos": (_sunos_path, _bsd_args, _bsd_re,), + } +def _find_addresses_via_config(): + # originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API. + + platform = _platform_map.get(sys.platform) + (pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,)) + + # If the platform isn't known then we attempt BSD-style ifconfig. If it + # turns out that we don't get anything resembling a dotted quad IPv4 address + # out of it, then we'll raise UnsupportedPlatformError. + + # If pathtotool is a fully qualified path then we just try that. + # If it is merely an executable name then we use Twisted's + # "which()" utility and try each executable in turn until one + # gives us something that resembles a dotted-quad IPv4 address. + + if os.path.isabs(pathtotool): + d = _query(pathtotool, args, regex) + else: + d = SequentialTrier(pathtotool, args, regex).when_tried() + + d.addCallback(_check_result) + return d + +def _check_result(result): + if not result and not _platform_map.has_key(sys.platform): + return failure.Failure(UnsupportedPlatformError(sys.platform)) + else: + return result + +def _query(path, args, regex): + d = getProcessOutput(path, args) + def _parse(output): + addresses = [] + outputsplit = output.split('\n') + for outline in outputsplit: + m = regex.match(outline) + if m: + addr = m.groupdict()['address'] + if addr not in addresses: + addresses.append(addr) + + return addresses + d.addCallback(_parse) + return d + +def _cygwin_hack_find_addresses(target): + addresses = [] + for h in [target, "localhost", "127.0.0.1",]: + try: + addr = get_local_ip_for(h) + if addr not in addresses: + addresses.append(addr) + except socket.gaierror: + pass + + return defer.succeed(addresses) diff --git a/libs/pyutil/mathutil.py b/libs/pyutil/mathutil.py index 46781b0c..9c169801 100644 --- a/libs/pyutil/mathutil.py +++ b/libs/pyutil/mathutil.py @@ -11,7 +11,7 @@ def div_ceil(n, d): """ The smallest integer k such that k*d >= n. """ - return (n/d) + (n%d != 0) + return int((n//d) + (n%d != 0)) def next_multiple(n, k): """ diff --git a/libs/pyutil/mathutil.py~ b/libs/pyutil/mathutil.py~ new file mode 100644 index 00000000..46781b0c --- /dev/null +++ b/libs/pyutil/mathutil.py~ @@ -0,0 +1,106 @@ +# Copyright (c) 2005-2010 Zooko Wilcox-O'Hearn +# This file is part of pyutil; see README.rst for licensing terms. + +""" +A few commonly needed functions. +""" + +import math + +def div_ceil(n, d): + """ + The smallest integer k such that k*d >= n. + """ + return (n/d) + (n%d != 0) + +def next_multiple(n, k): + """ + The smallest multiple of k which is >= n. Note that if n is 0 then the + answer is 0. + """ + return div_ceil(n, k) * k + +def pad_size(n, k): + """ + The smallest number that has to be added to n to equal a multiple of k. + """ + if n%k: + return k - n%k + else: + return 0 + +def is_power_of_k(n, k): + return k**int(math.log(n, k) + 0.5) == n + +def next_power_of_k(n, k): + p = 1 + while p < n: + p *= k + return p + +def ave(l): + return sum(l) / len(l) + +def log_ceil(n, b): + """ + The smallest integer k such that b^k >= n. + + log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. + the number of bits needed to store any of 128 possible values is 7. + """ + p = 1 + k = 0 + while p < n: + p *= b + k += 1 + return k + +def log_floor(n, b): + """ + The largest integer k such that b^k <= n. + """ + p = 1 + k = 0 + while p <= n: + p *= b + k += 1 + return k - 1 + +def linear_fit_slope(ps): + """ + Single-independent-variable linear regression -- least squares method. + + At least, I *think* this function computes that answer. I no longer + remember where I learned this trick and at the moment I can't prove to + myself that this is correct. + + @param ps a sequence of tuples of (x, y) + """ + avex = ave([x for (x, y) in ps]) + avey = ave([y for (x, y) in ps]) + sxy = sum([ (x - avex) * (y - avey) for (x, y) in ps ]) + sxx = sum([ (x - avex) ** 2 for (x, y) in ps ]) + if sxx == 0: + return None + return sxy / sxx + +def permute(l): + """ + Return all possible permutations of l. + + @type l: sequence + @rtype a set of sequences + """ + if len(l) == 1: + return [l,] + + res = [] + for i in range(len(l)): + l2 = list(l[:]) + x = l2.pop(i) + for l3 in permute(l2): + l3.append(x) + res.append(l3) + + return res + diff --git a/libs/pyutil/odict.py~ b/libs/pyutil/odict.py~ deleted file mode 100644 index 0ed5ce7b..00000000 --- a/libs/pyutil/odict.py~ +++ /dev/null @@ -1,552 +0,0 @@ -# Copyright (c) 2002-2009 Zooko "Zooko" Wilcox-O'Hearn - -""" -This module offers a Ordered Dict, which is a dict that preserves -insertion order. See PEP 372 for description of the problem. This -implementation uses a linked-list to get good O(1) asymptotic -performance. (Actually it is O(hashtable-update-cost), but whatever.) - -Warning: if -O optimizations are not turned on then OrderedDict performs -extensive self-analysis in every function call, which can take minutes -and minutes for a large cache. Turn on -O, or comment out assert -self._assert_invariants() -""" - -import operator - -from assertutil import _assert, precondition -from humanreadable import hr - -class OrderedDict: - """ - An efficient ordered dict. - - Adding an item that is already in the dict *does not* make it the - most- recently-added item although it may change the state of the - dict itself (if the value is different than the previous value). - - See also SmallOrderedDict (below), which is faster in some cases. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - k = self.i - precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) - (v, p, n,) = self.c.d[k] - self.i = p - return (k, v,) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - k = self.i - precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) - (v, p, n,) = self.c.d[k] - self.i = p - return k - - class ValIterator: - def __init__(self, c): - self.c = c - self.i = c.d[c.ts][1] - def __iter__(self): - return self - def next(self): - if self.i is self.c.hs: - raise StopIteration - precondition(self.c.d.has_key(self.i), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - (v, p, n,) = self.c.d[self.i] - self.i = p - return v - - class Sentinel: - def __init__(self, msg): - self.msg = msg - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.msg,) - - def __init__(self, initialdata={}): - self.d = {} # k: k, v: [v, prev, next,] # the dict - self.hs = OrderedDict.Sentinel("hs") - self.ts = OrderedDict.Sentinel("ts") - self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.update(initialdata) - - assert self._assert_invariants() - - def __repr_n__(self, n=None): - s = ["{",] - try: - iter = self.iteritems() - x = iter.next() - s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - i = 1 - while (n is None) or (i < n): - x = iter.next() - s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1])) - except StopIteration: - pass - s.append("}") - return ''.join(s) - - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) - - def __str__(self): - return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) - - def _assert_invariants(self): - _assert((len(self.d) > 2) == (self.d[self.hs][2] is not self.ts) == (self.d[self.ts][1] is not self.hs), "Head and tail point to something other than each other if and only if there is at least one element in the dictionary.", self.hs, self.ts, len(self.d)) - foundprevsentinel = 0 - foundnextsentinel = 0 - for (k, (v, p, n,)) in self.d.iteritems(): - _assert(v not in (self.hs, self.ts,)) - _assert(p is not self.ts, "A reference to the tail sentinel may not appear in prev.", k, v, p, n) - _assert(n is not self.hs, "A reference to the head sentinel may not appear in next.", k, v, p, n) - _assert(p in self.d, "Each prev is required to appear as a key in the dict.", k, v, p, n) - _assert(n in self.d, "Each next is required to appear as a key in the dict.", k, v, p, n) - if p is self.hs: - foundprevsentinel += 1 - _assert(foundprevsentinel <= 2, "No more than two references to the head sentinel may appear as a prev.", k, v, p, n) - if n is self.ts: - foundnextsentinel += 1 - _assert(foundnextsentinel <= 2, "No more than one reference to the tail sentinel may appear as a next.", k, v, p, n) - _assert(foundprevsentinel == 2, "A reference to the head sentinel is required appear as a prev (plus a self-referential reference).") - _assert(foundnextsentinel == 2, "A reference to the tail sentinel is required appear as a next (plus a self-referential reference).") - - count = 0 - for (k, v,) in self.iteritems(): - _assert(k not in (self.hs, self.ts,), k, self.hs, self.ts) - count += 1 - _assert(count == len(self.d)-2, count, len(self.d)) # -2 for the sentinels - - return True - - def move_to_most_recent(self, k, strictkey=False): - assert self._assert_invariants() - - if not self.d.has_key(k): - if strictkey: - raise KeyError, k - return - - node = self.d[k] - - # relink - self.d[node[1]][2] = node[2] - self.d[node[2]][1] = node[1] - - # move to front - hnode = self.d[self.hs] - - node[1] = self.hs - node[2] = hnode[2] - hnode[2] = k - self.d[node[2]][1] = k - - assert self._assert_invariants() - - def iteritems(self): - return OrderedDict.ItemIterator(self) - - def itervalues(self): - return OrderedDict.ValIterator(self) - - def iterkeys(self): - return self.__iter__() - - def __iter__(self): - return OrderedDict.KeyIterator(self) - - def __getitem__(self, key, default=None, strictkey=True): - node = self.d.get(key) - if not node: - if strictkey: - raise KeyError, key - return default - return node[0] - - def __setitem__(self, k, v=None): - assert self._assert_invariants() - - node = self.d.get(k) - if node: - node[0] = v - return - - hnode = self.d[self.hs] - n = hnode[2] - self.d[k] = [v, self.hs, n,] - hnode[2] = k - self.d[n][1] = k - - assert self._assert_invariants() - return v - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the value removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if self.d.has_key(key): - node = self.d[key] - # relink - self.d[node[1]][2] = node[2] - self.d[node[2]][1] = node[1] - del self.d[key] - assert self._assert_invariants() - return node[0] - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def has_key(self, key): - assert self._assert_invariants() - if self.d.has_key(key): - assert self._assert_invariants() - return True - else: - assert self._assert_invariants() - return False - - def clear(self): - assert self._assert_invariants() - self.d.clear() - self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - - for (k, v,) in otherdict.iteritems(): - assert self._assert_invariants() - self[k] = v - assert self._assert_invariants() - - def pop(self): - assert self._assert_invariants() - if len(self.d) < 2: # the +2 is for the sentinels - raise KeyError, 'popitem(): dictionary is empty' - k = self.d[self.hs][2] - self.remove(k) - assert self._assert_invariants() - return k - - def popitem(self): - assert self._assert_invariants() - if len(self.d) < 2: # the +2 is for the sentinels - raise KeyError, 'popitem(): dictionary is empty' - k = self.d[self.hs][2] - val = self.remove(k) - assert self._assert_invariants() - return (k, val,) - - def keys_unsorted(self): - assert self._assert_invariants() - t = self.d.copy() - del t[self.hs] - del t[self.ts] - assert self._assert_invariants() - return t.keys() - - def keys(self): - res = [None] * len(self) - i = 0 - for k in self.iterkeys(): - res[i] = k - i += 1 - return res - - def values_unsorted(self): - assert self._assert_invariants() - t = self.d.copy() - del t[self.hs] - del t[self.ts] - assert self._assert_invariants() - return map(operator.__getitem__, t.values(), [0]*len(t)) - - def values(self): - res = [None] * len(self) - i = 0 - for v in self.itervalues(): - res[i] = v - i += 1 - return res - - def items(self): - res = [None] * len(self) - i = 0 - for it in self.iteritems(): - res[i] = it - i += 1 - return res - - def __len__(self): - return len(self.d) - 2 - - def insert(self, key, val=None): - assert self._assert_invariants() - result = self.__setitem__(key, val) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def get(self, key, default=None): - return self.__getitem__(key, default, strictkey=False) - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - -class SmallOrderedDict(dict): - """ - SmallOrderedDict is faster than OrderedDict for small sets. How small? That - depends on your machine and which operations you use most often. Use - performance profiling to determine whether the cache class that you are - using makes any difference to the performance of your program, and if it - does, then run "quick_bench()" in test/test_cache.py to see which cache - implementation is faster for the size of your datasets. - - A simple least-recently-used cache. It keeps an LRU queue, and - when the number of items in the cache reaches maxsize, it removes - the least recently used item. - - "Looking" at an item or a key such as with "has_key()" makes that - item become the most recently used item. - - You can also use "refresh()" to explicitly make an item become the most - recently used item. - - Adding an item that is already in the dict *does* make it the - most- recently-used item although it does not change the state of - the dict itself. - """ - class ItemIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return (k, dict.__getitem__(self.c, k),) - - class KeyIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return k - - class ValueIterator: - def __init__(self, c): - self.c = c - self.i = 0 - def __iter__(self): - return self - def next(self): - precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) - precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) - if self.i == len(self.c._lru): - raise StopIteration - k = self.i - self.i += 1 - return dict.__getitem__(self.c, k) - - def __init__(self, initialdata={}, maxsize=128): - dict.__init__(self, initialdata) - self._lru = initialdata.keys() # contains keys - self._maxsize = maxsize - over = len(self) - self._maxsize - if over > 0: - map(dict.__delitem__, [self]*over, self._lru[:over]) - del self._lru[:over] - assert self._assert_invariants() - - def _assert_invariants(self): - _assert(len(self._lru) <= self._maxsize, "Size is required to be <= maxsize.") - _assert(len(filter(lambda x: dict.has_key(self, x), self._lru)) == len(self._lru), "Each key in self._lru is required to be in dict.", filter(lambda x: not dict.has_key(self, x), self._lru), len(self._lru), self._lru, len(self), self) - _assert(len(filter(lambda x: x in self._lru, self.keys())) == len(self), "Each key in dict is required to be in self._lru.", filter(lambda x: x not in self._lru, self.keys()), len(self._lru), self._lru, len(self), self) - _assert(len(self._lru) == len(self), "internal consistency", filter(lambda x: x not in self.keys(), self._lru), len(self._lru), self._lru, len(self), self) - _assert(len(self._lru) <= self._maxsize, "internal consistency", len(self._lru), self._lru, self._maxsize) - return True - - def insert(self, key, item=None): - assert self._assert_invariants() - result = self.__setitem__(key, item) - assert self._assert_invariants() - return result - - def setdefault(self, key, default=None): - assert self._assert_invariants() - if not self.has_key(key): - self[key] = default - assert self._assert_invariants() - return self[key] - - def __setitem__(self, key, item=None): - assert self._assert_invariants() - if dict.has_key(self, key): - self._lru.remove(key) - else: - if len(self._lru) == self._maxsize: - # If this insert is going to increase the size of the cache to bigger than maxsize: - killkey = self._lru.pop(0) - dict.__delitem__(self, killkey) - dict.__setitem__(self, key, item) - self._lru.append(key) - assert self._assert_invariants() - return item - - def remove(self, key, default=None, strictkey=True): - assert self._assert_invariants() - result = self.__delitem__(key, default, strictkey) - assert self._assert_invariants() - return result - - def __delitem__(self, key, default=None, strictkey=True): - """ - @param strictkey: True if you want a KeyError in the case that - key is not there, False if you want a reference to default - in the case that key is not there - @param default: the object to return if key is not there; This - is ignored if strictkey. - - @return: the object removed or default if there is not item by - that key and strictkey is False - """ - assert self._assert_invariants() - if dict.has_key(self, key): - val = dict.__getitem__(self, key) - dict.__delitem__(self, key) - self._lru.remove(key) - assert self._assert_invariants() - return val - elif strictkey: - assert self._assert_invariants() - raise KeyError, key - else: - assert self._assert_invariants() - return default - - def clear(self): - assert self._assert_invariants() - dict.clear(self) - self._lru = [] - assert self._assert_invariants() - - def update(self, otherdict): - """ - @return: self - """ - assert self._assert_invariants() - if len(otherdict) > self._maxsize: - # Handling this special case here makes it possible to implement the - # other more common cases faster below. - dict.clear(self) - self._lru = [] - if self._maxsize > (len(otherdict) - self._maxsize): - dict.update(self, otherdict) - while len(self) > self._maxsize: - dict.popitem(self) - else: - for k, v, in otherdict.iteritems(): - if len(self) == self._maxsize: - break - dict.__setitem__(self, k, v) - self._lru = dict.keys(self) - assert self._assert_invariants() - return self - - for k in otherdict.iterkeys(): - if dict.has_key(self, k): - self._lru.remove(k) - self._lru.extend(otherdict.keys()) - dict.update(self, otherdict) - - over = len(self) - self._maxsize - if over > 0: - map(dict.__delitem__, [self]*over, self._lru[:over]) - del self._lru[:over] - - assert self._assert_invariants() - return self - - def has_key(self, key): - assert self._assert_invariants() - if dict.has_key(self, key): - assert key in self._lru, "key: %s, self._lru: %s" % tuple(map(hr, (key, self._lru,))) - self._lru.remove(key) - self._lru.append(key) - assert self._assert_invariants() - return True - else: - assert self._assert_invariants() - return False - - def refresh(self, key, strictkey=True): - """ - @param strictkey: raise a KeyError exception if key isn't present - """ - assert self._assert_invariants() - if not dict.has_key(self, key): - if strictkey: - raise KeyError, key - return - self._lru.remove(key) - self._lru.append(key) - - def popitem(self): - if not self._lru: - raise KeyError, 'popitem(): dictionary is empty' - k = self._lru[-1] - obj = self.remove(k) - return (k, obj,) diff --git a/libs/pyutil/randutil.py b/libs/pyutil/randutil.py index a3efb74f..82eb3e18 100644 --- a/libs/pyutil/randutil.py +++ b/libs/pyutil/randutil.py @@ -80,6 +80,5 @@ seed = randobj.seed def randstr(n): return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) -import random as insecurerandom def insecurerandstr(n): - return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n))) + return os.urandom(n) diff --git a/libs/pyutil/randutil.py~ b/libs/pyutil/randutil.py~ deleted file mode 100644 index b0f1c4f9..00000000 --- a/libs/pyutil/randutil.py~ +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn -# This file is part of pyutil; see README.rst for licensing terms. - -import warnings -import os, random - -try: - import hashexpand - class SHA256Random(hashexpand.SHA256Expander, random.Random): - def __init__(self, seed=None, deterministic=True): - warnings.warn("deprecated", DeprecationWarning) - if not deterministic: - raise NotImplementedError, "SHA256Expander is always deterministic. For non-deterministic, try urandomRandom." - - hashexpand.SHA256Expander.__init__(self) - random.Random.__init__(self, seed) - self.seed(seed) - - def seed(self, seed=None): - if seed is None: - import increasing_timer - seed = repr(increasing_timer.time()) - hashexpand.SHA256Expander.seed(self, seed) - - - class SHA256Random(hashexpand.SHA256Expander, random.Random): - def __init__(self, seed=""): - warnings.warn("deprecated", DeprecationWarning) - hashexpand.SHA256Expander.__init__(self) - self.seed(seed) - - def seed(self, seed=None): - if seed is None: - seed = os.urandom(32) - hashexpand.SHA256Expander.seed(self, seed) -except ImportError, le: - class InsecureSHA256Random: - def __init__(self, seed=None): - raise ImportError, le - class SHA256Random: - def __init__(self, seed=""): - raise ImportError, le - -class devrandomRandom(random.Random): - """ The problem with using this one, of course, is that it blocks. This - is, of course, a security flaw. (On Linux and probably on other - systems.) --Zooko 2005-03-04 - - Not repeatable. - """ - def __init__(self): - warnings.warn("deprecated", DeprecationWarning) - self.dr = open("/dev/random", "r") - - def get(self, bytes): - return self.dr.read(bytes) - - -class devurandomRandom(random.Random): - """ The problem with using this one is that it gives answers even when it - has never been properly seeded, e.g. when you are booting from CD and have - just started up and haven't yet gathered enough entropy to actually be - unguessable. (On Linux and probably on other systems.) --Zooko 2005-03-04 - - Not repeatable. - """ - def get(self, bytes): - warnings.warn("deprecated", DeprecationWarning) - return os.urandom(bytes) - - -randobj = devurandomRandom() -get = randobj.get -random = randobj.random -randrange = randobj.randrange -shuffle = randobj.shuffle -choice = randobj.choice -seed = randobj.seed - -def randstr(n): - return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) - -import random as insecurerandom -def insecurerandstr(n): - return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n))) diff --git a/libs/pyutil/scripts/passphrase.py b/libs/pyutil/scripts/passphrase.py new file mode 100644 index 00000000..bed79c13 --- /dev/null +++ b/libs/pyutil/scripts/passphrase.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import argparse, math, random + +from pyutil.mathutil import div_ceil + +from pkg_resources import resource_stream + +def recursive_subset_sum(entropy_needed, wordlists): + # Pick a minimalish set of numbers which sum to at least + # entropy_needed. + + # Okay now what's the smallest number of words which will give us + # at least this much entropy? + entropy_of_biggest_wordlist = wordlists[-1][0] + assert isinstance(entropy_of_biggest_wordlist, float), wordlists[-1] + needed_words = div_ceil(entropy_needed, entropy_of_biggest_wordlist) + # How much entropy do we need from each word? + needed_entropy_per_word = entropy_needed / needed_words + # What's the smallest wordlist that offers at least this much + # entropy per word? + for (wlentropy, wl) in wordlists: + if wlentropy >= needed_entropy_per_word: + break + assert wlentropy >= needed_entropy_per_word, (wlentropy, needed_entropy_per_word) + + result = [(wlentropy, wl)] + # If we need more, recurse... + if wlentropy < entropy_needed: + rest = recursive_subset_sum(entropy_needed - wlentropy, wordlists) + result.extend(rest) + return result + +def gen_passphrase(entropy, allwords): + maxlenwords = [] + i = 2 # The smallest set is words of length 1 or 2. + words = [x for x in allwords if len(x) <= i] + maxlenwords.append((math.log(len(words), 2), words)) + while len(maxlenwords[-1][1]) < len(allwords): + i += 1 + words = [x for x in allwords if len(x) <= i] + maxlenwords.append((math.log(len(words), 2), words)) + + sr = random.SystemRandom() + passphrase = [] + + wordlists_to_use = recursive_subset_sum(entropy, maxlenwords) + + passphraseentropy = 0.0 + for (wle, wl) in wordlists_to_use: + passphrase.append(sr.choice(wl)) + passphraseentropy += wle + + return (u".".join(passphrase), passphraseentropy) + +def main(): + parser = argparse.ArgumentParser(prog="chbs", description="Create a random passphrase by picking a few random words.") + + parser.add_argument('-d', '--dictionary', help="what file to read a list of words from (or omit this option to use chbs's bundled dictionary)", type=argparse.FileType('rU'), metavar="DICT") + parser.add_argument('bits', help="how many bits of entropy minimum", type=float, metavar="BITS") + args = parser.parse_args() + + dicti = args.dictionary + if not dicti: + dicti = resource_stream('pyutil', 'data/wordlist.txt') + allwords = set([x.decode('utf-8').strip().lower() for x in dicti.readlines()]) + + passphrase, bits = gen_passphrase(args.bits, allwords) + + print u"Your new password is: '%s'. It is worth about %s bits." % (passphrase, bits) diff --git a/libs/pyutil/scripts/time_comparisons.py b/libs/pyutil/scripts/time_comparisons.py new file mode 100644 index 00000000..15a38852 --- /dev/null +++ b/libs/pyutil/scripts/time_comparisons.py @@ -0,0 +1,209 @@ +# If you run this file, it will make up a random secret and then crack it +# using timing information from a string comparison function. Maybe--if it +# gets lucky. It takes a long, long time to work. + +# So, the thing I need help with is statistics. The way this thing works is +# extremely stupid. Suppose you want to know which function invocation takes +# longer: comparison(secret, guess1) or comparison(secret, guess2)? + +# If you can correctly determine that one of them takes longer than the +# other, then (a) you can use that to crack the secret, and (b) this is a +# unit test demonstrating that comparison() is not timing-safe. + +# So how does this script do it? Extremely stupidly. First of all, you can't +# reliably measure tiny times, so to measure the time that a function takes, +# we run that function 10,000 times in a row, measure how long that took, and +# divide by 10,000 to estimate how long any one run would have taken. + +# Then, we do that 100 times in a row, and take the fastest of 100 runs. (I +# also experimented with taking the mean of 100 runs instead of the fastest.) + +# Then, we just say whichever comparison took longer (for its fastest run of +# 100 runs of 10,000 executions per run) is the one we think is a closer +# guess to the secret. + +# Now I would *like* to think that there is some kind of statistical analysis +# more sophisticated than "take the slowest of the fastest of 100 runs of +# 10,000 executions". Such improved statistical analysis would hopefully be +# able to answer these two questions: + +# 1. Are these two function calls -- comparison(secret, guess1) and +# comparison(secret, guess2) -- drawing from the same distribution or +# different? If you can answer that question, then you've answered the +# question of whether "comparison" is timing-safe or not. + +# And, this would also allow the cracker to recover from a false step. If it +# incorrectly decides the the prefix of the secret is ABCX, when the real +# secret is ABCD, then after that every next step it takes will be the +# "drawing from the same distribution" kind -- any difference between ABCXQ +# and ABCXR will be just due to noise, since both are equally far from the +# correct answer, which startsw with ABCD. If it could realize that there is +# no real difference between the distributions, then it could back-track and +# recover. + +# 2. Giving the ability to measure, noisily, the time taken by comparison(), +# how can you most efficiently figure out which guess takes the longest? If +# you can do that more efficiently, you can crack secrets more efficiently. + +# The script takes two arguments. The first is how many symbols in the +# secret, and the second is how big the alphabet from which the symbols are +# drawn. To prove that this script can *ever* work, try passing length 5 and +# alphabet size 2. Also try editing the code to let is use sillycomp. That'll +# definitely make it work. If you can improve this script (as per the thing +# above about "needing better statistics") to the degree that it can crack a +# secret with length 32 and alphabet size 256, then that would be awesome. + +# See the result of this commandline: + +# $ python -c 'import time_comparisons ; time_comparisons.print_measurements()' + + +from pyutil import benchutil + +import hashlib, random, os + +from decimal import Decimal +D=Decimal + +p1 = 'a'*32 +p1a = 'a'*32 +p2 = 'a'*31+'b' # close, but no cigar +p3 = 'b'*32 # different in the first byte + +def randstr(n, alphabetsize): + alphabet = [ chr(x) for x in range(alphabetsize) ] + return ''.join([random.choice(alphabet) for i in range(n)]) + +def compare(n, f, a, b): + for i in xrange(n): + f(a, b) + +def eqeqcomp(a, b): + return a == b + +def sillycomp(a, b): + # This exposes a lot of information in its timing about how many leading bytes match. + for i in range(len(a)): + if a[i] != b[i]: + return False + for i in xrange(2**9): + pass + if len(a) == len(b): + return True + else: + return False + +def hashcomp(a, b): + # Brian Warner invented this for Tahoe-LAFS. It seems like it should be very safe agaist timing leakage of any kind, because of the inclusion of a new random randkey every time. Note that exposing the value of the hash (i.e. the output of md5(randkey+secret)) is *not* a security problem. You can post that on your web site and let all attackers have it, no problem. (Provided that the value of "randkey" remains secret.) + + randkey = os.urandom(32) + return hashlib.md5(randkey+ a).digest() == hashlib.md5(randkey+b).digest() + +def xorcomp(a, b): + # This appears to be the most popular timing-insensitive string comparison function. I'm not completely sure it is fully timing-insensitive. (There are all sorts of funny things inside Python, such as caching of integer objects < 100...) + if len(a) != len(b): + return False + result = 0 + for x, y in zip(a, b): + result |= ord(x) ^ ord(y) + return result == 0 + +def print_measurements(): + N=10**4 + REPS=10**2 + + print "all times are in nanoseconds per comparison (in scientific notation)" + print + + for comparator in [eqeqcomp, hashcomp, xorcomp, sillycomp]: + print "using comparator ", comparator + + # for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]: + trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')] + random.shuffle(trials) + for (a, b, desc) in trials: + print "comparing two strings that are %s to each other" % (desc,) + + def f(n): + compare(n, comparator, a, b) + + benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) + + print + +def try_to_crack_secret(cracker, comparator, secretlen, alphabetsize): + secret = randstr(secretlen, alphabetsize) + + def test_guess(x): + return comparator(secret, x) + + print "Giving cracker %s a chance to figure out the secret. Don't tell him, but the secret is %s. Whenever he makes a guess, we'll use comparator %s to decide if his guess is right ..." % (cracker, secret.encode('hex'), comparator,) + + guess = cracker(test_guess, secretlen, alphabetsize) + + print "Cracker %s guessed %r" % (cracker, guess,) + if guess == secret: + print "HE FIGURED IT OUT!? HOW DID HE DO THAT." + else: + print "HAHA. Our secret is safe." + +def byte_at_a_time_cracker(test_guess, secretlen, alphabetsize): + # If we were cleverer, we'd add some backtracking behaviour where, if we can't find any x such that ABCx stands out from the crowd as taking longer than all the other ABCy's, then we start to think that we've taken a wrong step and we go back to trying ABy's. Make sense? But we're not that clever. Once we take a step, we don't backtrack. + + print + + guess=[] + + while len(guess) < secretlen: + best_next_byte = None + best_next_byte_time = None + + # For each possible byte... + for next_byte in range(alphabetsize): + c = chr(next_byte) + + # Construct a guess with our best candidate so far... + candidate_guess = guess[:] + + # Plus that byte... + candidate_guess.append(c) + s = ''.join(candidate_guess) + + # Plus random bytes... + s += os.urandom(32 - len(s)) + + # And see how long it takes the test_guess to consider it... + def f(n): + for i in xrange(n): + test_guess(s) + + times = benchutil.rep_bench(f, 10**7, MAXREPS=10**3, quiet=True) + + fastesttime = times['mean'] + + print "%s..."%(c.encode('hex'),), + if best_next_byte is None or fastesttime > best_next_byte_time: + print "new candidate for slowest next-char: %s, took: %s" % (c.encode('hex'), fastesttime,), + + best_next_byte_time = fastesttime + best_next_byte = c + + # Okay we've tried all possible next bytes. Our guess is this one (the one that took longest to be tested by test_guess): + guess.append(best_next_byte) + print "SLOWEST next-char %s! Current guess at secret: %s" % (best_next_byte.encode('hex'), ''.join(guess).encode('hex'),) + + guess = ''.join(guess) + print "Our guess for the secret: %r" % (guess,) + return guess + +if __name__ == '__main__': + import sys + secretlen = int(sys.argv[1]) + alphabetsize = int(sys.argv[2]) + if alphabetsize > 256: + raise Exception("We assume we can fit one element of the alphabet into a byte.") + + print "secretlen: %d, alphabetsize: %d" % (secretlen, alphabetsize,) + + # try_to_crack_secret(byte_at_a_time_cracker, sillycomp, secretlen, alphabetsize) + try_to_crack_secret(byte_at_a_time_cracker, eqeqcomp, secretlen, alphabetsize) diff --git a/libs/pyutil/test/current/test_mathutil.py b/libs/pyutil/test/current/test_mathutil.py index 7c189dfb..da788758 100644 --- a/libs/pyutil/test/current/test_mathutil.py +++ b/libs/pyutil/test/current/test_mathutil.py @@ -42,6 +42,13 @@ class MathUtilTestCase(unittest.TestCase): self.failUnlessEqual(f(5, 3), 2) self.failUnlessEqual(f(6, 3), 2) self.failUnlessEqual(f(7, 3), 3) + self.failUnless(isinstance(f(0.0, 1), int)) + self.failUnlessEqual(f(7.0, 3.0), 3) + self.failUnlessEqual(f(7, 3.0), 3) + self.failUnlessEqual(f(7.0, 3), 3) + self.failUnlessEqual(f(6.0, 3.0), 2) + self.failUnlessEqual(f(6.0, 3), 2) + self.failUnlessEqual(f(6, 3.0), 2) def test_next_multiple(self): f = mathutil.next_multiple diff --git a/libs/pyutil/test/current/test_mathutil.py~ b/libs/pyutil/test/current/test_mathutil.py~ new file mode 100644 index 00000000..7c189dfb --- /dev/null +++ b/libs/pyutil/test/current/test_mathutil.py~ @@ -0,0 +1,135 @@ +#!/usr/bin/env python + +import unittest + +from pyutil import mathutil +from pyutil.assertutil import _assert + +class MathUtilTestCase(unittest.TestCase): + def _help_test_is_power_of_k(self, k): + for i in range(2, 40): + _assert(mathutil.is_power_of_k(k**i, k), k, i) + + def test_is_power_of_k(self): + for i in range(2, 5): + self._help_test_is_power_of_k(i) + + def test_log_ceil(self): + f = mathutil.log_ceil + self.failUnlessEqual(f(1, 2), 0) + self.failUnlessEqual(f(1, 3), 0) + self.failUnlessEqual(f(2, 2), 1) + self.failUnlessEqual(f(2, 3), 1) + self.failUnlessEqual(f(3, 2), 2) + + def test_log_floor(self): + f = mathutil.log_floor + self.failUnlessEqual(f(1, 2), 0) + self.failUnlessEqual(f(1, 3), 0) + self.failUnlessEqual(f(2, 2), 1) + self.failUnlessEqual(f(2, 3), 0) + self.failUnlessEqual(f(3, 2), 1) + + def test_div_ceil(self): + f = mathutil.div_ceil + self.failUnlessEqual(f(0, 1), 0) + self.failUnlessEqual(f(0, 2), 0) + self.failUnlessEqual(f(0, 3), 0) + self.failUnlessEqual(f(1, 3), 1) + self.failUnlessEqual(f(2, 3), 1) + self.failUnlessEqual(f(3, 3), 1) + self.failUnlessEqual(f(4, 3), 2) + self.failUnlessEqual(f(5, 3), 2) + self.failUnlessEqual(f(6, 3), 2) + self.failUnlessEqual(f(7, 3), 3) + + def test_next_multiple(self): + f = mathutil.next_multiple + self.failUnlessEqual(f(5, 1), 5) + self.failUnlessEqual(f(5, 2), 6) + self.failUnlessEqual(f(5, 3), 6) + self.failUnlessEqual(f(5, 4), 8) + self.failUnlessEqual(f(5, 5), 5) + self.failUnlessEqual(f(5, 6), 6) + self.failUnlessEqual(f(32, 1), 32) + self.failUnlessEqual(f(32, 2), 32) + self.failUnlessEqual(f(32, 3), 33) + self.failUnlessEqual(f(32, 4), 32) + self.failUnlessEqual(f(32, 5), 35) + self.failUnlessEqual(f(32, 6), 36) + self.failUnlessEqual(f(32, 7), 35) + self.failUnlessEqual(f(32, 8), 32) + self.failUnlessEqual(f(32, 9), 36) + self.failUnlessEqual(f(32, 10), 40) + self.failUnlessEqual(f(32, 11), 33) + self.failUnlessEqual(f(32, 12), 36) + self.failUnlessEqual(f(32, 13), 39) + self.failUnlessEqual(f(32, 14), 42) + self.failUnlessEqual(f(32, 15), 45) + self.failUnlessEqual(f(32, 16), 32) + self.failUnlessEqual(f(32, 17), 34) + self.failUnlessEqual(f(32, 18), 36) + self.failUnlessEqual(f(32, 589), 589) + + def test_pad_size(self): + f = mathutil.pad_size + self.failUnlessEqual(f(0, 4), 0) + self.failUnlessEqual(f(1, 4), 3) + self.failUnlessEqual(f(2, 4), 2) + self.failUnlessEqual(f(3, 4), 1) + self.failUnlessEqual(f(4, 4), 0) + self.failUnlessEqual(f(5, 4), 3) + + def test_is_power_of_k_part_2(self): + f = mathutil.is_power_of_k + for i in range(1, 100): + if i in (1, 2, 4, 8, 16, 32, 64): + self.failUnless(f(i, 2), "but %d *is* a power of 2" % i) + else: + self.failIf(f(i, 2), "but %d is *not* a power of 2" % i) + for i in range(1, 100): + if i in (1, 3, 9, 27, 81): + self.failUnless(f(i, 3), "but %d *is* a power of 3" % i) + else: + self.failIf(f(i, 3), "but %d is *not* a power of 3" % i) + + def test_next_power_of_k(self): + f = mathutil.next_power_of_k + self.failUnlessEqual(f(0,2), 1) + self.failUnlessEqual(f(1,2), 1) + self.failUnlessEqual(f(2,2), 2) + self.failUnlessEqual(f(3,2), 4) + self.failUnlessEqual(f(4,2), 4) + for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i) + for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i) + for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i) + for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i) + for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i) + + self.failUnlessEqual(f(0,3), 1) + self.failUnlessEqual(f(1,3), 1) + self.failUnlessEqual(f(2,3), 3) + self.failUnlessEqual(f(3,3), 3) + for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i) + for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i) + for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i) + for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i) + + def test_ave(self): + f = mathutil.ave + self.failUnlessEqual(f([1,2,3]), 2) + self.failUnlessEqual(f([0,0,0,4]), 1) + self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) + + def failUnlessEqualContents(self, a, b): + self.failUnlessEqual(sorted(a), sorted(b)) + + def test_permute(self): + f = mathutil.permute + self.failUnlessEqualContents(f([]), []) + self.failUnlessEqualContents(f([1]), [[1]]) + self.failUnlessEqualContents(f([1,2]), [[1,2], [2,1]]) + self.failUnlessEqualContents(f([1,2,3]), + [[1,2,3], [1,3,2], + [2,1,3], [2,3,1], + [3,1,2], [3,2,1]]) diff --git a/libs/pyutil/time_comparisons.py b/libs/pyutil/time_comparisons.py deleted file mode 100644 index ee1bcfa0..00000000 --- a/libs/pyutil/time_comparisons.py +++ /dev/null @@ -1,44 +0,0 @@ -from pyutil import benchutil - -import hashlib, random, os - -from decimal import Decimal -D=Decimal - -p1 = 'a'*32 -p1a = 'a'*32 -p2 = 'a'*31+'b' # close, but no cigar -p3 = 'b'*32 # different in the first byte - -def compare(n, f, a, b): - for i in xrange(n): - f(a, b) - -def eqeqcomp(a, b): - return a == b - -def hashcomp(a, b): - salt = os.urandom(32) - return hashlib.md5(salt+ a).digest() == hashlib.md5(salt+b).digest() - -N=10**4 -REPS=10**2 - -print "all times are in nanoseconds per comparison (scientific notation)" -print - -for comparator in [eqeqcomp, hashcomp]: - print "using comparator ", comparator - - # for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]: - trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')] - random.shuffle(trials) - for (a, b, desc) in trials: - print "comparing two strings that are %s to each other" % (desc,) - - def f(n): - compare(n, comparator, a, b) - - benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - - print diff --git a/libs/pyutil/time_comparisons.py~ b/libs/pyutil/time_comparisons.py~ deleted file mode 100644 index abf151ad..00000000 --- a/libs/pyutil/time_comparisons.py~ +++ /dev/null @@ -1,72 +0,0 @@ -from pyutil import benchutil - -import hashlib -import os - -from decimal import Decimal -D=Decimal - -p1 = 'a'*32 -p1a = 'a'*32 -p2 = 'a'*31+'b' # close, but no cigar -p3 = 'b'*32 # different in the first byte - -def compare(n, f, a, b): - for i in xrange(n): - f(a, b) - -def eqeq(a, b): - return a == b - -def equalsequals_s(n): - # return compare(n, eqeq, - for i in xrange(n): - p1 == p1a - -def equalsequals_c(n): - for i in xrange(n): - p1 == p2 - -def equalsequals_f(n): - for i in xrange(n): - p1 == p3 - -def hash_s(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p1a).digest() - -def hash_c(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p2).digest() - -def hash_f(n): - for i in xrange(n): - salt = os.urandom(32) - hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p3).digest() - -N=10**4 -REPS=10**2 - -print "using '=='" - -print "same" -benchutil.rep_bench(equalsequals_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "close" -benchutil.rep_bench(equalsequals_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "far" -benchutil.rep_bench(equalsequals_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "using hash" - -print "same" -benchutil.rep_bench(hash_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "far" -benchutil.rep_bench(hash_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) - -print "close" -benchutil.rep_bench(hash_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS) diff --git a/libs/tornado/auth.py b/libs/tornado/auth.py index 0cbfa7c0..a2cef356 100755 --- a/libs/tornado/auth.py +++ b/libs/tornado/auth.py @@ -549,7 +549,7 @@ class OAuth2Mixin(object): @return_future def authorize_redirect(self, redirect_uri=None, client_id=None, client_secret=None, extra_params=None, - callback=None): + callback=None, scope=None, response_type="code"): """Redirects the user to obtain OAuth authorization for this service. Some providers require that you register a redirect URL with @@ -566,10 +566,13 @@ class OAuth2Mixin(object): """ args = { "redirect_uri": redirect_uri, - "client_id": client_id + "client_id": client_id, + "response_type": response_type } if extra_params: args.update(extra_params) + if scope: + args['scope'] = ' '.join(scope) self.redirect( url_concat(self._OAUTH_AUTHORIZE_URL, args)) callback() @@ -945,6 +948,67 @@ class GoogleMixin(OpenIdMixin, OAuthMixin): return OpenIdMixin.get_authenticated_user(self) +class GoogleOAuth2Mixin(OAuth2Mixin): + """Google authentication using OAuth2.""" + _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/auth" + _OAUTH_ACCESS_TOKEN_URL = "https://accounts.google.com/o/oauth2/token" + _OAUTH_NO_CALLBACKS = False + _OAUTH_SETTINGS_KEY = 'google_oauth' + + @_auth_return_future + def get_authenticated_user(self, redirect_uri, code, callback): + """Handles the login for the Google user, returning a user object. + + Example usage:: + + class GoogleOAuth2LoginHandler(LoginHandler, tornado.auth.GoogleOAuth2Mixin): + @tornado.web.asynchronous + @tornado.gen.coroutine + def get(self): + if self.get_argument("code", False): + user = yield self.get_authenticated_user( + redirect_uri='http://your.site.com/auth/google', + code=self.get_argument("code")) + # Save the user with e.g. set_secure_cookie + else: + yield self.authorize_redirect( + redirect_uri='http://your.site.com/auth/google', + client_id=self.settings["google_consumer_key"], + scope=['openid', 'email'], + response_type='code', + extra_params={"approval_prompt": "auto"}) + """ + http = self.get_auth_http_client() + body = urllib_parse.urlencode({ + "redirect_uri": redirect_uri, + "code": code, + "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'], + "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'], + "grant_type": "authorization_code", + }) + + http.fetch(self._OAUTH_ACCESS_TOKEN_URL, + self.async_callback(self._on_access_token, callback), + method="POST", headers={'Content-Type': 'application/x-www-form-urlencoded'}, body=body) + + def _on_access_token(self, future, response): + """Callback function for the exchange to the access token.""" + if response.error: + future.set_exception(AuthError('Google auth error: %s' % str(response))) + return + + args = escape.json_decode(response.body) + future.set_result(args) + + def get_auth_http_client(self): + """Returns the `.AsyncHTTPClient` instance to be used for auth requests. + + May be overridden by subclasses to use an HTTP client other than + the default. + """ + return httpclient.AsyncHTTPClient() + + class FacebookMixin(object): """Facebook Connect authentication. diff --git a/libs/tornado/autoreload.py b/libs/tornado/autoreload.py index 05754299..79cccb49 100755 --- a/libs/tornado/autoreload.py +++ b/libs/tornado/autoreload.py @@ -16,11 +16,15 @@ """xAutomatically restart the server when a source file is modified. -Most applications should not access this module directly. Instead, pass the -keyword argument ``debug=True`` to the `tornado.web.Application` constructor. -This will enable autoreload mode as well as checking for changes to templates -and static resources. Note that restarting is a destructive operation -and any requests in progress will be aborted when the process restarts. +Most applications should not access this module directly. Instead, +pass the keyword argument ``autoreload=True`` to the +`tornado.web.Application` constructor (or ``debug=True``, which +enables this setting and several others). This will enable autoreload +mode as well as checking for changes to templates and static +resources. Note that restarting is a destructive operation and any +requests in progress will be aborted when the process restarts. (If +you want to disable autoreload while using other debug-mode features, +pass both ``debug=True`` and ``autoreload=False``). This module can also be used as a command-line wrapper around scripts such as unit test runners. See the `main` method for details. @@ -38,6 +42,7 @@ Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) because it re-executes Python using ``sys.executable`` and ``sys.argv``. Additionally, modifying these variables will cause reloading to behave incorrectly. + """ from __future__ import absolute_import, division, print_function, with_statement diff --git a/libs/tornado/curl_httpclient.py b/libs/tornado/curl_httpclient.py index e0900569..cb97710a 100755 --- a/libs/tornado/curl_httpclient.py +++ b/libs/tornado/curl_httpclient.py @@ -360,6 +360,7 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') + curl.unsetopt(pycurl.PROXYUSERPWD) if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) @@ -382,6 +383,8 @@ def _curl_setup_request(curl, request, buffer, headers): # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) + else: + curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) # Set the request method through curl's irritating interface which makes # up names for almost every single method @@ -404,6 +407,11 @@ def _curl_setup_request(curl, request, buffer, headers): # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): + if request.body is None: + raise AssertionError( + 'Body must not be empty for "%s" request' + % request.method) + request_buffer = BytesIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": @@ -414,6 +422,9 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) + elif request.method == "GET": + if request.body is not None: + raise AssertionError('Body must be empty for GET request') if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') diff --git a/libs/tornado/gen.py b/libs/tornado/gen.py index 92b7458e..217ebdf5 100755 --- a/libs/tornado/gen.py +++ b/libs/tornado/gen.py @@ -38,8 +38,8 @@ since it is both shorter and provides better exception handling):: def get(self): yield gen.Task(AsyncHTTPClient().fetch, "http://example.com") -You can also yield a list of ``Futures`` and/or ``Tasks``, which will be -started at the same time and run in parallel; a list of results will +You can also yield a list or dict of ``Futures`` and/or ``Tasks``, which will be +started at the same time and run in parallel; a list or dict of results will be returned when they are all finished:: @gen.coroutine @@ -47,6 +47,13 @@ be returned when they are all finished:: http_client = AsyncHTTPClient() response1, response2 = yield [http_client.fetch(url1), http_client.fetch(url2)] + response_dict = yield dict(response3=http_client.fetch(url3), + response4=http_client.fetch(url4)) + response3 = response_dict['response3'] + response4 = response_dict['response4'] + +.. versionchanged:: 3.2 + Dict support added. For more complicated interfaces, `Task` can be split into two parts: `Callback` and `Wait`:: @@ -404,6 +411,10 @@ class Multi(YieldPoint): a list of ``YieldPoints``. """ def __init__(self, children): + self.keys = None + if isinstance(children, dict): + self.keys = list(children.keys()) + children = children.values() self.children = [] for i in children: if isinstance(i, Future): @@ -423,7 +434,11 @@ class Multi(YieldPoint): return not self.unfinished_children def get_result(self): - return [i.get_result() for i in self.children] + result = (i.get_result() for i in self.children) + if self.keys is not None: + return dict(zip(self.keys, result)) + else: + return list(result) class _NullYieldPoint(YieldPoint): @@ -523,7 +538,7 @@ class Runner(object): self.finished = True self.yield_point = _null_yield_point raise - if isinstance(yielded, list): + if isinstance(yielded, (list, dict)): yielded = Multi(yielded) elif isinstance(yielded, Future): yielded = YieldFuture(yielded) diff --git a/libs/tornado/httpclient.py b/libs/tornado/httpclient.py index 67675894..b58a8348 100755 --- a/libs/tornado/httpclient.py +++ b/libs/tornado/httpclient.py @@ -282,7 +282,8 @@ class HTTPRequest(object): :arg int max_redirects: Limit for ``follow_redirects`` :arg string user_agent: String to send as ``User-Agent`` header :arg bool use_gzip: Request gzip encoding from the server - :arg string network_interface: Network interface to use for request + :arg string network_interface: Network interface to use for request. + ``curl_httpclient`` only; see note below. :arg callable streaming_callback: If set, ``streaming_callback`` will be run with each chunk of data as it is received, and ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in @@ -310,14 +311,26 @@ class HTTPRequest(object): :arg bool validate_cert: For HTTPS requests, validate the server's certificate? :arg string ca_certs: filename of CA certificates in PEM format, - or None to use defaults. Note that in ``curl_httpclient``, if - any request uses a custom ``ca_certs`` file, they all must (they - don't have to all use the same ``ca_certs``, but it's not possible - to mix requests with ``ca_certs`` and requests that use the defaults. + or None to use defaults. See note below when used with + ``curl_httpclient``. :arg bool allow_ipv6: Use IPv6 when available? Default is false in ``simple_httpclient`` and true in ``curl_httpclient`` - :arg string client_key: Filename for client SSL key, if any - :arg string client_cert: Filename for client SSL certificate, if any + :arg string client_key: Filename for client SSL key, if any. See + note below when used with ``curl_httpclient``. + :arg string client_cert: Filename for client SSL certificate, if any. + See note below when used with ``curl_httpclient``. + + .. note:: + + When using ``curl_httpclient`` certain options may be + inherited by subsequent fetches because ``pycurl`` does + not allow them to be cleanly reset. This applies to the + ``ca_certs``, ``client_key``, ``client_cert``, and + ``network_interface`` arguments. If you use these + options, you should pass them on every request (you don't + have to always use the same values, but it's not possible + to mix requests that specify these options with ones that + use the defaults). .. versionadded:: 3.1 The ``auth_mode`` argument. @@ -372,6 +385,9 @@ class HTTPResponse(object): * headers: `tornado.httputil.HTTPHeaders` object + * effective_url: final location of the resource after following any + redirects + * buffer: ``cStringIO`` object for response body * body: response body as string (created on demand from ``self.buffer``) diff --git a/libs/tornado/httpserver.py b/libs/tornado/httpserver.py index d005545e..34e7b768 100755 --- a/libs/tornado/httpserver.py +++ b/libs/tornado/httpserver.py @@ -29,6 +29,7 @@ from __future__ import absolute_import, division, print_function, with_statement import socket import ssl import time +import copy from tornado.escape import native_str, parse_qs_bytes from tornado import httputil @@ -326,8 +327,8 @@ class HTTPConnection(object): self.request_callback(self._request) except _BadRequestException as e: - gen_log.info("Malformed HTTP request from %s: %s", - self.address[0], e) + gen_log.info("Malformed HTTP request from %r: %s", + self.address, e) self.close() return @@ -336,7 +337,10 @@ class HTTPConnection(object): if self._request.method in ("POST", "PATCH", "PUT"): httputil.parse_body_arguments( self._request.headers.get("Content-Type", ""), data, - self._request.arguments, self._request.files) + self._request.body_arguments, self._request.files) + + for k, v in self._request.body_arguments.items(): + self._request.arguments.setdefault(k, []).extend(v) self.request_callback(self._request) @@ -403,6 +407,20 @@ class HTTPRequest(object): `.RequestHandler.get_argument`, which returns argument values as unicode strings. + .. attribute:: query_arguments + + Same format as ``arguments``, but contains only arguments extracted + from the query string. + + .. versionadded:: 3.2 + + .. attribute:: body_arguments + + Same format as ``arguments``, but contains only arguments extracted + from the request body. + + .. versionadded:: 3.2 + .. attribute:: files File uploads are available in the files property, which maps file @@ -457,6 +475,8 @@ class HTTPRequest(object): self.path, sep, self.query = uri.partition('?') self.arguments = parse_qs_bytes(self.query, keep_blank_values=True) + self.query_arguments = copy.deepcopy(self.arguments) + self.body_arguments = {} def supports_http_1_1(self): """Returns True if this request supports HTTP/1.1 semantics""" diff --git a/libs/tornado/httputil.py b/libs/tornado/httputil.py index 3e7337d9..2575bc56 100755 --- a/libs/tornado/httputil.py +++ b/libs/tornado/httputil.py @@ -320,7 +320,11 @@ def parse_body_arguments(content_type, body, arguments, files): with the parsed contents. """ if content_type.startswith("application/x-www-form-urlencoded"): - uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + try: + uri_arguments = parse_qs_bytes(native_str(body), keep_blank_values=True) + except Exception as e: + gen_log.warning('Invalid x-www-form-urlencoded body: %s', e) + uri_arguments = {} for name, values in uri_arguments.items(): if values: arguments.setdefault(name, []).extend(values) diff --git a/libs/tornado/ioloop.py b/libs/tornado/ioloop.py index 91ee2c5b..a36ab7a5 100755 --- a/libs/tornado/ioloop.py +++ b/libs/tornado/ioloop.py @@ -676,8 +676,7 @@ class PollIOLoop(IOLoop): while self._events: fd, events = self._events.popitem() try: - if self._handlers.has_key(fd): - self._handlers[fd](fd, events) + self._handlers[fd](fd, events) except (OSError, IOError) as e: if e.args[0] == errno.EPIPE: # Happens when the client closes the connection diff --git a/libs/tornado/iostream.py b/libs/tornado/iostream.py index 6bdc6397..08430cea 100755 --- a/libs/tornado/iostream.py +++ b/libs/tornado/iostream.py @@ -774,7 +774,7 @@ class IOStream(BaseIOStream): # Sometimes setsockopt will fail if the socket is closed # at the wrong time. This can happen with HTTPServer # resetting the value to false between requests. - if e.errno != errno.EINVAL: + if e.errno not in (errno.EINVAL, errno.ECONNRESET): raise diff --git a/libs/tornado/log.py b/libs/tornado/log.py index fa11f379..648db5c6 100755 --- a/libs/tornado/log.py +++ b/libs/tornado/log.py @@ -51,7 +51,7 @@ gen_log = logging.getLogger("tornado.general") def _stderr_supports_color(): color = False - if curses and sys.stderr.isatty(): + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): try: curses.setupterm() if curses.tigetnum("colors") > 0: diff --git a/libs/tornado/netutil.py b/libs/tornado/netutil.py index 9dc8506e..21db4755 100755 --- a/libs/tornado/netutil.py +++ b/libs/tornado/netutil.py @@ -20,7 +20,6 @@ from __future__ import absolute_import, division, print_function, with_statement import errno import os -import re import socket import ssl import stat @@ -30,6 +29,13 @@ from tornado.ioloop import IOLoop from tornado.platform.auto import set_close_exec from tornado.util import Configurable +if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ + ssl_match_hostname = ssl.match_hostname + SSLCertificateError = ssl.CertificateError +else: + import backports.ssl_match_hostname + ssl_match_hostname = backports.ssl_match_hostname.match_hostname + SSLCertificateError = backports.ssl_match_hostname.CertificateError def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None): """Creates listening sockets bound to the given port and address. @@ -391,73 +397,3 @@ def ssl_wrap_socket(socket, ssl_options, server_hostname=None, **kwargs): return context.wrap_socket(socket, **kwargs) else: return ssl.wrap_socket(socket, **dict(context, **kwargs)) - -if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'): # python 3.2+ - ssl_match_hostname = ssl.match_hostname - SSLCertificateError = ssl.CertificateError -else: - # match_hostname was added to the standard library ssl module in python 3.2. - # The following code was backported for older releases and copied from - # https://bitbucket.org/brandon/backports.ssl_match_hostname - class SSLCertificateError(ValueError): - pass - - def _dnsname_to_pat(dn, max_wildcards=1): - pats = [] - for frag in dn.split(r'.'): - if frag.count('*') > max_wildcards: - # Issue #17980: avoid denials of service by refusing more - # than one wildcard per fragment. A survery of established - # policy among SSL implementations showed it to be a - # reasonable choice. - raise SSLCertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) - if frag == '*': - # When '*' is a fragment by itself, it matches a non-empty dotless - # fragment. - pats.append('[^.]+') - else: - # Otherwise, '*' matches any dotless fragment. - frag = re.escape(frag) - pats.append(frag.replace(r'\*', '[^.]*')) - return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) - - def ssl_match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules - are mostly followed, but IP addresses are not accepted for *hostname*. - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate") - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_to_pat(value).match(hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise SSLCertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise SSLCertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise SSLCertificateError("no appropriate commonName or " - "subjectAltName fields were found") diff --git a/libs/tornado/platform/asyncio.py b/libs/tornado/platform/asyncio.py new file mode 100644 index 00000000..a8f5bad4 --- /dev/null +++ b/libs/tornado/platform/asyncio.py @@ -0,0 +1,134 @@ +"""Bridges between the `asyncio` module and Tornado IOLoop. + +This is a work in progress and interfaces are subject to change. + +To test: +python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOLoop +python3.4 -m tornado.test.runtests --ioloop=tornado.platform.asyncio.AsyncIOMainLoop +(the tests log a few warnings with AsyncIOMainLoop because they leave some +unfinished callbacks on the event loop that fail when it resumes) +""" +import asyncio +import datetime +import functools +import os + +from tornado.ioloop import IOLoop +from tornado import stack_context + +class BaseAsyncIOLoop(IOLoop): + def initialize(self, asyncio_loop, close_loop=False): + self.asyncio_loop = asyncio_loop + self.close_loop = close_loop + self.asyncio_loop.call_soon(self.make_current) + # Maps fd to handler function (as in IOLoop.add_handler) + self.handlers = {} + # Set of fds listening for reads/writes + self.readers = set() + self.writers = set() + self.closing = False + + def close(self, all_fds=False): + self.closing = True + for fd in list(self.handlers): + self.remove_handler(fd) + if all_fds: + os.close(fd) + if self.close_loop: + self.asyncio_loop.close() + + def add_handler(self, fd, handler, events): + if fd in self.handlers: + raise ValueError("fd %d added twice" % fd) + self.handlers[fd] = stack_context.wrap(handler) + if events & IOLoop.READ: + self.asyncio_loop.add_reader( + fd, self._handle_events, fd, IOLoop.READ) + self.readers.add(fd) + if events & IOLoop.WRITE: + self.asyncio_loop.add_writer( + fd, self._handle_events, fd, IOLoop.WRITE) + self.writers.add(fd) + + def update_handler(self, fd, events): + if events & IOLoop.READ: + if fd not in self.readers: + self.asyncio_loop.add_reader( + fd, self._handle_events, fd, IOLoop.READ) + self.readers.add(fd) + else: + if fd in self.readers: + self.asyncio_loop.remove_reader(fd) + self.readers.remove(fd) + if events & IOLoop.WRITE: + if fd not in self.writers: + self.asyncio_loop.add_writer( + fd, self._handle_events, fd, IOLoop.WRITE) + self.writers.add(fd) + else: + if fd in self.writers: + self.asyncio_loop.remove_writer(fd) + self.writers.remove(fd) + + def remove_handler(self, fd): + if fd not in self.handlers: + return + if fd in self.readers: + self.asyncio_loop.remove_reader(fd) + self.readers.remove(fd) + if fd in self.writers: + self.asyncio_loop.remove_writer(fd) + self.writers.remove(fd) + del self.handlers[fd] + + def _handle_events(self, fd, events): + self.handlers[fd](fd, events) + + def start(self): + self.asyncio_loop.run_forever() + + def stop(self): + self.asyncio_loop.stop() + + def _run_callback(self, callback, *args, **kwargs): + try: + callback(*args, **kwargs) + except Exception: + self.handle_callback_exception(callback) + + def add_timeout(self, deadline, callback): + if isinstance(deadline, (int, float)): + delay = max(deadline - self.time(), 0) + elif isinstance(deadline, datetime.timedelta): + delay = deadline.total_seconds() + else: + raise TypeError("Unsupported deadline %r", deadline) + return self.asyncio_loop.call_later(delay, self._run_callback, + stack_context.wrap(callback)) + + def remove_timeout(self, timeout): + timeout.cancel() + + def add_callback(self, callback, *args, **kwargs): + if self.closing: + raise RuntimeError("IOLoop is closing") + if kwargs: + self.asyncio_loop.call_soon_threadsafe(functools.partial( + self._run_callback, stack_context.wrap(callback), + *args, **kwargs)) + else: + self.asyncio_loop.call_soon_threadsafe( + self._run_callback, stack_context.wrap(callback), *args) + + add_callback_from_signal = add_callback + + +class AsyncIOMainLoop(BaseAsyncIOLoop): + def initialize(self): + super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), + close_loop=False) + +class AsyncIOLoop(BaseAsyncIOLoop): + def initialize(self): + super(AsyncIOLoop, self).initialize(asyncio.new_event_loop(), + close_loop=True) diff --git a/libs/tornado/process.py b/libs/tornado/process.py index ffd2d29d..942c5c3f 100755 --- a/libs/tornado/process.py +++ b/libs/tornado/process.py @@ -92,7 +92,8 @@ def fork_processes(num_processes, max_restarts=100): between any server code. Note that multiple processes are not compatible with the autoreload - module (or the debug=True option to `tornado.web.Application`). + module (or the ``autoreload=True`` option to `tornado.web.Application` + which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``fork_processes``. diff --git a/libs/tornado/simple_httpclient.py b/libs/tornado/simple_httpclient.py index d8dbb271..2558ada8 100755 --- a/libs/tornado/simple_httpclient.py +++ b/libs/tornado/simple_httpclient.py @@ -72,6 +72,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.max_clients = max_clients self.queue = collections.deque() self.active = {} + self.waiting = {} self.max_buffer_size = max_buffer_size if resolver: self.resolver = resolver @@ -89,7 +90,16 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): self.resolver.close() def fetch_impl(self, request, callback): - self.queue.append((request, callback)) + key = object() + self.queue.append((key, request, callback)) + if not len(self.active) < self.max_clients: + timeout_handle = self.io_loop.add_timeout( + self.io_loop.time() + min(request.connect_timeout, + request.request_timeout), + functools.partial(self._on_timeout, key)) + else: + timeout_handle = None + self.waiting[key] = (request, callback, timeout_handle) self._process_queue() if self.queue: gen_log.debug("max_clients limit reached, request queued. " @@ -99,8 +109,10 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): def _process_queue(self): with stack_context.NullContext(): while self.queue and len(self.active) < self.max_clients: - request, callback = self.queue.popleft() - key = object() + key, request, callback = self.queue.popleft() + if key not in self.waiting: + continue + self._remove_timeout(key) self.active[key] = (request, callback) release_callback = functools.partial(self._release_fetch, key) self._handle_request(request, release_callback, callback) @@ -113,6 +125,22 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): del self.active[key] self._process_queue() + def _remove_timeout(self, key): + if key in self.waiting: + request, callback, timeout_handle = self.waiting[key] + if timeout_handle is not None: + self.io_loop.remove_timeout(timeout_handle) + del self.waiting[key] + + def _on_timeout(self, key): + request, callback, timeout_handle = self.waiting[key] + self.queue.remove((key, request, callback)) + timeout_response = HTTPResponse( + request, 599, error=HTTPError(599, "Timeout"), + request_time=self.io_loop.time() - request.start_time) + self.io_loop.add_callback(callback, timeout_response) + del self.waiting[key] + class _HTTPConnection(object): _SUPPORTED_METHODS = set(["GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]) @@ -162,15 +190,18 @@ class _HTTPConnection(object): # so restrict to ipv4 by default. af = socket.AF_INET + timeout = min(self.request.connect_timeout, self.request.request_timeout) + if timeout: + self._timeout = self.io_loop.add_timeout( + self.start_time + timeout, + stack_context.wrap(self._on_timeout)) self.resolver.resolve(host, port, af, callback=self._on_resolve) def _on_resolve(self, addrinfo): + if self.final_callback is None: + # final_callback is cleared if we've hit our timeout + return self.stream = self._create_stream(addrinfo) - timeout = min(self.request.connect_timeout, self.request.request_timeout) - if timeout: - self._timeout = self.io_loop.add_timeout( - self.start_time + timeout, - stack_context.wrap(self._on_timeout)) self.stream.set_close_callback(self._on_close) # ipv6 addresses are broken (in self.parsed.hostname) until # 2.7, here is correctly parsed value calculated in __init__ @@ -199,10 +230,10 @@ class _HTTPConnection(object): # the SSL_OP_NO_SSLv2, but that wasn't exposed to python # until 3.2. Python 2.7 adds the ciphers argument, which # can also be used to disable SSLv2. As a last resort - # on python 2.6, we set ssl_version to SSLv3. This is + # on python 2.6, we set ssl_version to TLSv1. This is # more narrow than we'd like since it also breaks - # compatibility with servers configured for TLSv1 only, - # but nearly all servers support SSLv3: + # compatibility with servers configured for SSLv3 only, + # but nearly all servers support both SSLv3 and TLSv1: # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html if sys.version_info >= (2, 7): ssl_options["ciphers"] = "DEFAULT:!SSLv2" @@ -210,7 +241,7 @@ class _HTTPConnection(object): # This is really only necessary for pre-1.0 versions # of openssl, but python 2.6 doesn't expose version # information. - ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 + ssl_options["ssl_version"] = ssl.PROTOCOL_TLSv1 return SSLIOStream(socket.socket(af), io_loop=self.io_loop, @@ -233,6 +264,8 @@ class _HTTPConnection(object): def _on_connect(self): self._remove_timeout() + if self.final_callback is None: + return if self.request.request_timeout: self._timeout = self.io_loop.add_timeout( self.start_time + self.request.request_timeout, @@ -269,9 +302,15 @@ class _HTTPConnection(object): self.request.headers["User-Agent"] = self.request.user_agent if not self.request.allow_nonstandard_methods: if self.request.method in ("POST", "PATCH", "PUT"): - assert self.request.body is not None + if self.request.body is None: + raise AssertionError( + 'Body must not be empty for "%s" request' + % self.request.method) else: - assert self.request.body is None + if self.request.body is not None: + raise AssertionError( + 'Body must be empty for "%s" request' + % self.request.method) if self.request.body is not None: self.request.headers["Content-Length"] = str(len( self.request.body)) diff --git a/libs/tornado/speedups.c b/libs/tornado/speedups.c new file mode 100644 index 00000000..8a316c58 --- /dev/null +++ b/libs/tornado/speedups.c @@ -0,0 +1,49 @@ +#include <Python.h> + +static PyObject* websocket_mask(PyObject* self, PyObject* args) { + const char* mask; + int mask_len; + const char* data; + int data_len; + int i; + + if (!PyArg_ParseTuple(args, "s#s#", &mask, &mask_len, &data, &data_len)) { + return NULL; + } + + PyObject* result = PyBytes_FromStringAndSize(NULL, data_len); + if (!result) { + return NULL; + } + char* buf = PyBytes_AsString(result); + for (i = 0; i < data_len; i++) { + buf[i] = data[i] ^ mask[i % 4]; + } + + return result; +} + +static PyMethodDef methods[] = { + {"websocket_mask", websocket_mask, METH_VARARGS, ""}, + {NULL, NULL, 0, NULL} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef speedupsmodule = { + PyModuleDef_HEAD_INIT, + "speedups", + NULL, + -1, + methods +}; + +PyMODINIT_FUNC +PyInit_speedups() { + return PyModule_Create(&speedupsmodule); +} +#else // Python 2.x +PyMODINIT_FUNC +initspeedups() { + Py_InitModule("tornado.speedups", methods); +} +#endif diff --git a/libs/tornado/tcpserver.py b/libs/tornado/tcpserver.py index 8473a21a..c0773732 100755 --- a/libs/tornado/tcpserver.py +++ b/libs/tornado/tcpserver.py @@ -180,7 +180,8 @@ class TCPServer(object): between any server code. Note that multiple processes are not compatible with the autoreload - module (or the ``debug=True`` option to `tornado.web.Application`). + module (or the ``autoreload=True`` option to `tornado.web.Application` + which defaults to True when ``debug=True``). When using multiple processes, no IOLoops can be created or referenced until after the call to ``TCPServer.start(n)``. """ diff --git a/libs/tornado/web.py b/libs/tornado/web.py index 5f8d6091..b6d7e97e 100755 --- a/libs/tornado/web.py +++ b/libs/tornado/web.py @@ -250,7 +250,7 @@ class RequestHandler(object): not self.request.connection.no_keep_alive): conn_header = self.request.headers.get("Connection") if conn_header and (conn_header.lower() == "keep-alive"): - self.set_header("Connection", "Keep-Alive") + self._headers["Connection"] = "Keep-Alive" self._write_buffer = [] self._status_code = 200 self._reason = httputil.responses[200] @@ -348,12 +348,7 @@ class RequestHandler(object): The returned value is always unicode. """ - args = self.get_arguments(name, strip=strip) - if not args: - if default is self._ARG_DEFAULT: - raise MissingArgumentError(name) - return default - return args[-1] + return self._get_argument(name, default, self.request.arguments, strip) def get_arguments(self, name, strip=True): """Returns a list of the arguments with the given name. @@ -362,9 +357,73 @@ class RequestHandler(object): The returned values are always unicode. """ + return self._get_arguments(name, self.request.arguments, strip) + def get_body_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name + from the request body. + + If default is not provided, the argument is considered to be + required, and we raise a `MissingArgumentError` if it is missing. + + If the argument appears in the url more than once, we return the + last value. + + The returned value is always unicode. + + .. versionadded:: 3.2 + """ + return self._get_argument(name, default, self.request.body_arguments, strip) + + def get_body_arguments(self, name, strip=True): + """Returns a list of the body arguments with the given name. + + If the argument is not present, returns an empty list. + + The returned values are always unicode. + + .. versionadded:: 3.2 + """ + return self._get_arguments(name, self.request.body_arguments, strip) + + def get_query_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name + from the request query string. + + If default is not provided, the argument is considered to be + required, and we raise a `MissingArgumentError` if it is missing. + + If the argument appears in the url more than once, we return the + last value. + + The returned value is always unicode. + + .. versionadded:: 3.2 + """ + return self._get_argument(name, default, self.request.query_arguments, strip) + + def get_query_arguments(self, name, strip=True): + """Returns a list of the query arguments with the given name. + + If the argument is not present, returns an empty list. + + The returned values are always unicode. + + .. versionadded:: 3.2 + """ + return self._get_arguments(name, self.request.query_arguments, strip) + + def _get_argument(self, name, default, source, strip=True): + args = self._get_arguments(name, source, strip=strip) + if not args: + if default is self._ARG_DEFAULT: + raise MissingArgumentError(name) + return default + return args[-1] + + def _get_arguments(self, name, source, strip=True): values = [] - for v in self.request.arguments.get(name, []): + for v in source.get(name, []): v = self.decode_argument(v, name=name) if isinstance(v, unicode_type): # Get rid of any weird control chars (unless decoding gave @@ -838,7 +897,7 @@ class RequestHandler(object): else: self.finish(self.get_error_html(status_code, **kwargs)) return - if self.settings.get("debug") and "exc_info" in kwargs: + if self.settings.get("serve_traceback") and "exc_info" in kwargs: # in debug mode, try to send a traceback self.set_header('Content-Type', 'text/plain') for line in traceback.format_exception(*kwargs["exc_info"]): @@ -1318,6 +1377,12 @@ def asynchronous(method): if not self._finished: self.finish() IOLoop.current().add_future(result, future_complete) + # Once we have done this, hide the Future from our + # caller (i.e. RequestHandler._when_complete), which + # would otherwise set up its own callback and + # exception handler (resulting in exceptions being + # logged twice). + return None return result return wrapper @@ -1383,10 +1448,16 @@ class Application(object): or (regexp, request_class) tuples. When we receive requests, we iterate over the list in order and instantiate an instance of the first request class whose regexp matches the request path. + The request class can be specified as either a class object or a + (fully-qualified) name. - Each tuple can contain an optional third element, which should be - a dictionary if it is present. That dictionary is passed as - keyword arguments to the contructor of the handler. This pattern + Each tuple can contain additional elements, which correspond to the + arguments to the `URLSpec` constructor. (Prior to Tornado 3.2, this + only tuples of two or three elements were allowed). + + A dictionary may be passed as the third element of the tuple, + which will be used as keyword arguments to the handler's + constructor and `~RequestHandler.initialize` method. This pattern is used for the `StaticFileHandler` in this example (note that a `StaticFileHandler` can be installed automatically with the static_path setting described below):: @@ -1409,6 +1480,7 @@ class Application(object): and ``/robots.txt`` from the same directory. A custom subclass of `StaticFileHandler` can be specified with the ``static_handler_class`` setting. + """ def __init__(self, handlers=None, default_host="", transforms=None, wsgi=False, **settings): @@ -1447,8 +1519,14 @@ class Application(object): if handlers: self.add_handlers(".*$", handlers) + if self.settings.get('debug'): + self.settings.setdefault('autoreload', True) + self.settings.setdefault('compiled_template_cache', False) + self.settings.setdefault('static_hash_cache', False) + self.settings.setdefault('serve_traceback', True) + # Automatically reload modified modules - if self.settings.get("debug") and not wsgi: + if self.settings.get('autoreload') and not wsgi: from tornado import autoreload autoreload.start() @@ -1493,20 +1571,8 @@ class Application(object): for spec in host_handlers: if isinstance(spec, (tuple, list)): - assert len(spec) in (2, 3) - pattern = spec[0] - handler = spec[1] - - if isinstance(handler, str): - # import the Module and instantiate the class - # Must be a fully qualified name (module.ClassName) - handler = import_object(handler) - - if len(spec) == 3: - kwargs = spec[2] - else: - kwargs = {} - spec = URLSpec(pattern, handler, kwargs) + assert len(spec) in (2, 3, 4) + spec = URLSpec(*spec) handlers.append(spec) if spec.name: if spec.name in self.named_handlers: @@ -1597,14 +1663,23 @@ class Application(object): args = [unquote(s) for s in match.groups()] break if not handler: - handler = ErrorHandler(self, request, status_code=404) + if self.settings.get('default_handler_class'): + handler_class = self.settings['default_handler_class'] + handler_args = self.settings.get( + 'default_handler_args', {}) + else: + handler_class = ErrorHandler + handler_args = dict(status_code=404) + handler = handler_class(self, request, **handler_args) - # In debug mode, re-compile templates and reload static files on every + # If template cache is disabled (usually in the debug mode), + # re-compile templates and reload static files on every # request so you don't need to restart to see changes - if self.settings.get("debug"): + if not self.settings.get("compiled_template_cache", True): with RequestHandler._template_loader_lock: for loader in RequestHandler._template_loaders.values(): loader.reset() + if not self.settings.get('static_hash_cache', True): StaticFileHandler.reset() handler._execute(transforms, *args, **kwargs) @@ -2454,7 +2529,7 @@ class _UIModuleNamespace(object): class URLSpec(object): """Specifies mappings between URLs and handlers.""" - def __init__(self, pattern, handler_class, kwargs=None, name=None): + def __init__(self, pattern, handler, kwargs=None, name=None): """Parameters: * ``pattern``: Regular expression to be matched. Any groups @@ -2475,7 +2550,13 @@ class URLSpec(object): assert len(self.regex.groupindex) in (0, self.regex.groups), \ ("groups in url regexes must either be all named or all " "positional: %r" % self.regex.pattern) - self.handler_class = handler_class + + if isinstance(handler, str): + # import the Module and instantiate the class + # Must be a fully qualified name (module.ClassName) + handler = import_object(handler) + + self.handler_class = handler self.kwargs = kwargs or {} self.name = name self._path, self._group_count = self._find_groups() diff --git a/libs/tornado/websocket.py b/libs/tornado/websocket.py index 676d21bf..8c2f5a64 100755 --- a/libs/tornado/websocket.py +++ b/libs/tornado/websocket.py @@ -33,7 +33,7 @@ import tornado.web from tornado.concurrent import TracebackFuture from tornado.escape import utf8, native_str -from tornado import httpclient +from tornado import httpclient, httputil from tornado.ioloop import IOLoop from tornado.iostream import StreamClosedError from tornado.log import gen_log, app_log @@ -52,6 +52,10 @@ class WebSocketError(Exception): class WebSocketClosedError(WebSocketError): + """Raised by operations on a closed connection. + + .. versionadded:: 3.2 + """ pass @@ -163,6 +167,12 @@ class WebSocketHandler(tornado.web.RequestHandler): encoded as json). If the ``binary`` argument is false, the message will be sent as utf8; in binary mode any byte string is allowed. + + If the connection is already closed, raises `WebSocketClosedError`. + + .. versionchanged:: 3.2 + `WebSocketClosedError` was added (previously a closed connection + would raise an `AttributeError`) """ if self.ws_connection is None: raise WebSocketClosedError() @@ -586,7 +596,7 @@ class WebSocketProtocol13(WebSocketProtocol): frame += struct.pack("!BQ", 127 | mask_bit, l) if self.mask_outgoing: mask = os.urandom(4) - data = mask + self._apply_mask(mask, data) + data = mask + _websocket_mask(mask, data) frame += data self.stream.write(frame) @@ -671,21 +681,8 @@ class WebSocketProtocol13(WebSocketProtocol): except StreamClosedError: self._abort() - def _apply_mask(self, mask, data): - mask = array.array("B", mask) - unmasked = array.array("B", data) - for i in xrange(len(data)): - unmasked[i] = unmasked[i] ^ mask[i % 4] - if hasattr(unmasked, 'tobytes'): - # tostring was deprecated in py32. It hasn't been removed, - # but since we turn on deprecation warnings in our tests - # we need to use the right one. - return unmasked.tobytes() - else: - return unmasked.tostring() - def _on_masked_frame_data(self, data): - self._on_frame_data(self._apply_mask(self._frame_mask, data)) + self._on_frame_data(_websocket_mask(self._frame_mask, data)) def _on_frame_data(self, data): if self._frame_opcode_is_control: @@ -771,7 +768,11 @@ class WebSocketProtocol13(WebSocketProtocol): class WebSocketClientConnection(simple_httpclient._HTTPConnection): - """WebSocket client connection.""" + """WebSocket client connection. + + This class should not be instantiated directly; use the + `websocket_connect` function instead. + """ def __init__(self, io_loop, request): self.connect_future = TracebackFuture() self.read_future = None @@ -793,9 +794,19 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.resolver) + def close(self): + """Closes the websocket connection. + + .. versionadded:: 3.2 + """ + if self.protocol is not None: + self.protocol.close() + self.protocol = None + def _on_close(self): self.on_message(None) self.resolver.close() + super(WebSocketClientConnection, self)._on_close() def _on_http_response(self, response): if not self.connect_future.done(): @@ -859,13 +870,54 @@ def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None): Takes a url and returns a Future whose result is a `WebSocketClientConnection`. + + .. versionchanged:: 3.2 + Also accepts ``HTTPRequest`` objects in place of urls. """ if io_loop is None: io_loop = IOLoop.current() - request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) + if isinstance(url, httpclient.HTTPRequest): + assert connect_timeout is None + request = url + # Copy and convert the headers dict/object (see comments in + # AsyncHTTPClient.fetch) + request.headers = httputil.HTTPHeaders(request.headers) + else: + request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) request = httpclient._RequestProxy( request, httpclient.HTTPRequest._DEFAULTS) conn = WebSocketClientConnection(io_loop, request) if callback is not None: io_loop.add_future(conn.connect_future, callback) return conn.connect_future + +def _websocket_mask_python(mask, data): + """Websocket masking function. + + `mask` is a `bytes` object of length 4; `data` is a `bytes` object of any length. + Returns a `bytes` object of the same length as `data` with the mask applied + as specified in section 5.3 of RFC 6455. + + This pure-python implementation may be replaced by an optimized version when available. + """ + mask = array.array("B", mask) + unmasked = array.array("B", data) + for i in xrange(len(data)): + unmasked[i] = unmasked[i] ^ mask[i % 4] + if hasattr(unmasked, 'tobytes'): + # tostring was deprecated in py32. It hasn't been removed, + # but since we turn on deprecation warnings in our tests + # we need to use the right one. + return unmasked.tobytes() + else: + return unmasked.tostring() + +if os.environ.get('TORNADO_NO_EXTENSION'): + # This environment variable exists to make it easier to do performance comparisons; + # it's not guaranteed to remain supported in the future. + _websocket_mask = _websocket_mask_python +else: + try: + from tornado.speedups import websocket_mask as _websocket_mask + except ImportError: + _websocket_mask = _websocket_mask_python diff --git a/libs/tornado/wsgi.py b/libs/tornado/wsgi.py index 5e25a564..8e5ddedb 100755 --- a/libs/tornado/wsgi.py +++ b/libs/tornado/wsgi.py @@ -33,6 +33,7 @@ from __future__ import absolute_import, division, print_function, with_statement import sys import time +import copy import tornado from tornado import escape @@ -142,11 +143,14 @@ class HTTPRequest(object): self.path += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", ""))) self.uri = self.path self.arguments = {} + self.query_arguments = {} + self.body_arguments = {} self.query = environ.get("QUERY_STRING", "") if self.query: self.uri += "?" + self.query self.arguments = parse_qs_bytes(native_str(self.query), keep_blank_values=True) + self.query_arguments = copy.deepcopy(self.arguments) self.version = "HTTP/1.1" self.headers = httputil.HTTPHeaders() if environ.get("CONTENT_TYPE"): @@ -171,7 +175,10 @@ class HTTPRequest(object): # Parse request body self.files = {} httputil.parse_body_arguments(self.headers.get("Content-Type", ""), - self.body, self.arguments, self.files) + self.body, self.body_arguments, self.files) + + for k, v in self.body_arguments.items(): + self.arguments.setdefault(k, []).extend(v) self._start_time = time.time() self._finish_time = None