":
+ if element.text:
+ rv.append(element.text)
+ if element.tail:
+ finalText = element.tail
+
+ for child in element:
+ serializeElement(child)
+
+ elif type(element.tag) == type(ElementTree.Comment):
+ rv.append(""%(element.text,))
+ else:
+ #This is assumed to be an ordinary element
+ if not element.attrib:
+ rv.append("<%s>"%(filter.fromXmlName(element.tag),))
+ else:
+ attr = " ".join(["%s=\"%s\""%(
+ filter.fromXmlName(name), value)
+ for name, value in element.attrib.iteritems()])
+ rv.append("<%s %s>"%(element.tag, attr))
+ if element.text:
+ rv.append(element.text)
+
+ for child in element:
+ serializeElement(child)
+
+ rv.append(""%(element.tag,))
+
+ if element.tail:
+ rv.append(element.tail)
+
+ serializeElement(element)
+
+ if finalText is not None:
+ rv.append("%s\""%(' '*2, finalText))
+
+ return "".join(rv)
+
+ class TreeBuilder(_base.TreeBuilder):
+ documentClass = Document
+ doctypeClass = DocumentType
+ elementClass = Element
+ commentClass = Comment
+ fragmentClass = DocumentFragment
+
+ def testSerializer(self, element):
+ return testSerializer(element)
+
+ def getDocument(self):
+ if fullTree:
+ return self.document._element
+ else:
+ if self.defaultNamespace is not None:
+ return self.document._element.find(
+ "{%s}html"%self.defaultNamespace)
+ else:
+ return self.document._element.find("html")
+
+ def getFragment(self):
+ return _base.TreeBuilder.getFragment(self)._element
+
+ return locals()
diff --git a/libs/html5lib/treebuilders/etree_lxml.py b/libs/html5lib/treebuilders/etree_lxml.py
new file mode 100644
index 00000000..eee1e3b2
--- /dev/null
+++ b/libs/html5lib/treebuilders/etree_lxml.py
@@ -0,0 +1,336 @@
+import warnings
+import re
+
+import _base
+from html5lib.constants import DataLossWarning
+import html5lib.constants as constants
+import etree as etree_builders
+from html5lib import ihatexml
+
+try:
+ import lxml.etree as etree
+except ImportError:
+ pass
+
+fullTree = True
+tag_regexp = re.compile("{([^}]*)}(.*)")
+
+"""Module for supporting the lxml.etree library. The idea here is to use as much
+of the native library as possible, without using fragile hacks like custom element
+names that break between releases. The downside of this is that we cannot represent
+all possible trees; specifically the following are known to cause problems:
+
+Text or comments as siblings of the root element
+Docypes with no name
+
+When any of these things occur, we emit a DataLossWarning
+"""
+
+class DocumentType(object):
+ def __init__(self, name, publicId, systemId):
+ self.name = name
+ self.publicId = publicId
+ self.systemId = systemId
+
+class Document(object):
+ def __init__(self):
+ self._elementTree = None
+ self._childNodes = []
+
+ def appendChild(self, element):
+ self._elementTree.getroot().addnext(element._element)
+
+ def _getChildNodes(self):
+ return self._childNodes
+
+ childNodes = property(_getChildNodes)
+
+def testSerializer(element):
+ rv = []
+ finalText = None
+ filter = ihatexml.InfosetFilter()
+ def serializeElement(element, indent=0):
+ if not hasattr(element, "tag"):
+ if hasattr(element, "getroot"):
+ #Full tree case
+ rv.append("#document")
+ if element.docinfo.internalDTD:
+ if not (element.docinfo.public_id or
+ element.docinfo.system_url):
+ dtd_str = ""%element.docinfo.root_name
+ else:
+ dtd_str = """"""%(
+ element.docinfo.root_name,
+ element.docinfo.public_id,
+ element.docinfo.system_url)
+ rv.append("|%s%s"%(' '*(indent+2), dtd_str))
+ next_element = element.getroot()
+ while next_element.getprevious() is not None:
+ next_element = next_element.getprevious()
+ while next_element is not None:
+ serializeElement(next_element, indent+2)
+ next_element = next_element.getnext()
+ elif isinstance(element, basestring):
+ #Text in a fragment
+ rv.append("|%s\"%s\""%(' '*indent, element))
+ else:
+ #Fragment case
+ rv.append("#document-fragment")
+ for next_element in element:
+ serializeElement(next_element, indent+2)
+ elif type(element.tag) == type(etree.Comment):
+ rv.append("|%s"%(' '*indent, element.text))
+ else:
+ nsmatch = etree_builders.tag_regexp.match(element.tag)
+ if nsmatch is not None:
+ ns = nsmatch.group(1)
+ tag = nsmatch.group(2)
+ prefix = constants.prefixes[ns]
+ rv.append("|%s<%s %s>"%(' '*indent, prefix,
+ filter.fromXmlName(tag)))
+ else:
+ rv.append("|%s<%s>"%(' '*indent,
+ filter.fromXmlName(element.tag)))
+
+ if hasattr(element, "attrib"):
+ attributes = []
+ for name, value in element.attrib.iteritems():
+ nsmatch = tag_regexp.match(name)
+ if nsmatch is not None:
+ ns, name = nsmatch.groups()
+ name = filter.fromXmlName(name)
+ prefix = constants.prefixes[ns]
+ attr_string = "%s %s"%(prefix, name)
+ else:
+ attr_string = filter.fromXmlName(name)
+ attributes.append((attr_string, value))
+
+ for name, value in sorted(attributes):
+ rv.append('|%s%s="%s"' % (' '*(indent+2), name, value))
+
+ if element.text:
+ rv.append("|%s\"%s\"" %(' '*(indent+2), element.text))
+ indent += 2
+ for child in element.getchildren():
+ serializeElement(child, indent)
+ if hasattr(element, "tail") and element.tail:
+ rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail))
+ serializeElement(element, 0)
+
+ if finalText is not None:
+ rv.append("|%s\"%s\""%(' '*2, finalText))
+
+ return "\n".join(rv)
+
+def tostring(element):
+ """Serialize an element and its child nodes to a string"""
+ rv = []
+ finalText = None
+ def serializeElement(element):
+ if not hasattr(element, "tag"):
+ if element.docinfo.internalDTD:
+ if element.docinfo.doctype:
+ dtd_str = element.docinfo.doctype
+ else:
+ dtd_str = ""%element.docinfo.root_name
+ rv.append(dtd_str)
+ serializeElement(element.getroot())
+
+ elif type(element.tag) == type(etree.Comment):
+ rv.append(""%(element.text,))
+
+ else:
+ #This is assumed to be an ordinary element
+ if not element.attrib:
+ rv.append("<%s>"%(element.tag,))
+ else:
+ attr = " ".join(["%s=\"%s\""%(name, value)
+ for name, value in element.attrib.iteritems()])
+ rv.append("<%s %s>"%(element.tag, attr))
+ if element.text:
+ rv.append(element.text)
+
+ for child in element.getchildren():
+ serializeElement(child)
+
+ rv.append(""%(element.tag,))
+
+ if hasattr(element, "tail") and element.tail:
+ rv.append(element.tail)
+
+ serializeElement(element)
+
+ if finalText is not None:
+ rv.append("%s\""%(' '*2, finalText))
+
+ return "".join(rv)
+
+
+class TreeBuilder(_base.TreeBuilder):
+ documentClass = Document
+ doctypeClass = DocumentType
+ elementClass = None
+ commentClass = None
+ fragmentClass = Document
+
+ def __init__(self, namespaceHTMLElements, fullTree = False):
+ builder = etree_builders.getETreeModule(etree, fullTree=fullTree)
+ filter = self.filter = ihatexml.InfosetFilter()
+ self.namespaceHTMLElements = namespaceHTMLElements
+
+ class Attributes(dict):
+ def __init__(self, element, value={}):
+ self._element = element
+ dict.__init__(self, value)
+ for key, value in self.iteritems():
+ if isinstance(key, tuple):
+ name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1]))
+ else:
+ name = filter.coerceAttribute(key)
+ self._element._element.attrib[name] = value
+
+ def __setitem__(self, key, value):
+ dict.__setitem__(self, key, value)
+ if isinstance(key, tuple):
+ name = "{%s}%s"%(key[2], filter.coerceAttribute(key[1]))
+ else:
+ name = filter.coerceAttribute(key)
+ self._element._element.attrib[name] = value
+
+ class Element(builder.Element):
+ def __init__(self, name, namespace):
+ name = filter.coerceElement(name)
+ builder.Element.__init__(self, name, namespace=namespace)
+ self._attributes = Attributes(self)
+
+ def _setName(self, name):
+ self._name = filter.coerceElement(name)
+ self._element.tag = self._getETreeTag(
+ self._name, self._namespace)
+
+ def _getName(self):
+ return filter.fromXmlName(self._name)
+
+ name = property(_getName, _setName)
+
+ def _getAttributes(self):
+ return self._attributes
+
+ def _setAttributes(self, attributes):
+ self._attributes = Attributes(self, attributes)
+
+ attributes = property(_getAttributes, _setAttributes)
+
+ def insertText(self, data, insertBefore=None):
+ data = filter.coerceCharacters(data)
+ builder.Element.insertText(self, data, insertBefore)
+
+ def appendChild(self, child):
+ builder.Element.appendChild(self, child)
+
+
+ class Comment(builder.Comment):
+ def __init__(self, data):
+ data = filter.coerceComment(data)
+ builder.Comment.__init__(self, data)
+
+ def _setData(self, data):
+ data = filter.coerceComment(data)
+ self._element.text = data
+
+ def _getData(self):
+ return self._element.text
+
+ data = property(_getData, _setData)
+
+ self.elementClass = Element
+ self.commentClass = builder.Comment
+ #self.fragmentClass = builder.DocumentFragment
+ _base.TreeBuilder.__init__(self, namespaceHTMLElements)
+
+ def reset(self):
+ _base.TreeBuilder.reset(self)
+ self.insertComment = self.insertCommentInitial
+ self.initial_comments = []
+ self.doctype = None
+
+ def testSerializer(self, element):
+ return testSerializer(element)
+
+ def getDocument(self):
+ if fullTree:
+ return self.document._elementTree
+ else:
+ return self.document._elementTree.getroot()
+
+ def getFragment(self):
+ fragment = []
+ element = self.openElements[0]._element
+ if element.text:
+ fragment.append(element.text)
+ fragment.extend(element.getchildren())
+ if element.tail:
+ fragment.append(element.tail)
+ return fragment
+
+ def insertDoctype(self, token):
+ name = token["name"]
+ publicId = token["publicId"]
+ systemId = token["systemId"]
+
+ if not name or ihatexml.nonXmlNameBMPRegexp.search(name) or name[0] == '"':
+ warnings.warn("lxml cannot represent null or non-xml doctype", DataLossWarning)
+
+ doctype = self.doctypeClass(name, publicId, systemId)
+ self.doctype = doctype
+
+ def insertCommentInitial(self, data, parent=None):
+ self.initial_comments.append(data)
+
+ def insertRoot(self, token):
+ """Create the document root"""
+ #Because of the way libxml2 works, it doesn't seem to be possible to
+ #alter information like the doctype after the tree has been parsed.
+ #Therefore we need to use the built-in parser to create our iniial
+ #tree, after which we can add elements like normal
+ docStr = ""
+ if self.doctype and self.doctype.name and not self.doctype.name.startswith('"'):
+ docStr += ""
+ docStr += ""
+
+ try:
+ root = etree.fromstring(docStr)
+ except etree.XMLSyntaxError:
+ print docStr
+ raise
+
+ #Append the initial comments:
+ for comment_token in self.initial_comments:
+ root.addprevious(etree.Comment(comment_token["data"]))
+
+ #Create the root document and add the ElementTree to it
+ self.document = self.documentClass()
+ self.document._elementTree = root.getroottree()
+
+ # Give the root element the right name
+ name = token["name"]
+ namespace = token.get("namespace", self.defaultNamespace)
+ if namespace is None:
+ etree_tag = name
+ else:
+ etree_tag = "{%s}%s"%(namespace, name)
+ root.tag = etree_tag
+
+ #Add the root element to the internal child/open data structures
+ root_element = self.elementClass(name, namespace)
+ root_element._element = root
+ self.document._childNodes.append(root_element)
+ self.openElements.append(root_element)
+
+ #Reset to the default insert comment function
+ self.insertComment = super(TreeBuilder, self).insertComment
diff --git a/libs/html5lib/treebuilders/simpletree.py b/libs/html5lib/treebuilders/simpletree.py
new file mode 100755
index 00000000..67fe7583
--- /dev/null
+++ b/libs/html5lib/treebuilders/simpletree.py
@@ -0,0 +1,256 @@
+import _base
+from html5lib.constants import voidElements, namespaces, prefixes
+from xml.sax.saxutils import escape
+
+# Really crappy basic implementation of a DOM-core like thing
+class Node(_base.Node):
+ type = -1
+ def __init__(self, name):
+ self.name = name
+ self.parent = None
+ self.value = None
+ self.childNodes = []
+ self._flags = []
+
+ def __iter__(self):
+ for node in self.childNodes:
+ yield node
+ for item in node:
+ yield item
+
+ def __unicode__(self):
+ return self.name
+
+ def toxml(self):
+ raise NotImplementedError
+
+ def printTree(self, indent=0):
+ tree = '\n|%s%s' % (' '* indent, unicode(self))
+ for child in self.childNodes:
+ tree += child.printTree(indent + 2)
+ return tree
+
+ def appendChild(self, node):
+ assert isinstance(node, Node)
+ if (isinstance(node, TextNode) and self.childNodes and
+ isinstance(self.childNodes[-1], TextNode)):
+ self.childNodes[-1].value += node.value
+ else:
+ self.childNodes.append(node)
+ node.parent = self
+
+ def insertText(self, data, insertBefore=None):
+ assert isinstance(data, unicode), "data %s is of type %s expected unicode"%(repr(data), type(data))
+ if insertBefore is None:
+ self.appendChild(TextNode(data))
+ else:
+ self.insertBefore(TextNode(data), insertBefore)
+
+ def insertBefore(self, node, refNode):
+ index = self.childNodes.index(refNode)
+ if (isinstance(node, TextNode) and index > 0 and
+ isinstance(self.childNodes[index - 1], TextNode)):
+ self.childNodes[index - 1].value += node.value
+ else:
+ self.childNodes.insert(index, node)
+ node.parent = self
+
+ def removeChild(self, node):
+ try:
+ self.childNodes.remove(node)
+ except:
+ # XXX
+ raise
+ node.parent = None
+
+ def cloneNode(self):
+ raise NotImplementedError
+
+ def hasContent(self):
+ """Return true if the node has children or text"""
+ return bool(self.childNodes)
+
+ def getNameTuple(self):
+ if self.namespace == None:
+ return namespaces["html"], self.name
+ else:
+ return self.namespace, self.name
+
+ nameTuple = property(getNameTuple)
+
+class Document(Node):
+ type = 1
+ def __init__(self):
+ Node.__init__(self, None)
+
+ def __str__(self):
+ return "#document"
+
+ def __unicode__(self):
+ return str(self)
+
+ def appendChild(self, child):
+ Node.appendChild(self, child)
+
+ def toxml(self, encoding="utf=8"):
+ result = ""
+ for child in self.childNodes:
+ result += child.toxml()
+ return result.encode(encoding)
+
+ def hilite(self, encoding="utf-8"):
+ result = ""
+ for child in self.childNodes:
+ result += child.hilite()
+ return result.encode(encoding) + "
"
+
+ def printTree(self):
+ tree = unicode(self)
+ for child in self.childNodes:
+ tree += child.printTree(2)
+ return tree
+
+ def cloneNode(self):
+ return Document()
+
+class DocumentFragment(Document):
+ type = 2
+ def __str__(self):
+ return "#document-fragment"
+
+ def __unicode__(self):
+ return str(self)
+
+ def cloneNode(self):
+ return DocumentFragment()
+
+class DocumentType(Node):
+ type = 3
+ def __init__(self, name, publicId, systemId):
+ Node.__init__(self, name)
+ self.publicId = publicId
+ self.systemId = systemId
+
+ def __unicode__(self):
+ if self.publicId or self.systemId:
+ publicId = self.publicId or ""
+ systemId = self.systemId or ""
+ return """"""%(
+ self.name, publicId, systemId)
+
+ else:
+ return u"" % self.name
+
+
+ toxml = __unicode__
+
+ def hilite(self):
+ return '<!DOCTYPE %s>' % self.name
+
+ def cloneNode(self):
+ return DocumentType(self.name, self.publicId, self.systemId)
+
+class TextNode(Node):
+ type = 4
+ def __init__(self, value):
+ Node.__init__(self, None)
+ self.value = value
+
+ def __unicode__(self):
+ return u"\"%s\"" % self.value
+
+ def toxml(self):
+ return escape(self.value)
+
+ hilite = toxml
+
+ def cloneNode(self):
+ return TextNode(self.value)
+
+class Element(Node):
+ type = 5
+ def __init__(self, name, namespace=None):
+ Node.__init__(self, name)
+ self.namespace = namespace
+ self.attributes = {}
+
+ def __unicode__(self):
+ if self.namespace == None:
+ return u"<%s>" % self.name
+ else:
+ return u"<%s %s>"%(prefixes[self.namespace], self.name)
+
+ def toxml(self):
+ result = '<' + self.name
+ if self.attributes:
+ for name,value in self.attributes.iteritems():
+ result += u' %s="%s"' % (name, escape(value,{'"':'"'}))
+ if self.childNodes:
+ result += '>'
+ for child in self.childNodes:
+ result += child.toxml()
+ result += u'' % self.name
+ else:
+ result += u'/>'
+ return result
+
+ def hilite(self):
+ result = '<%s' % self.name
+ if self.attributes:
+ for name, value in self.attributes.iteritems():
+ result += ' %s="%s"' % (name, escape(value, {'"':'"'}))
+ if self.childNodes:
+ result += ">"
+ for child in self.childNodes:
+ result += child.hilite()
+ elif self.name in voidElements:
+ return result + ">"
+ return result + '</%s>' % self.name
+
+ def printTree(self, indent):
+ tree = '\n|%s%s' % (' '*indent, unicode(self))
+ indent += 2
+ if self.attributes:
+ for name, value in sorted(self.attributes.iteritems()):
+ if isinstance(name, tuple):
+ name = "%s %s"%(name[0], name[1])
+ tree += '\n|%s%s="%s"' % (' ' * indent, name, value)
+ for child in self.childNodes:
+ tree += child.printTree(indent)
+ return tree
+
+ def cloneNode(self):
+ newNode = Element(self.name)
+ if hasattr(self, 'namespace'):
+ newNode.namespace = self.namespace
+ for attr, value in self.attributes.iteritems():
+ newNode.attributes[attr] = value
+ return newNode
+
+class CommentNode(Node):
+ type = 6
+ def __init__(self, data):
+ Node.__init__(self, None)
+ self.data = data
+
+ def __unicode__(self):
+ return "" % self.data
+
+ def toxml(self):
+ return "" % self.data
+
+ def hilite(self):
+ return '' % escape(self.data)
+
+ def cloneNode(self):
+ return CommentNode(self.data)
+
+class TreeBuilder(_base.TreeBuilder):
+ documentClass = Document
+ doctypeClass = DocumentType
+ elementClass = Element
+ commentClass = CommentNode
+ fragmentClass = DocumentFragment
+
+ def testSerializer(self, node):
+ return node.printTree()
diff --git a/libs/html5lib/treebuilders/soup.py b/libs/html5lib/treebuilders/soup.py
new file mode 100644
index 00000000..9bc5ff0e
--- /dev/null
+++ b/libs/html5lib/treebuilders/soup.py
@@ -0,0 +1,236 @@
+import warnings
+
+warnings.warn("BeautifulSoup 3.x (as of 3.1) is not fully compatible with html5lib and support will be removed in the future", DeprecationWarning)
+
+from BeautifulSoup import BeautifulSoup, Tag, NavigableString, Comment, Declaration
+
+import _base
+from html5lib.constants import namespaces, DataLossWarning
+
+class AttrList(object):
+ def __init__(self, element):
+ self.element = element
+ self.attrs = dict(self.element.attrs)
+ def __iter__(self):
+ return self.attrs.items().__iter__()
+ def __setitem__(self, name, value):
+ "set attr", name, value
+ self.element[name] = value
+ def items(self):
+ return self.attrs.items()
+ def keys(self):
+ return self.attrs.keys()
+ def __getitem__(self, name):
+ return self.attrs[name]
+ def __contains__(self, name):
+ return name in self.attrs.keys()
+ def __eq__(self, other):
+ if len(self.keys()) != len(other.keys()):
+ return False
+ for item in self.keys():
+ if item not in other:
+ return False
+ if self[item] != other[item]:
+ return False
+ return True
+
+class Element(_base.Node):
+ def __init__(self, element, soup, namespace):
+ _base.Node.__init__(self, element.name)
+ self.element = element
+ self.soup = soup
+ self.namespace = namespace
+
+ def _nodeIndex(self, node, refNode):
+ # Finds a node by identity rather than equality
+ for index in range(len(self.element.contents)):
+ if id(self.element.contents[index]) == id(refNode.element):
+ return index
+ return None
+
+ def appendChild(self, node):
+ if (node.element.__class__ == NavigableString and self.element.contents
+ and self.element.contents[-1].__class__ == NavigableString):
+ # Concatenate new text onto old text node
+ # (TODO: This has O(n^2) performance, for input like "aaa...")
+ newStr = NavigableString(self.element.contents[-1]+node.element)
+
+ # Remove the old text node
+ # (Can't simply use .extract() by itself, because it fails if
+ # an equal text node exists within the parent node)
+ oldElement = self.element.contents[-1]
+ del self.element.contents[-1]
+ oldElement.parent = None
+ oldElement.extract()
+
+ self.element.insert(len(self.element.contents), newStr)
+ else:
+ self.element.insert(len(self.element.contents), node.element)
+ node.parent = self
+
+ def getAttributes(self):
+ return AttrList(self.element)
+
+ def setAttributes(self, attributes):
+ if attributes:
+ for name, value in attributes.items():
+ self.element[name] = value
+
+ attributes = property(getAttributes, setAttributes)
+
+ def insertText(self, data, insertBefore=None):
+ text = TextNode(NavigableString(data), self.soup)
+ if insertBefore:
+ self.insertBefore(text, insertBefore)
+ else:
+ self.appendChild(text)
+
+ def insertBefore(self, node, refNode):
+ index = self._nodeIndex(node, refNode)
+ if (node.element.__class__ == NavigableString and self.element.contents
+ and self.element.contents[index-1].__class__ == NavigableString):
+ # (See comments in appendChild)
+ newStr = NavigableString(self.element.contents[index-1]+node.element)
+ oldNode = self.element.contents[index-1]
+ del self.element.contents[index-1]
+ oldNode.parent = None
+ oldNode.extract()
+
+ self.element.insert(index-1, newStr)
+ else:
+ self.element.insert(index, node.element)
+ node.parent = self
+
+ def removeChild(self, node):
+ index = self._nodeIndex(node.parent, node)
+ del node.parent.element.contents[index]
+ node.element.parent = None
+ node.element.extract()
+ node.parent = None
+
+ def reparentChildren(self, newParent):
+ while self.element.contents:
+ child = self.element.contents[0]
+ child.extract()
+ if isinstance(child, Tag):
+ newParent.appendChild(Element(child, self.soup, namespaces["html"]))
+ else:
+ newParent.appendChild(TextNode(child, self.soup))
+
+ def cloneNode(self):
+ node = Element(Tag(self.soup, self.element.name), self.soup, self.namespace)
+ for key,value in self.attributes:
+ node.attributes[key] = value
+ return node
+
+ def hasContent(self):
+ return self.element.contents
+
+ def getNameTuple(self):
+ if self.namespace == None:
+ return namespaces["html"], self.name
+ else:
+ return self.namespace, self.name
+
+ nameTuple = property(getNameTuple)
+
+class TextNode(Element):
+ def __init__(self, element, soup):
+ _base.Node.__init__(self, None)
+ self.element = element
+ self.soup = soup
+
+ def cloneNode(self):
+ raise NotImplementedError
+
+class TreeBuilder(_base.TreeBuilder):
+ def __init__(self, namespaceHTMLElements):
+ if namespaceHTMLElements:
+ warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning)
+ _base.TreeBuilder.__init__(self, namespaceHTMLElements)
+
+ def documentClass(self):
+ self.soup = BeautifulSoup("")
+ return Element(self.soup, self.soup, None)
+
+ def insertDoctype(self, token):
+ name = token["name"]
+ publicId = token["publicId"]
+ systemId = token["systemId"]
+
+ if publicId:
+ self.soup.insert(0, Declaration("DOCTYPE %s PUBLIC \"%s\" \"%s\""%(name, publicId, systemId or "")))
+ elif systemId:
+ self.soup.insert(0, Declaration("DOCTYPE %s SYSTEM \"%s\""%
+ (name, systemId)))
+ else:
+ self.soup.insert(0, Declaration("DOCTYPE %s"%name))
+
+ def elementClass(self, name, namespace):
+ if namespace is not None:
+ warnings.warn("BeautifulSoup cannot represent elements in any namespace", DataLossWarning)
+ return Element(Tag(self.soup, name), self.soup, namespace)
+
+ def commentClass(self, data):
+ return TextNode(Comment(data), self.soup)
+
+ def fragmentClass(self):
+ self.soup = BeautifulSoup("")
+ self.soup.name = "[document_fragment]"
+ return Element(self.soup, self.soup, None)
+
+ def appendChild(self, node):
+ self.soup.insert(len(self.soup.contents), node.element)
+
+ def testSerializer(self, element):
+ return testSerializer(element)
+
+ def getDocument(self):
+ return self.soup
+
+ def getFragment(self):
+ return _base.TreeBuilder.getFragment(self).element
+
+def testSerializer(element):
+ import re
+ rv = []
+ def serializeElement(element, indent=0):
+ if isinstance(element, Declaration):
+ doctype_regexp = r'DOCTYPE\s+(?P[^\s]*)( PUBLIC "(?P.*)" "(?P.*)"| SYSTEM "(?P.*)")?'
+ m = re.compile(doctype_regexp).match(element.string)
+ assert m is not None, "DOCTYPE did not match expected format"
+ name = m.group('name')
+ publicId = m.group('publicId')
+ if publicId is not None:
+ systemId = m.group('systemId1') or ""
+ else:
+ systemId = m.group('systemId2')
+
+ if publicId is not None or systemId is not None:
+ rv.append("""|%s"""%
+ (' '*indent, name, publicId or "", systemId or ""))
+ else:
+ rv.append("|%s"%(' '*indent, name))
+
+ elif isinstance(element, BeautifulSoup):
+ if element.name == "[document_fragment]":
+ rv.append("#document-fragment")
+ else:
+ rv.append("#document")
+
+ elif isinstance(element, Comment):
+ rv.append("|%s"%(' '*indent, element.string))
+ elif isinstance(element, unicode):
+ rv.append("|%s\"%s\"" %(' '*indent, element))
+ else:
+ rv.append("|%s<%s>"%(' '*indent, element.name))
+ if element.attrs:
+ for name, value in sorted(element.attrs):
+ rv.append('|%s%s="%s"' % (' '*(indent+2), name, value))
+ indent += 2
+ if hasattr(element, "contents"):
+ for child in element.contents:
+ serializeElement(child, indent)
+ serializeElement(element, 0)
+
+ return "\n".join(rv)
diff --git a/libs/html5lib/treewalkers/__init__.py b/libs/html5lib/treewalkers/__init__.py
new file mode 100644
index 00000000..3a606a8b
--- /dev/null
+++ b/libs/html5lib/treewalkers/__init__.py
@@ -0,0 +1,52 @@
+"""A collection of modules for iterating through different kinds of
+tree, generating tokens identical to those produced by the tokenizer
+module.
+
+To create a tree walker for a new type of tree, you need to do
+implement a tree walker object (called TreeWalker by convention) that
+implements a 'serialize' method taking a tree as sole argument and
+returning an iterator generating tokens.
+"""
+
+treeWalkerCache = {}
+
+def getTreeWalker(treeType, implementation=None, **kwargs):
+ """Get a TreeWalker class for various types of tree with built-in support
+
+ treeType - the name of the tree type required (case-insensitive). Supported
+ values are "simpletree", "dom", "etree" and "beautifulsoup"
+
+ "simpletree" - a built-in DOM-ish tree type with support for some
+ more pythonic idioms.
+ "dom" - The xml.dom.minidom DOM implementation
+ "pulldom" - The xml.dom.pulldom event stream
+ "etree" - A generic walker for tree implementations exposing an
+ elementtree-like interface (known to work with
+ ElementTree, cElementTree and lxml.etree).
+ "lxml" - Optimized walker for lxml.etree
+ "beautifulsoup" - Beautiful soup (if installed)
+ "genshi" - a Genshi stream
+
+ implementation - (Currently applies to the "etree" tree type only). A module
+ implementing the tree type e.g. xml.etree.ElementTree or
+ cElementTree."""
+
+ treeType = treeType.lower()
+ if treeType not in treeWalkerCache:
+ if treeType in ("dom", "pulldom", "simpletree"):
+ mod = __import__(treeType, globals())
+ treeWalkerCache[treeType] = mod.TreeWalker
+ elif treeType == "genshi":
+ import genshistream
+ treeWalkerCache[treeType] = genshistream.TreeWalker
+ elif treeType == "beautifulsoup":
+ import soup
+ treeWalkerCache[treeType] = soup.TreeWalker
+ elif treeType == "lxml":
+ import lxmletree
+ treeWalkerCache[treeType] = lxmletree.TreeWalker
+ elif treeType == "etree":
+ import etree
+ # XXX: NEVER cache here, caching is done in the etree submodule
+ return etree.getETreeModule(implementation, **kwargs).TreeWalker
+ return treeWalkerCache.get(treeType)
diff --git a/libs/html5lib/treewalkers/_base.py b/libs/html5lib/treewalkers/_base.py
new file mode 100644
index 00000000..5929ba05
--- /dev/null
+++ b/libs/html5lib/treewalkers/_base.py
@@ -0,0 +1,176 @@
+import gettext
+_ = gettext.gettext
+
+from html5lib.constants import voidElements, spaceCharacters
+spaceCharacters = u"".join(spaceCharacters)
+
+class TreeWalker(object):
+ def __init__(self, tree):
+ self.tree = tree
+
+ def __iter__(self):
+ raise NotImplementedError
+
+ def error(self, msg):
+ return {"type": "SerializeError", "data": msg}
+
+ def normalizeAttrs(self, attrs):
+ newattrs = {}
+ if attrs:
+ #TODO: treewalkers should always have attrs
+ for (namespace,name),value in attrs.iteritems():
+ namespace = unicode(namespace) if namespace else None
+ name = unicode(name)
+ value = unicode(value)
+ newattrs[(namespace,name)] = value
+ return newattrs
+
+ def emptyTag(self, namespace, name, attrs, hasChildren=False):
+ yield {"type": "EmptyTag", "name": unicode(name),
+ "namespace":unicode(namespace),
+ "data": self.normalizeAttrs(attrs)}
+ if hasChildren:
+ yield self.error(_("Void element has children"))
+
+ def startTag(self, namespace, name, attrs):
+ return {"type": "StartTag",
+ "name": unicode(name),
+ "namespace":unicode(namespace),
+ "data": self.normalizeAttrs(attrs)}
+
+ def endTag(self, namespace, name):
+ return {"type": "EndTag",
+ "name": unicode(name),
+ "namespace":unicode(namespace),
+ "data": {}}
+
+ def text(self, data):
+ data = unicode(data)
+ middle = data.lstrip(spaceCharacters)
+ left = data[:len(data)-len(middle)]
+ if left:
+ yield {"type": "SpaceCharacters", "data": left}
+ data = middle
+ middle = data.rstrip(spaceCharacters)
+ right = data[len(middle):]
+ if middle:
+ yield {"type": "Characters", "data": middle}
+ if right:
+ yield {"type": "SpaceCharacters", "data": right}
+
+ def comment(self, data):
+ return {"type": "Comment", "data": unicode(data)}
+
+ def doctype(self, name, publicId=None, systemId=None, correct=True):
+ return {"type": "Doctype",
+ "name": name is not None and unicode(name) or u"",
+ "publicId": publicId,
+ "systemId": systemId,
+ "correct": correct}
+
+ def entity(self, name):
+ return {"type": "Entity", "name": unicode(name)}
+
+ def unknown(self, nodeType):
+ return self.error(_("Unknown node type: ") + nodeType)
+
+class RecursiveTreeWalker(TreeWalker):
+ def walkChildren(self, node):
+ raise NodeImplementedError
+
+ def element(self, node, namespace, name, attrs, hasChildren):
+ if name in voidElements:
+ for token in self.emptyTag(namespace, name, attrs, hasChildren):
+ yield token
+ else:
+ yield self.startTag(name, attrs)
+ if hasChildren:
+ for token in self.walkChildren(node):
+ yield token
+ yield self.endTag(name)
+
+from xml.dom import Node
+
+DOCUMENT = Node.DOCUMENT_NODE
+DOCTYPE = Node.DOCUMENT_TYPE_NODE
+TEXT = Node.TEXT_NODE
+ELEMENT = Node.ELEMENT_NODE
+COMMENT = Node.COMMENT_NODE
+ENTITY = Node.ENTITY_NODE
+UNKNOWN = "<#UNKNOWN#>"
+
+class NonRecursiveTreeWalker(TreeWalker):
+ def getNodeDetails(self, node):
+ raise NotImplementedError
+
+ def getFirstChild(self, node):
+ raise NotImplementedError
+
+ def getNextSibling(self, node):
+ raise NotImplementedError
+
+ def getParentNode(self, node):
+ raise NotImplementedError
+
+ def __iter__(self):
+ currentNode = self.tree
+ while currentNode is not None:
+ details = self.getNodeDetails(currentNode)
+ type, details = details[0], details[1:]
+ hasChildren = False
+ endTag = None
+
+ if type == DOCTYPE:
+ yield self.doctype(*details)
+
+ elif type == TEXT:
+ for token in self.text(*details):
+ yield token
+
+ elif type == ELEMENT:
+ namespace, name, attributes, hasChildren = details
+ if name in voidElements:
+ for token in self.emptyTag(namespace, name, attributes,
+ hasChildren):
+ yield token
+ hasChildren = False
+ else:
+ endTag = name
+ yield self.startTag(namespace, name, attributes)
+
+ elif type == COMMENT:
+ yield self.comment(details[0])
+
+ elif type == ENTITY:
+ yield self.entity(details[0])
+
+ elif type == DOCUMENT:
+ hasChildren = True
+
+ else:
+ yield self.unknown(details[0])
+
+ if hasChildren:
+ firstChild = self.getFirstChild(currentNode)
+ else:
+ firstChild = None
+
+ if firstChild is not None:
+ currentNode = firstChild
+ else:
+ while currentNode is not None:
+ details = self.getNodeDetails(currentNode)
+ type, details = details[0], details[1:]
+ if type == ELEMENT:
+ namespace, name, attributes, hasChildren = details
+ if name not in voidElements:
+ yield self.endTag(namespace, name)
+ if self.tree is currentNode:
+ currentNode = None
+ break
+ nextSibling = self.getNextSibling(currentNode)
+ if nextSibling is not None:
+ currentNode = nextSibling
+ break
+ else:
+ currentNode = self.getParentNode(currentNode)
diff --git a/libs/html5lib/treewalkers/dom.py b/libs/html5lib/treewalkers/dom.py
new file mode 100644
index 00000000..383b46cb
--- /dev/null
+++ b/libs/html5lib/treewalkers/dom.py
@@ -0,0 +1,41 @@
+from xml.dom import Node
+
+import gettext
+_ = gettext.gettext
+
+import _base
+from html5lib.constants import voidElements
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+ def getNodeDetails(self, node):
+ if node.nodeType == Node.DOCUMENT_TYPE_NODE:
+ return _base.DOCTYPE, node.name, node.publicId, node.systemId
+
+ elif node.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE):
+ return _base.TEXT, node.nodeValue
+
+ elif node.nodeType == Node.ELEMENT_NODE:
+ attrs = {}
+ for attr in node.attributes.keys():
+ attr = node.getAttributeNode(attr)
+ attrs[(attr.namespaceURI,attr.localName)] = attr.value
+ return (_base.ELEMENT, node.namespaceURI, node.nodeName,
+ attrs, node.hasChildNodes())
+
+ elif node.nodeType == Node.COMMENT_NODE:
+ return _base.COMMENT, node.nodeValue
+
+ elif node.nodeType in (Node.DOCUMENT_NODE, Node.DOCUMENT_FRAGMENT_NODE):
+ return (_base.DOCUMENT,)
+
+ else:
+ return _base.UNKNOWN, node.nodeType
+
+ def getFirstChild(self, node):
+ return node.firstChild
+
+ def getNextSibling(self, node):
+ return node.nextSibling
+
+ def getParentNode(self, node):
+ return node.parentNode
diff --git a/libs/html5lib/treewalkers/etree.py b/libs/html5lib/treewalkers/etree.py
new file mode 100644
index 00000000..13b03194
--- /dev/null
+++ b/libs/html5lib/treewalkers/etree.py
@@ -0,0 +1,141 @@
+import gettext
+_ = gettext.gettext
+
+try:
+ from types import ModuleType
+except:
+ from new import module as ModuleType
+import copy
+import re
+
+import _base
+from html5lib.constants import voidElements
+
+tag_regexp = re.compile("{([^}]*)}(.*)")
+
+moduleCache = {}
+
+def getETreeModule(ElementTreeImplementation):
+ name = "_" + ElementTreeImplementation.__name__+"builder"
+ if name in moduleCache:
+ return moduleCache[name]
+ else:
+ mod = ModuleType("_" + ElementTreeImplementation.__name__+"builder")
+ objs = getETreeBuilder(ElementTreeImplementation)
+ mod.__dict__.update(objs)
+ moduleCache[name] = mod
+ return mod
+
+def getETreeBuilder(ElementTreeImplementation):
+ ElementTree = ElementTreeImplementation
+
+ class TreeWalker(_base.NonRecursiveTreeWalker):
+ """Given the particular ElementTree representation, this implementation,
+ to avoid using recursion, returns "nodes" as tuples with the following
+ content:
+
+ 1. The current element
+
+ 2. The index of the element relative to its parent
+
+ 3. A stack of ancestor elements
+
+ 4. A flag "text", "tail" or None to indicate if the current node is a
+ text node; either the text or tail of the current element (1)
+ """
+ def getNodeDetails(self, node):
+ if isinstance(node, tuple): # It might be the root Element
+ elt, key, parents, flag = node
+ if flag in ("text", "tail"):
+ return _base.TEXT, getattr(elt, flag)
+ else:
+ node = elt
+
+ if not(hasattr(node, "tag")):
+ node = node.getroot()
+
+ if node.tag in ("", ""):
+ return (_base.DOCUMENT,)
+
+ elif node.tag == "":
+ return (_base.DOCTYPE, node.text,
+ node.get("publicId"), node.get("systemId"))
+
+ elif node.tag == ElementTree.Comment:
+ return _base.COMMENT, node.text
+
+ else:
+ assert type(node.tag) in (str, unicode), type(node.tag)
+ #This is assumed to be an ordinary element
+ match = tag_regexp.match(node.tag)
+ if match:
+ namespace, tag = match.groups()
+ else:
+ namespace = None
+ tag = node.tag
+ attrs = {}
+ for name, value in node.attrib.items():
+ match = tag_regexp.match(name)
+ if match:
+ attrs[(match.group(1),match.group(2))] = value
+ else:
+ attrs[(None,name)] = value
+ return (_base.ELEMENT, namespace, tag,
+ attrs, len(node) or node.text)
+
+ def getFirstChild(self, node):
+ if isinstance(node, tuple):
+ element, key, parents, flag = node
+ else:
+ element, key, parents, flag = node, None, [], None
+
+ if flag in ("text", "tail"):
+ return None
+ else:
+ if element.text:
+ return element, key, parents, "text"
+ elif len(element):
+ parents.append(element)
+ return element[0], 0, parents, None
+ else:
+ return None
+
+ def getNextSibling(self, node):
+ if isinstance(node, tuple):
+ element, key, parents, flag = node
+ else:
+ return None
+
+ if flag == "text":
+ if len(element):
+ parents.append(element)
+ return element[0], 0, parents, None
+ else:
+ return None
+ else:
+ if element.tail and flag != "tail":
+ return element, key, parents, "tail"
+ elif key < len(parents[-1]) - 1:
+ return parents[-1][key+1], key+1, parents, None
+ else:
+ return None
+
+ def getParentNode(self, node):
+ if isinstance(node, tuple):
+ element, key, parents, flag = node
+ else:
+ return None
+
+ if flag == "text":
+ if not parents:
+ return element
+ else:
+ return element, key, parents, None
+ else:
+ parent = parents.pop()
+ if not parents:
+ return parent
+ else:
+ return parent, list(parents[-1]).index(parent), parents, None
+
+ return locals()
diff --git a/libs/html5lib/treewalkers/genshistream.py b/libs/html5lib/treewalkers/genshistream.py
new file mode 100644
index 00000000..ef71a83e
--- /dev/null
+++ b/libs/html5lib/treewalkers/genshistream.py
@@ -0,0 +1,70 @@
+from genshi.core import START, END, XML_NAMESPACE, DOCTYPE, TEXT
+from genshi.core import START_NS, END_NS, START_CDATA, END_CDATA, PI, COMMENT
+from genshi.output import NamespaceFlattener
+
+import _base
+
+from html5lib.constants import voidElements
+
+class TreeWalker(_base.TreeWalker):
+ def __iter__(self):
+ depth = 0
+ ignore_until = None
+ previous = None
+ for event in self.tree:
+ if previous is not None:
+ if previous[0] == START:
+ depth += 1
+ if ignore_until <= depth:
+ ignore_until = None
+ if ignore_until is None:
+ for token in self.tokens(previous, event):
+ yield token
+ if token["type"] == "EmptyTag":
+ ignore_until = depth
+ if previous[0] == END:
+ depth -= 1
+ previous = event
+ if previous is not None:
+ if ignore_until is None or ignore_until <= depth:
+ for token in self.tokens(previous, None):
+ yield token
+ elif ignore_until is not None:
+ raise ValueError("Illformed DOM event stream: void element without END_ELEMENT")
+
+ def tokens(self, event, next):
+ kind, data, pos = event
+ if kind == START:
+ tag, attrib = data
+ name = tag.localname
+ namespace = tag.namespace
+ if tag in voidElements:
+ for token in self.emptyTag(namespace, name, list(attrib),
+ not next or next[0] != END
+ or next[1] != tag):
+ yield token
+ else:
+ yield self.startTag(namespace, name, list(attrib))
+
+ elif kind == END:
+ name = data.localname
+ namespace = data.namespace
+ if name not in voidElements:
+ yield self.endTag(namespace, name)
+
+ elif kind == COMMENT:
+ yield self.comment(data)
+
+ elif kind == TEXT:
+ for token in self.text(data):
+ yield token
+
+ elif kind == DOCTYPE:
+ yield self.doctype(*data)
+
+ elif kind in (XML_NAMESPACE, DOCTYPE, START_NS, END_NS, \
+ START_CDATA, END_CDATA, PI):
+ pass
+
+ else:
+ yield self.unknown(kind)
diff --git a/libs/html5lib/treewalkers/lxmletree.py b/libs/html5lib/treewalkers/lxmletree.py
new file mode 100644
index 00000000..5f328059
--- /dev/null
+++ b/libs/html5lib/treewalkers/lxmletree.py
@@ -0,0 +1,186 @@
+from lxml import etree
+from html5lib.treebuilders.etree import tag_regexp
+
+from gettext import gettext
+_ = gettext
+
+import _base
+
+from html5lib.constants import voidElements
+from html5lib import ihatexml
+
+class Root(object):
+ def __init__(self, et):
+ self.elementtree = et
+ self.children = []
+ if et.docinfo.internalDTD:
+ self.children.append(Doctype(self, et.docinfo.root_name,
+ et.docinfo.public_id,
+ et.docinfo.system_url))
+ root = et.getroot()
+ node = root
+
+ while node.getprevious() is not None:
+ node = node.getprevious()
+ while node is not None:
+ self.children.append(node)
+ node = node.getnext()
+
+ self.text = None
+ self.tail = None
+
+ def __getitem__(self, key):
+ return self.children[key]
+
+ def getnext(self):
+ return None
+
+ def __len__(self):
+ return 1
+
+class Doctype(object):
+ def __init__(self, root_node, name, public_id, system_id):
+ self.root_node = root_node
+ self.name = name
+ self.public_id = public_id
+ self.system_id = system_id
+
+ self.text = None
+ self.tail = None
+
+ def getnext(self):
+ return self.root_node.children[1]
+
+class FragmentRoot(Root):
+ def __init__(self, children):
+ self.children = [FragmentWrapper(self, child) for child in children]
+ self.text = self.tail = None
+
+ def getnext(self):
+ return None
+
+class FragmentWrapper(object):
+ def __init__(self, fragment_root, obj):
+ self.root_node = fragment_root
+ self.obj = obj
+ if hasattr(self.obj, 'text'):
+ self.text = self.obj.text
+ else:
+ self.text = None
+ if hasattr(self.obj, 'tail'):
+ self.tail = self.obj.tail
+ else:
+ self.tail = None
+ self.isstring = isinstance(obj, basestring)
+
+ def __getattr__(self, name):
+ return getattr(self.obj, name)
+
+ def getnext(self):
+ siblings = self.root_node.children
+ idx = siblings.index(self)
+ if idx < len(siblings) - 1:
+ return siblings[idx + 1]
+ else:
+ return None
+
+ def __getitem__(self, key):
+ return self.obj[key]
+
+ def __nonzero__(self):
+ return bool(self.obj)
+
+ def getparent(self):
+ return None
+
+ def __str__(self):
+ return str(self.obj)
+
+ def __unicode__(self):
+ return unicode(self.obj)
+
+ def __len__(self):
+ return len(self.obj)
+
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+ def __init__(self, tree):
+ if hasattr(tree, "getroot"):
+ tree = Root(tree)
+ elif isinstance(tree, list):
+ tree = FragmentRoot(tree)
+ _base.NonRecursiveTreeWalker.__init__(self, tree)
+ self.filter = ihatexml.InfosetFilter()
+ def getNodeDetails(self, node):
+ if isinstance(node, tuple): # Text node
+ node, key = node
+ assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+ return _base.TEXT, getattr(node, key)
+
+ elif isinstance(node, Root):
+ return (_base.DOCUMENT,)
+
+ elif isinstance(node, Doctype):
+ return _base.DOCTYPE, node.name, node.public_id, node.system_id
+
+ elif isinstance(node, FragmentWrapper) and node.isstring:
+ return _base.TEXT, node
+
+ elif node.tag == etree.Comment:
+ return _base.COMMENT, node.text
+
+ elif node.tag == etree.Entity:
+ return _base.ENTITY, node.text[1:-1] # strip &;
+
+ else:
+ #This is assumed to be an ordinary element
+ match = tag_regexp.match(node.tag)
+ if match:
+ namespace, tag = match.groups()
+ else:
+ namespace = None
+ tag = node.tag
+ attrs = {}
+ for name, value in node.attrib.items():
+ match = tag_regexp.match(name)
+ if match:
+ attrs[(match.group(1),match.group(2))] = value
+ else:
+ attrs[(None,name)] = value
+ return (_base.ELEMENT, namespace, self.filter.fromXmlName(tag),
+ attrs, len(node) > 0 or node.text)
+
+ def getFirstChild(self, node):
+ assert not isinstance(node, tuple), _("Text nodes have no children")
+
+ assert len(node) or node.text, "Node has no children"
+ if node.text:
+ return (node, "text")
+ else:
+ return node[0]
+
+ def getNextSibling(self, node):
+ if isinstance(node, tuple): # Text node
+ node, key = node
+ assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+ if key == "text":
+ # XXX: we cannot use a "bool(node) and node[0] or None" construct here
+ # because node[0] might evaluate to False if it has no child element
+ if len(node):
+ return node[0]
+ else:
+ return None
+ else: # tail
+ return node.getnext()
+
+ return node.tail and (node, "tail") or node.getnext()
+
+ def getParentNode(self, node):
+ if isinstance(node, tuple): # Text node
+ node, key = node
+ assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+ if key == "text":
+ return node
+ # else: fallback to "normal" processing
+
+ return node.getparent()
diff --git a/libs/html5lib/treewalkers/pulldom.py b/libs/html5lib/treewalkers/pulldom.py
new file mode 100644
index 00000000..1f8b95b8
--- /dev/null
+++ b/libs/html5lib/treewalkers/pulldom.py
@@ -0,0 +1,60 @@
+from xml.dom.pulldom import START_ELEMENT, END_ELEMENT, \
+ COMMENT, IGNORABLE_WHITESPACE, CHARACTERS
+
+import _base
+
+from html5lib.constants import voidElements
+
+class TreeWalker(_base.TreeWalker):
+ def __iter__(self):
+ ignore_until = None
+ previous = None
+ for event in self.tree:
+ if previous is not None and \
+ (ignore_until is None or previous[1] is ignore_until):
+ if previous[1] is ignore_until:
+ ignore_until = None
+ for token in self.tokens(previous, event):
+ yield token
+ if token["type"] == "EmptyTag":
+ ignore_until = previous[1]
+ previous = event
+ if ignore_until is None or previous[1] is ignore_until:
+ for token in self.tokens(previous, None):
+ yield token
+ elif ignore_until is not None:
+ raise ValueError("Illformed DOM event stream: void element without END_ELEMENT")
+
+ def tokens(self, event, next):
+ type, node = event
+ if type == START_ELEMENT:
+ name = node.nodeName
+ namespace = node.namespaceURI
+ attrs = {}
+ for attr in node.attributes.keys():
+ attr = node.getAttributeNode(attr)
+ attrs[(attr.namespaceURI,attr.localName)] = attr.value
+ if name in voidElements:
+ for token in self.emptyTag(namespace,
+ name,
+ attrs,
+ not next or next[1] is not node):
+ yield token
+ else:
+ yield self.startTag(namespace, name, attrs)
+
+ elif type == END_ELEMENT:
+ name = node.nodeName
+ namespace = node.namespaceURI
+ if name not in voidElements:
+ yield self.endTag(namespace, name)
+
+ elif type == COMMENT:
+ yield self.comment(node.nodeValue)
+
+ elif type in (IGNORABLE_WHITESPACE, CHARACTERS):
+ for token in self.text(node.nodeValue):
+ yield token
+
+ else:
+ yield self.unknown(type)
diff --git a/libs/html5lib/treewalkers/simpletree.py b/libs/html5lib/treewalkers/simpletree.py
new file mode 100644
index 00000000..9e6bd4c5
--- /dev/null
+++ b/libs/html5lib/treewalkers/simpletree.py
@@ -0,0 +1,78 @@
+import gettext
+_ = gettext.gettext
+
+import _base
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+ """Given that simpletree has no performant way of getting a node's
+ next sibling, this implementation returns "nodes" as tuples with the
+ following content:
+
+ 1. The parent Node (Element, Document or DocumentFragment)
+
+ 2. The child index of the current node in its parent's children list
+
+ 3. A list used as a stack of all ancestors. It is a pair tuple whose
+ first item is a parent Node and second item is a child index.
+ """
+
+ def getNodeDetails(self, node):
+ if isinstance(node, tuple): # It might be the root Node
+ parent, idx, parents = node
+ node = parent.childNodes[idx]
+
+ # testing node.type allows us not to import treebuilders.simpletree
+ if node.type in (1, 2): # Document or DocumentFragment
+ return (_base.DOCUMENT,)
+
+ elif node.type == 3: # DocumentType
+ return _base.DOCTYPE, node.name, node.publicId, node.systemId
+
+ elif node.type == 4: # TextNode
+ return _base.TEXT, node.value
+
+ elif node.type == 5: # Element
+ attrs = {}
+ for name, value in node.attributes.items():
+ if isinstance(name, tuple):
+ attrs[(name[2],name[1])] = value
+ else:
+ attrs[(None,name)] = value
+ return (_base.ELEMENT, node.namespace, node.name,
+ attrs, node.hasContent())
+
+ elif node.type == 6: # CommentNode
+ return _base.COMMENT, node.data
+
+ else:
+ return _node.UNKNOWN, node.type
+
+ def getFirstChild(self, node):
+ if isinstance(node, tuple): # It might be the root Node
+ parent, idx, parents = node
+ parents.append((parent, idx))
+ node = parent.childNodes[idx]
+ else:
+ parents = []
+
+ assert node.hasContent(), "Node has no children"
+ return (node, 0, parents)
+
+ def getNextSibling(self, node):
+ assert isinstance(node, tuple), "Node is not a tuple: " + str(node)
+ parent, idx, parents = node
+ idx += 1
+ if len(parent.childNodes) > idx:
+ return (parent, idx, parents)
+ else:
+ return None
+
+ def getParentNode(self, node):
+ assert isinstance(node, tuple)
+ parent, idx, parents = node
+ if parents:
+ parent, idx = parents.pop()
+ return parent, idx, parents
+ else:
+ # HACK: We could return ``parent`` but None will stop the algorithm the same way
+ return None
diff --git a/libs/html5lib/treewalkers/soup.py b/libs/html5lib/treewalkers/soup.py
new file mode 100644
index 00000000..fca65ecb
--- /dev/null
+++ b/libs/html5lib/treewalkers/soup.py
@@ -0,0 +1,60 @@
+import re
+import gettext
+_ = gettext.gettext
+
+from BeautifulSoup import BeautifulSoup, Declaration, Comment, Tag
+from html5lib.constants import namespaces
+import _base
+
+class TreeWalker(_base.NonRecursiveTreeWalker):
+ doctype_regexp = re.compile(
+ r'DOCTYPE\s+(?P[^\s]*)(\s*PUBLIC\s*"(?P.*)"\s*"(?P.*)"|\s*SYSTEM\s*"(?P.*)")?')
+ def getNodeDetails(self, node):
+ if isinstance(node, BeautifulSoup): # Document or DocumentFragment
+ return (_base.DOCUMENT,)
+
+ elif isinstance(node, Declaration): # DocumentType
+ string = unicode(node.string)
+ #Slice needed to remove markup added during unicode conversion,
+ #but only in some versions of BeautifulSoup/Python
+ if string.startswith(''):
+ string = string[2:-1]
+ m = self.doctype_regexp.match(string)
+ #This regexp approach seems wrong and fragile
+ #but beautiful soup stores the doctype as a single thing and we want the seperate bits
+ #It should work as long as the tree is created by html5lib itself but may be wrong if it's
+ #been modified at all
+ #We could just feed to it a html5lib tokenizer, I guess...
+ assert m is not None, "DOCTYPE did not match expected format"
+
+ name = m.group('name')
+ publicId = m.group('publicId')
+ if publicId is not None:
+ systemId = m.group('systemId1')
+ else:
+ systemId = m.group('systemId2')
+ return _base.DOCTYPE, name, publicId or "", systemId or ""
+
+ elif isinstance(node, Comment):
+ string = unicode(node.string)
+ if string.startswith(''):
+ string = string[4:-3]
+ return _base.COMMENT, string
+
+ elif isinstance(node, unicode): # TextNode
+ return _base.TEXT, node
+
+ elif isinstance(node, Tag): # Element
+ return (_base.ELEMENT, namespaces["html"], node.name,
+ dict(node.attrs).items(), node.contents)
+ else:
+ return _base.UNKNOWN, node.__class__.__name__
+
+ def getFirstChild(self, node):
+ return node.contents[0]
+
+ def getNextSibling(self, node):
+ return node.nextSibling
+
+ def getParentNode(self, node):
+ return node.parent
diff --git a/libs/html5lib/utils.py b/libs/html5lib/utils.py
new file mode 100644
index 00000000..d53f6788
--- /dev/null
+++ b/libs/html5lib/utils.py
@@ -0,0 +1,175 @@
+try:
+ frozenset
+except NameError:
+ #Import from the sets module for python 2.3
+ from sets import Set as set
+ from sets import ImmutableSet as frozenset
+
+class MethodDispatcher(dict):
+ """Dict with 2 special properties:
+
+ On initiation, keys that are lists, sets or tuples are converted to
+ multiple keys so accessing any one of the items in the original
+ list-like object returns the matching value
+
+ md = MethodDispatcher({("foo", "bar"):"baz"})
+ md["foo"] == "baz"
+
+ A default value which can be set through the default attribute.
+ """
+
+ def __init__(self, items=()):
+ # Using _dictEntries instead of directly assigning to self is about
+ # twice as fast. Please do careful performance testing before changing
+ # anything here.
+ _dictEntries = []
+ for name,value in items:
+ if type(name) in (list, tuple, frozenset, set):
+ for item in name:
+ _dictEntries.append((item, value))
+ else:
+ _dictEntries.append((name, value))
+ dict.__init__(self, _dictEntries)
+ self.default = None
+
+ def __getitem__(self, key):
+ return dict.get(self, key, self.default)
+
+#Pure python implementation of deque taken from the ASPN Python Cookbook
+#Original code by Raymond Hettinger
+
+class deque(object):
+
+ def __init__(self, iterable=(), maxsize=-1):
+ if not hasattr(self, 'data'):
+ self.left = self.right = 0
+ self.data = {}
+ self.maxsize = maxsize
+ self.extend(iterable)
+
+ def append(self, x):
+ self.data[self.right] = x
+ self.right += 1
+ if self.maxsize != -1 and len(self) > self.maxsize:
+ self.popleft()
+
+ def appendleft(self, x):
+ self.left -= 1
+ self.data[self.left] = x
+ if self.maxsize != -1 and len(self) > self.maxsize:
+ self.pop()
+
+ def pop(self):
+ if self.left == self.right:
+ raise IndexError('cannot pop from empty deque')
+ self.right -= 1
+ elem = self.data[self.right]
+ del self.data[self.right]
+ return elem
+
+ def popleft(self):
+ if self.left == self.right:
+ raise IndexError('cannot pop from empty deque')
+ elem = self.data[self.left]
+ del self.data[self.left]
+ self.left += 1
+ return elem
+
+ def clear(self):
+ self.data.clear()
+ self.left = self.right = 0
+
+ def extend(self, iterable):
+ for elem in iterable:
+ self.append(elem)
+
+ def extendleft(self, iterable):
+ for elem in iterable:
+ self.appendleft(elem)
+
+ def rotate(self, n=1):
+ if self:
+ n %= len(self)
+ for i in xrange(n):
+ self.appendleft(self.pop())
+
+ def __getitem__(self, i):
+ if i < 0:
+ i += len(self)
+ try:
+ return self.data[i + self.left]
+ except KeyError:
+ raise IndexError
+
+ def __setitem__(self, i, value):
+ if i < 0:
+ i += len(self)
+ try:
+ self.data[i + self.left] = value
+ except KeyError:
+ raise IndexError
+
+ def __delitem__(self, i):
+ size = len(self)
+ if not (-size <= i < size):
+ raise IndexError
+ data = self.data
+ if i < 0:
+ i += size
+ for j in xrange(self.left+i, self.right-1):
+ data[j] = data[j+1]
+ self.pop()
+
+ def __len__(self):
+ return self.right - self.left
+
+ def __cmp__(self, other):
+ if type(self) != type(other):
+ return cmp(type(self), type(other))
+ return cmp(list(self), list(other))
+
+ def __repr__(self, _track=[]):
+ if id(self) in _track:
+ return '...'
+ _track.append(id(self))
+ r = 'deque(%r)' % (list(self),)
+ _track.remove(id(self))
+ return r
+
+ def __getstate__(self):
+ return (tuple(self),)
+
+ def __setstate__(self, s):
+ self.__init__(s[0])
+
+ def __hash__(self):
+ raise TypeError
+
+ def __copy__(self):
+ return self.__class__(self)
+
+ def __deepcopy__(self, memo={}):
+ from copy import deepcopy
+ result = self.__class__()
+ memo[id(self)] = result
+ result.__init__(deepcopy(tuple(self), memo))
+ return result
+
+#Some utility functions to dal with weirdness around UCS2 vs UCS4
+#python builds
+
+def encodingType():
+ if len() == 2:
+ return "UCS2"
+ else:
+ return "UCS4"
+
+def isSurrogatePair(data):
+ return (len(data) == 2 and
+ ord(data[0]) >= 0xD800 and ord(data[0]) <= 0xDBFF and
+ ord(data[1]) >= 0xDC00 and ord(data[1]) <= 0xDFFF)
+
+def surrogatePairToCodepoint(data):
+ char_val = (0x10000 + (ord(data[0]) - 0xD800) * 0x400 +
+ (ord(data[1]) - 0xDC00))
+ return char_val
diff --git a/libs/oauthlib/__init__.py b/libs/oauthlib/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/libs/oauthlib/common.py b/libs/oauthlib/common.py
new file mode 100644
index 00000000..4cdfd0d4
--- /dev/null
+++ b/libs/oauthlib/common.py
@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+"""
+oauthlib.common
+~~~~~~~~~~~~~~
+
+This module provides data structures and utilities common
+to all implementations of OAuth.
+"""
+
+import re
+import urllib
+import urlparse
+
+
+always_safe = (u'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+ u'abcdefghijklmnopqrstuvwxyz'
+ u'0123456789' u'_.-')
+
+
+def quote(s, safe=u'/'):
+ encoded = s.encode("utf-8")
+ quoted = urllib.quote(encoded, safe)
+ return quoted.decode("utf-8")
+
+
+def unquote(s):
+ encoded = s.encode("utf-8")
+ unquoted = urllib.unquote(encoded)
+ return unquoted.decode("utf-8")
+
+
+def urlencode(params):
+ utf8_params = encode_params_utf8(params)
+ urlencoded = urllib.urlencode(utf8_params)
+ return urlencoded.decode("utf-8")
+
+
+def encode_params_utf8(params):
+ """Ensures that all parameters in a list of 2-element tuples are encoded to
+ bytestrings using UTF-8
+ """
+ encoded = []
+ for k, v in params:
+ encoded.append((
+ k.encode('utf-8') if isinstance(k, unicode) else k,
+ v.encode('utf-8') if isinstance(v, unicode) else v))
+ return encoded
+
+
+def decode_params_utf8(params):
+ """Ensures that all parameters in a list of 2-element tuples are decoded to
+ unicode using UTF-8.
+ """
+ decoded = []
+ for k, v in params:
+ decoded.append((
+ k.decode('utf-8') if isinstance(k, str) else k,
+ v.decode('utf-8') if isinstance(v, str) else v))
+ return decoded
+
+
+urlencoded = set(always_safe) | set(u'=&;%+~')
+
+
+def urldecode(query):
+ """Decode a query string in x-www-form-urlencoded format into a sequence
+ of two-element tuples.
+
+ Unlike urlparse.parse_qsl(..., strict_parsing=True) urldecode will enforce
+ correct formatting of the query string by validation. If validation fails
+ a ValueError will be raised. urllib.parse_qsl will only raise errors if
+ any of name-value pairs omits the equals sign.
+ """
+ # Check if query contains invalid characters
+ if query and not set(query) <= urlencoded:
+ raise ValueError('Invalid characters in query string.')
+
+ # Check for correctly hex encoded values using a regular expression
+ # All encoded values begin with % followed by two hex characters
+ # correct = %00, %A0, %0A, %FF
+ # invalid = %G0, %5H, %PO
+ invalid_hex = u'%[^0-9A-Fa-f]|%[0-9A-Fa-f][^0-9A-Fa-f]'
+ if len(re.findall(invalid_hex, query)):
+ raise ValueError('Invalid hex encoding in query string.')
+
+ query = query.decode('utf-8') if isinstance(query, str) else query
+ # We want to allow queries such as "c2" whereas urlparse.parse_qsl
+ # with the strict_parsing flag will not.
+ params = urlparse.parse_qsl(query, keep_blank_values=True)
+
+ # unicode all the things
+ return decode_params_utf8(params)
+
+
+def extract_params(raw):
+ """Extract parameters and return them as a list of 2-tuples.
+
+ Will successfully extract parameters from urlencoded query strings,
+ dicts, or lists of 2-tuples. Empty strings/dicts/lists will return an
+ empty list of parameters. Any other input will result in a return
+ value of None.
+ """
+ if isinstance(raw, basestring):
+ try:
+ params = urldecode(raw)
+ except ValueError:
+ params = None
+ elif hasattr(raw, '__iter__'):
+ try:
+ dict(raw)
+ except ValueError:
+ params = None
+ except TypeError:
+ params = None
+ else:
+ params = list(raw.items() if isinstance(raw, dict) else raw)
+ params = decode_params_utf8(params)
+ else:
+ params = None
+
+ return params
+
+
+class Request(object):
+ """A malleable representation of a signable HTTP request.
+
+ Body argument may contain any data, but parameters will only be decoded if
+ they are one of:
+
+ * urlencoded query string
+ * dict
+ * list of 2-tuples
+
+ Anything else will be treated as raw body data to be passed through
+ unmolested.
+ """
+
+ def __init__(self, uri, http_method=u'GET', body=None, headers=None):
+ self.uri = uri
+ self.http_method = http_method
+ self.headers = headers or {}
+ self.body = body
+ self.decoded_body = extract_params(body)
+ self.oauth_params = []
+
+ @property
+ def uri_query(self):
+ return urlparse.urlparse(self.uri).query
+
+ @property
+ def uri_query_params(self):
+ return urlparse.parse_qsl(self.uri_query, keep_blank_values=True,
+ strict_parsing=True)
diff --git a/libs/oauthlib/oauth1/__init__.py b/libs/oauthlib/oauth1/__init__.py
new file mode 100644
index 00000000..ef692b57
--- /dev/null
+++ b/libs/oauthlib/oauth1/__init__.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+"""
+oauthlib.oauth1
+~~~~~~~~~~~~~~
+
+This module is a wrapper for the most recent implementation of OAuth 1.0 Client
+and Server classes.
+"""
+
+from .rfc5849 import Client, Server
+
diff --git a/libs/oauthlib/oauth1/rfc5849/__init__.py b/libs/oauthlib/oauth1/rfc5849/__init__.py
new file mode 100644
index 00000000..03fb8b25
--- /dev/null
+++ b/libs/oauthlib/oauth1/rfc5849/__init__.py
@@ -0,0 +1,350 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+"""
+oauthlib.oauth1.rfc5849
+~~~~~~~~~~~~~~
+
+This module is an implementation of various logic needed
+for signing and checking OAuth 1.0 RFC 5849 requests.
+"""
+
+import logging
+import urlparse
+
+from oauthlib.common import Request, urlencode
+from . import parameters, signature, utils
+
+SIGNATURE_HMAC = u"HMAC-SHA1"
+SIGNATURE_RSA = u"RSA-SHA1"
+SIGNATURE_PLAINTEXT = u"PLAINTEXT"
+SIGNATURE_METHODS = (SIGNATURE_HMAC, SIGNATURE_RSA, SIGNATURE_PLAINTEXT)
+
+SIGNATURE_TYPE_AUTH_HEADER = u'AUTH_HEADER'
+SIGNATURE_TYPE_QUERY = u'QUERY'
+SIGNATURE_TYPE_BODY = u'BODY'
+
+CONTENT_TYPE_FORM_URLENCODED = u'application/x-www-form-urlencoded'
+
+
+class Client(object):
+ """A client used to sign OAuth 1.0 RFC 5849 requests"""
+ def __init__(self, client_key,
+ client_secret=None,
+ resource_owner_key=None,
+ resource_owner_secret=None,
+ callback_uri=None,
+ signature_method=SIGNATURE_HMAC,
+ signature_type=SIGNATURE_TYPE_AUTH_HEADER,
+ rsa_key=None, verifier=None):
+ self.client_key = client_key
+ self.client_secret = client_secret
+ self.resource_owner_key = resource_owner_key
+ self.resource_owner_secret = resource_owner_secret
+ self.signature_method = signature_method
+ self.signature_type = signature_type
+ self.callback_uri = callback_uri
+ self.rsa_key = rsa_key
+ self.verifier = verifier
+
+ if self.signature_method == SIGNATURE_RSA and self.rsa_key is None:
+ raise ValueError('rsa_key is required when using RSA signature method.')
+
+ def get_oauth_signature(self, request):
+ """Get an OAuth signature to be used in signing a request
+ """
+ if self.signature_method == SIGNATURE_PLAINTEXT:
+ # fast-path
+ return signature.sign_plaintext(self.client_secret,
+ self.resource_owner_secret)
+
+ uri, headers, body = self._render(request)
+
+ collected_params = signature.collect_parameters(
+ uri_query=urlparse.urlparse(uri).query,
+ body=body,
+ headers=headers)
+ logging.debug("Collected params: {0}".format(collected_params))
+
+ normalized_params = signature.normalize_parameters(collected_params)
+ normalized_uri = signature.normalize_base_string_uri(request.uri)
+ logging.debug("Normalized params: {0}".format(normalized_params))
+ logging.debug("Normalized URI: {0}".format(normalized_uri))
+
+ base_string = signature.construct_base_string(request.http_method,
+ normalized_uri, normalized_params)
+
+ logging.debug("Base signing string: {0}".format(base_string))
+
+ if self.signature_method == SIGNATURE_HMAC:
+ sig = signature.sign_hmac_sha1(base_string, self.client_secret,
+ self.resource_owner_secret)
+ elif self.signature_method == SIGNATURE_RSA:
+ sig = signature.sign_rsa_sha1(base_string, self.rsa_key)
+ else:
+ sig = signature.sign_plaintext(self.client_secret,
+ self.resource_owner_secret)
+
+ logging.debug("Signature: {0}".format(sig))
+ return sig
+
+ def get_oauth_params(self):
+ """Get the basic OAuth parameters to be used in generating a signature.
+ """
+ params = [
+ (u'oauth_nonce', utils.generate_nonce()),
+ (u'oauth_timestamp', utils.generate_timestamp()),
+ (u'oauth_version', u'1.0'),
+ (u'oauth_signature_method', self.signature_method),
+ (u'oauth_consumer_key', self.client_key),
+ ]
+ if self.resource_owner_key:
+ params.append((u'oauth_token', self.resource_owner_key))
+ if self.callback_uri:
+ params.append((u'oauth_callback', self.callback_uri))
+ if self.verifier:
+ params.append((u'oauth_verifier', self.verifier))
+
+ return params
+
+ def _render(self, request, formencode=False):
+ """Render a signed request according to signature type
+
+ Returns a 3-tuple containing the request URI, headers, and body.
+
+ If the formencode argument is True and the body contains parameters, it
+ is escaped and returned as a valid formencoded string.
+ """
+ # TODO what if there are body params on a header-type auth?
+ # TODO what if there are query params on a body-type auth?
+
+ uri, headers, body = request.uri, request.headers, request.body
+
+ # TODO: right now these prepare_* methods are very narrow in scope--they
+ # only affect their little thing. In some cases (for example, with
+ # header auth) it might be advantageous to allow these methods to touch
+ # other parts of the request, like the headers—so the prepare_headers
+ # method could also set the Content-Type header to x-www-form-urlencoded
+ # like the spec requires. This would be a fundamental change though, and
+ # I'm not sure how I feel about it.
+ if self.signature_type == SIGNATURE_TYPE_AUTH_HEADER:
+ headers = parameters.prepare_headers(request.oauth_params, request.headers)
+ elif self.signature_type == SIGNATURE_TYPE_BODY and request.decoded_body is not None:
+ body = parameters.prepare_form_encoded_body(request.oauth_params, request.decoded_body)
+ if formencode:
+ body = urlencode(body)
+ headers['Content-Type'] = u'application/x-www-form-urlencoded'
+ elif self.signature_type == SIGNATURE_TYPE_QUERY:
+ uri = parameters.prepare_request_uri_query(request.oauth_params, request.uri)
+ else:
+ raise ValueError('Unknown signature type specified.')
+
+ return uri, headers, body
+
+ def sign(self, uri, http_method=u'GET', body=None, headers=None):
+ """Sign a request
+
+ Signs an HTTP request with the specified parts.
+
+ Returns a 3-tuple of the signed request's URI, headers, and body.
+ Note that http_method is not returned as it is unaffected by the OAuth
+ signing process.
+
+ The body argument may be a dict, a list of 2-tuples, or a formencoded
+ string. The Content-Type header must be 'application/x-www-form-urlencoded'
+ if it is present.
+
+ If the body argument is not one of the above, it will be returned
+ verbatim as it is unaffected by the OAuth signing process. Attempting to
+ sign a request with non-formencoded data using the OAuth body signature
+ type is invalid and will raise an exception.
+
+ If the body does contain parameters, it will be returned as a properly-
+ formatted formencoded string.
+
+ All string data MUST be unicode. This includes strings inside body
+ dicts, for example.
+ """
+ # normalize request data
+ request = Request(uri, http_method, body, headers)
+
+ # sanity check
+ content_type = request.headers.get('Content-Type', None)
+ multipart = content_type and content_type.startswith('multipart/')
+ should_have_params = content_type == CONTENT_TYPE_FORM_URLENCODED
+ has_params = request.decoded_body is not None
+ # 3.4.1.3.1. Parameter Sources
+ # [Parameters are collected from the HTTP request entity-body, but only
+ # if [...]:
+ # * The entity-body is single-part.
+ if multipart and has_params:
+ raise ValueError("Headers indicate a multipart body but body contains parameters.")
+ # * The entity-body follows the encoding requirements of the
+ # "application/x-www-form-urlencoded" content-type as defined by
+ # [W3C.REC-html40-19980424].
+ elif should_have_params and not has_params:
+ raise ValueError("Headers indicate a formencoded body but body was not decodable.")
+ # * The HTTP request entity-header includes the "Content-Type"
+ # header field set to "application/x-www-form-urlencoded".
+ elif not should_have_params and has_params:
+ raise ValueError("Body contains parameters but Content-Type header was not set.")
+
+ # 3.5.2. Form-Encoded Body
+ # Protocol parameters can be transmitted in the HTTP request entity-
+ # body, but only if the following REQUIRED conditions are met:
+ # o The entity-body is single-part.
+ # o The entity-body follows the encoding requirements of the
+ # "application/x-www-form-urlencoded" content-type as defined by
+ # [W3C.REC-html40-19980424].
+ # o The HTTP request entity-header includes the "Content-Type" header
+ # field set to "application/x-www-form-urlencoded".
+ elif self.signature_type == SIGNATURE_TYPE_BODY and not (
+ should_have_params and has_params and not multipart):
+ raise ValueError('Body signatures may only be used with form-urlencoded content')
+
+ # generate the basic OAuth parameters
+ request.oauth_params = self.get_oauth_params()
+
+ # generate the signature
+ request.oauth_params.append((u'oauth_signature', self.get_oauth_signature(request)))
+
+ # render the signed request and return it
+ return self._render(request, formencode=True)
+
+
+class Server(object):
+ """A server used to verify OAuth 1.0 RFC 5849 requests"""
+ def __init__(self, signature_method=SIGNATURE_HMAC, rsa_key=None):
+ self.signature_method = signature_method
+ self.rsa_key = rsa_key
+
+ def get_client_secret(self, client_key):
+ raise NotImplementedError("Subclasses must implement this function.")
+
+ def get_resource_owner_secret(self, resource_owner_key):
+ raise NotImplementedError("Subclasses must implement this function.")
+
+ def get_signature_type_and_params(self, uri_query, headers, body):
+ signature_types_with_oauth_params = filter(lambda s: s[1], (
+ (SIGNATURE_TYPE_AUTH_HEADER, utils.filter_oauth_params(
+ signature.collect_parameters(headers=headers,
+ exclude_oauth_signature=False))),
+ (SIGNATURE_TYPE_BODY, utils.filter_oauth_params(
+ signature.collect_parameters(body=body,
+ exclude_oauth_signature=False))),
+ (SIGNATURE_TYPE_QUERY, utils.filter_oauth_params(
+ signature.collect_parameters(uri_query=uri_query,
+ exclude_oauth_signature=False))),
+ ))
+
+ if len(signature_types_with_oauth_params) > 1:
+ raise ValueError('oauth_ params must come from only 1 signature type but were found in %s' % ', '.join(
+ [s[0] for s in signature_types_with_oauth_params]))
+ try:
+ signature_type, params = signature_types_with_oauth_params[0]
+ except IndexError:
+ raise ValueError('oauth_ params are missing. Could not determine signature type.')
+
+ return signature_type, dict(params)
+
+ def check_client_key(self, client_key):
+ raise NotImplementedError("Subclasses must implement this function.")
+
+ def check_resource_owner_key(self, client_key, resource_owner_key):
+ raise NotImplementedError("Subclasses must implement this function.")
+
+ def check_timestamp_and_nonce(self, timestamp, nonce):
+ raise NotImplementedError("Subclasses must implement this function.")
+
+ def check_request_signature(self, uri, http_method=u'GET', body='',
+ headers=None):
+ """Check a request's supplied signature to make sure the request is
+ valid.
+
+ Servers should return HTTP status 400 if a ValueError exception
+ is raised and HTTP status 401 on return value False.
+
+ Per `section 3.2`_ of the spec.
+
+ .. _`section 3.2`: http://tools.ietf.org/html/rfc5849#section-3.2
+ """
+ headers = headers or {}
+ signature_type = None
+ # FIXME: urlparse does not return unicode!
+ uri_query = urlparse.urlparse(uri).query
+
+ signature_type, params = self.get_signature_type_and_params(uri_query,
+ headers, body)
+
+ # the parameters may not include duplicate oauth entries
+ filtered_params = utils.filter_oauth_params(params)
+ if len(filtered_params) != len(params):
+ raise ValueError("Duplicate OAuth entries.")
+
+ params = dict(params)
+ request_signature = params.get(u'oauth_signature')
+ client_key = params.get(u'oauth_consumer_key')
+ resource_owner_key = params.get(u'oauth_token')
+ nonce = params.get(u'oauth_nonce')
+ timestamp = params.get(u'oauth_timestamp')
+ callback_uri = params.get(u'oauth_callback')
+ verifier = params.get(u'oauth_verifier')
+ signature_method = params.get(u'oauth_signature_method')
+
+ # ensure all mandatory parameters are present
+ if not all((request_signature, client_key, nonce,
+ timestamp, signature_method)):
+ raise ValueError("Missing OAuth parameters.")
+
+ # if version is supplied, it must be "1.0"
+ if u'oauth_version' in params and params[u'oauth_version'] != u'1.0':
+ raise ValueError("Invalid OAuth version.")
+
+ # signature method must be valid
+ if not signature_method in SIGNATURE_METHODS:
+ raise ValueError("Invalid signature method.")
+
+ # ensure client key is valid
+ if not self.check_client_key(client_key):
+ return False
+
+ # ensure resource owner key is valid and not expired
+ if not self.check_resource_owner_key(client_key, resource_owner_key):
+ return False
+
+ # ensure the nonce and timestamp haven't been used before
+ if not self.check_timestamp_and_nonce(timestamp, nonce):
+ return False
+
+ # FIXME: extract realm, then self.check_realm
+
+ # oauth_client parameters depend on client chosen signature method
+ # which may vary for each request, section 3.4
+ # HMAC-SHA1 and PLAINTEXT share parameters
+ if signature_method == SIGNATURE_RSA:
+ oauth_client = Client(client_key,
+ resource_owner_key=resource_owner_key,
+ callback_uri=callback_uri,
+ signature_method=signature_method,
+ signature_type=signature_type,
+ rsa_key=self.rsa_key, verifier=verifier)
+ else:
+ client_secret = self.get_client_secret(client_key)
+ resource_owner_secret = self.get_resource_owner_secret(
+ resource_owner_key)
+ oauth_client = Client(client_key,
+ client_secret=client_secret,
+ resource_owner_key=resource_owner_key,
+ resource_owner_secret=resource_owner_secret,
+ callback_uri=callback_uri,
+ signature_method=signature_method,
+ signature_type=signature_type,
+ verifier=verifier)
+
+ request = Request(uri, http_method, body, headers)
+ request.oauth_params = params
+
+ client_signature = oauth_client.get_oauth_signature(request)
+
+ # FIXME: use near constant time string compare to avoid timing attacks
+ return client_signature == request_signature
diff --git a/libs/oauthlib/oauth1/rfc5849/parameters.py b/libs/oauthlib/oauth1/rfc5849/parameters.py
new file mode 100644
index 00000000..dee23a43
--- /dev/null
+++ b/libs/oauthlib/oauth1/rfc5849/parameters.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+"""
+oauthlib.parameters
+~~~~~~~~~~~~~~~~~~~
+
+This module contains methods related to `section 3.5`_ of the OAuth 1.0a spec.
+
+.. _`section 3.5`: http://tools.ietf.org/html/rfc5849#section-3.5
+"""
+
+from urlparse import urlparse, urlunparse
+from . import utils
+from oauthlib.common import extract_params, urlencode
+
+
+# TODO: do we need filter_params now that oauth_params are handled by Request?
+# We can easily pass in just oauth protocol params.
+@utils.filter_params
+def prepare_headers(oauth_params, headers=None, realm=None):
+ """**Prepare the Authorization header.**
+ Per `section 3.5.1`_ of the spec.
+
+ Protocol parameters can be transmitted using the HTTP "Authorization"
+ header field as defined by `RFC2617`_ with the auth-scheme name set to
+ "OAuth" (case insensitive).
+
+ For example::
+
+ Authorization: OAuth realm="Example",
+ oauth_consumer_key="0685bd9184jfhq22",
+ oauth_token="ad180jjd733klru7",
+ oauth_signature_method="HMAC-SHA1",
+ oauth_signature="wOJIO9A2W5mFwDgiDvZbTSMK%2FPY%3D",
+ oauth_timestamp="137131200",
+ oauth_nonce="4572616e48616d6d65724c61686176",
+ oauth_version="1.0"
+
+
+ .. _`section 3.5.1`: http://tools.ietf.org/html/rfc5849#section-3.5.1
+ .. _`RFC2617`: http://tools.ietf.org/html/rfc2617
+ """
+ headers = headers or {}
+
+ # Protocol parameters SHALL be included in the "Authorization" header
+ # field as follows:
+ authorization_header_parameters_parts = []
+ for oauth_parameter_name, value in oauth_params:
+ # 1. Parameter names and values are encoded per Parameter Encoding
+ # (`Section 3.6`_)
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ escaped_name = utils.escape(oauth_parameter_name)
+ escaped_value = utils.escape(value)
+
+ # 2. Each parameter's name is immediately followed by an "=" character
+ # (ASCII code 61), a """ character (ASCII code 34), the parameter
+ # value (MAY be empty), and another """ character (ASCII code 34).
+ part = u'{0}="{1}"'.format(escaped_name, escaped_value)
+
+ authorization_header_parameters_parts.append(part)
+
+ # 3. Parameters are separated by a "," character (ASCII code 44) and
+ # OPTIONAL linear whitespace per `RFC2617`_.
+ #
+ # .. _`RFC2617`: http://tools.ietf.org/html/rfc2617
+ authorization_header_parameters = ', '.join(
+ authorization_header_parameters_parts)
+
+ # 4. The OPTIONAL "realm" parameter MAY be added and interpreted per
+ # `RFC2617 section 1.2`_.
+ #
+ # .. _`RFC2617 section 1.2`: http://tools.ietf.org/html/rfc2617#section-1.2
+ if realm:
+ # NOTE: realm should *not* be escaped
+ authorization_header_parameters = (u'realm="%s", ' % realm +
+ authorization_header_parameters)
+
+ # the auth-scheme name set to "OAuth" (case insensitive).
+ authorization_header = u'OAuth %s' % authorization_header_parameters
+
+ # contribute the Authorization header to the given headers
+ full_headers = {}
+ full_headers.update(headers)
+ full_headers[u'Authorization'] = authorization_header
+ return full_headers
+
+
+def _append_params(oauth_params, params):
+ """Append OAuth params to an existing set of parameters.
+
+ Both params and oauth_params is must be lists of 2-tuples.
+
+ Per `section 3.5.2`_ and `3.5.3`_ of the spec.
+
+ .. _`section 3.5.2`: http://tools.ietf.org/html/rfc5849#section-3.5.2
+ .. _`3.5.3`: http://tools.ietf.org/html/rfc5849#section-3.5.3
+
+ """
+ merged = list(params)
+ merged.extend(oauth_params)
+ # The request URI / entity-body MAY include other request-specific
+ # parameters, in which case, the protocol parameters SHOULD be appended
+ # following the request-specific parameters, properly separated by an "&"
+ # character (ASCII code 38)
+ merged.sort(key=lambda i: i[0].startswith('oauth_'))
+ return merged
+
+
+def prepare_form_encoded_body(oauth_params, body):
+ """Prepare the Form-Encoded Body.
+
+ Per `section 3.5.2`_ of the spec.
+
+ .. _`section 3.5.2`: http://tools.ietf.org/html/rfc5849#section-3.5.2
+
+ """
+ # append OAuth params to the existing body
+ return _append_params(oauth_params, body)
+
+
+def prepare_request_uri_query(oauth_params, uri):
+ """Prepare the Request URI Query.
+
+ Per `section 3.5.3`_ of the spec.
+
+ .. _`section 3.5.3`: http://tools.ietf.org/html/rfc5849#section-3.5.3
+
+ """
+ # append OAuth params to the existing set of query components
+ sch, net, path, par, query, fra = urlparse(uri)
+ query = urlencode(_append_params(oauth_params, extract_params(query) or []))
+ return urlunparse((sch, net, path, par, query, fra))
diff --git a/libs/oauthlib/oauth1/rfc5849/signature.py b/libs/oauthlib/oauth1/rfc5849/signature.py
new file mode 100644
index 00000000..99101d43
--- /dev/null
+++ b/libs/oauthlib/oauth1/rfc5849/signature.py
@@ -0,0 +1,501 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+"""
+oauthlib.oauth1.rfc5849.signature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module represents a direct implementation of `section 3.4`_ of the spec.
+
+Terminology:
+ * Client: software interfacing with an OAuth API
+ * Server: the API provider
+ * Resource Owner: the user who is granting authorization to the client
+
+Steps for signing a request:
+
+1. Collect parameters from the uri query, auth header, & body
+2. Normalize those parameters
+3. Normalize the uri
+4. Pass the normalized uri, normalized parameters, and http method to
+ construct the base string
+5. Pass the base string and any keys needed to a signing function
+
+.. _`section 3.4`: http://tools.ietf.org/html/rfc5849#section-3.4
+"""
+import binascii
+import hashlib
+import hmac
+import urlparse
+from . import utils
+from oauthlib.common import extract_params
+
+
+def construct_base_string(http_method, base_string_uri,
+ normalized_encoded_request_parameters):
+ """**String Construction**
+ Per `section 3.4.1.1`_ of the spec.
+
+ For example, the HTTP request::
+
+ POST /request?b5=%3D%253D&a3=a&c%40=&a2=r%20b HTTP/1.1
+ Host: example.com
+ Content-Type: application/x-www-form-urlencoded
+ Authorization: OAuth realm="Example",
+ oauth_consumer_key="9djdj82h48djs9d2",
+ oauth_token="kkk9d7dh3k39sjv7",
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp="137131201",
+ oauth_nonce="7d8f3e4a",
+ oauth_signature="bYT5CMsGcbgUdFHObYMEfcx6bsw%3D"
+
+ c2&a3=2+q
+
+ is represented by the following signature base string (line breaks
+ are for display purposes only)::
+
+ POST&http%3A%2F%2Fexample.com%2Frequest&a2%3Dr%2520b%26a3%3D2%2520q
+ %26a3%3Da%26b5%3D%253D%25253D%26c%2540%3D%26c2%3D%26oauth_consumer_
+ key%3D9djdj82h48djs9d2%26oauth_nonce%3D7d8f3e4a%26oauth_signature_m
+ ethod%3DHMAC-SHA1%26oauth_timestamp%3D137131201%26oauth_token%3Dkkk
+ 9d7dh3k39sjv7
+
+ .. _`section 3.4.1.1`: http://tools.ietf.org/html/rfc5849#section-3.4.1.1
+ """
+
+ # The signature base string is constructed by concatenating together,
+ # in order, the following HTTP request elements:
+
+ # 1. The HTTP request method in uppercase. For example: "HEAD",
+ # "GET", "POST", etc. If the request uses a custom HTTP method, it
+ # MUST be encoded (`Section 3.6`_).
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ base_string = utils.escape(http_method.upper())
+
+ # 2. An "&" character (ASCII code 38).
+ base_string += u'&'
+
+ # 3. The base string URI from `Section 3.4.1.2`_, after being encoded
+ # (`Section 3.6`_).
+ #
+ # .. _`Section 3.4.1.2`: http://tools.ietf.org/html/rfc5849#section-3.4.1.2
+ # .. _`Section 3.4.6`: http://tools.ietf.org/html/rfc5849#section-3.4.6
+ base_string += utils.escape(base_string_uri)
+
+ # 4. An "&" character (ASCII code 38).
+ base_string += u'&'
+
+ # 5. The request parameters as normalized in `Section 3.4.1.3.2`_, after
+ # being encoded (`Section 3.6`).
+ #
+ # .. _`Section 3.4.1.3.2`: http://tools.ietf.org/html/rfc5849#section-3.4.1.3.2
+ # .. _`Section 3.4.6`: http://tools.ietf.org/html/rfc5849#section-3.4.6
+ base_string += utils.escape(normalized_encoded_request_parameters)
+
+ return base_string
+
+
+def normalize_base_string_uri(uri):
+ """**Base String URI**
+ Per `section 3.4.1.2`_ of the spec.
+
+ For example, the HTTP request::
+
+ GET /r%20v/X?id=123 HTTP/1.1
+ Host: EXAMPLE.COM:80
+
+ is represented by the base string URI: "http://example.com/r%20v/X".
+
+ In another example, the HTTPS request::
+
+ GET /?q=1 HTTP/1.1
+ Host: www.example.net:8080
+
+ is represented by the base string URI: "https://www.example.net:8080/".
+
+ .. _`section 3.4.1.2`: http://tools.ietf.org/html/rfc5849#section-3.4.1.2
+ """
+ if not isinstance(uri, unicode):
+ raise ValueError('uri must be a unicode object.')
+
+ # FIXME: urlparse does not support unicode
+ scheme, netloc, path, params, query, fragment = urlparse.urlparse(uri)
+
+ # The scheme, authority, and path of the request resource URI `RFC3986`
+ # are included by constructing an "http" or "https" URI representing
+ # the request resource (without the query or fragment) as follows:
+ #
+ # .. _`RFC2616`: http://tools.ietf.org/html/rfc3986
+
+ # 1. The scheme and host MUST be in lowercase.
+ scheme = scheme.lower()
+ netloc = netloc.lower()
+
+ # 2. The host and port values MUST match the content of the HTTP
+ # request "Host" header field.
+ # TODO: enforce this constraint
+
+ # 3. The port MUST be included if it is not the default port for the
+ # scheme, and MUST be excluded if it is the default. Specifically,
+ # the port MUST be excluded when making an HTTP request `RFC2616`_
+ # to port 80 or when making an HTTPS request `RFC2818`_ to port 443.
+ # All other non-default port numbers MUST be included.
+ #
+ # .. _`RFC2616`: http://tools.ietf.org/html/rfc2616
+ # .. _`RFC2818`: http://tools.ietf.org/html/rfc2818
+ default_ports = (
+ (u'http', u'80'),
+ (u'https', u'443'),
+ )
+ if u':' in netloc:
+ host, port = netloc.split(u':', 1)
+ if (scheme, port) in default_ports:
+ netloc = host
+
+ return urlparse.urlunparse((scheme, netloc, path, u'', u'', u''))
+
+
+# ** Request Parameters **
+#
+# Per `section 3.4.1.3`_ of the spec.
+#
+# In order to guarantee a consistent and reproducible representation of
+# the request parameters, the parameters are collected and decoded to
+# their original decoded form. They are then sorted and encoded in a
+# particular manner that is often different from their original
+# encoding scheme, and concatenated into a single string.
+#
+# .. _`section 3.4.1.3`: http://tools.ietf.org/html/rfc5849#section-3.4.1.3
+
+def collect_parameters(uri_query='', body=[], headers=None,
+ exclude_oauth_signature=True):
+ """**Parameter Sources**
+
+ Parameters starting with `oauth_` will be unescaped.
+
+ Body parameters must be supplied as a dict, a list of 2-tuples, or a
+ formencoded query string.
+
+ Headers must be supplied as a dict.
+
+ Per `section 3.4.1.3.1`_ of the spec.
+
+ For example, the HTTP request::
+
+ POST /request?b5=%3D%253D&a3=a&c%40=&a2=r%20b HTTP/1.1
+ Host: example.com
+ Content-Type: application/x-www-form-urlencoded
+ Authorization: OAuth realm="Example",
+ oauth_consumer_key="9djdj82h48djs9d2",
+ oauth_token="kkk9d7dh3k39sjv7",
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp="137131201",
+ oauth_nonce="7d8f3e4a",
+ oauth_signature="djosJKDKJSD8743243%2Fjdk33klY%3D"
+
+ c2&a3=2+q
+
+ contains the following (fully decoded) parameters used in the
+ signature base sting::
+
+ +------------------------+------------------+
+ | Name | Value |
+ +------------------------+------------------+
+ | b5 | =%3D |
+ | a3 | a |
+ | c@ | |
+ | a2 | r b |
+ | oauth_consumer_key | 9djdj82h48djs9d2 |
+ | oauth_token | kkk9d7dh3k39sjv7 |
+ | oauth_signature_method | HMAC-SHA1 |
+ | oauth_timestamp | 137131201 |
+ | oauth_nonce | 7d8f3e4a |
+ | c2 | |
+ | a3 | 2 q |
+ +------------------------+------------------+
+
+ Note that the value of "b5" is "=%3D" and not "==". Both "c@" and
+ "c2" have empty values. While the encoding rules specified in this
+ specification for the purpose of constructing the signature base
+ string exclude the use of a "+" character (ASCII code 43) to
+ represent an encoded space character (ASCII code 32), this practice
+ is widely used in "application/x-www-form-urlencoded" encoded values,
+ and MUST be properly decoded, as demonstrated by one of the "a3"
+ parameter instances (the "a3" parameter is used twice in this
+ request).
+
+ .. _`section 3.4.1.3.1`: http://tools.ietf.org/html/rfc5849#section-3.4.1.3.1
+ """
+ headers = headers or {}
+ params = []
+
+ # The parameters from the following sources are collected into a single
+ # list of name/value pairs:
+
+ # * The query component of the HTTP request URI as defined by
+ # `RFC3986, Section 3.4`_. The query component is parsed into a list
+ # of name/value pairs by treating it as an
+ # "application/x-www-form-urlencoded" string, separating the names
+ # and values and decoding them as defined by
+ # `W3C.REC-html40-19980424`_, Section 17.13.4.
+ #
+ # .. _`RFC3986, Section 3.4`: http://tools.ietf.org/html/rfc3986#section-3.4
+ # .. _`W3C.REC-html40-19980424`: http://tools.ietf.org/html/rfc5849#ref-W3C.REC-html40-19980424
+ if uri_query:
+ params.extend(urlparse.parse_qsl(uri_query, keep_blank_values=True))
+
+ # * The OAuth HTTP "Authorization" header field (`Section 3.5.1`_) if
+ # present. The header's content is parsed into a list of name/value
+ # pairs excluding the "realm" parameter if present. The parameter
+ # values are decoded as defined by `Section 3.5.1`_.
+ #
+ # .. _`Section 3.5.1`: http://tools.ietf.org/html/rfc5849#section-3.5.1
+ if headers:
+ headers_lower = dict((k.lower(), v) for k, v in headers.items())
+ authorization_header = headers_lower.get(u'authorization')
+ if authorization_header is not None:
+ params.extend([i for i in utils.parse_authorization_header(
+ authorization_header) if i[0] != u'realm'])
+
+ # * The HTTP request entity-body, but only if all of the following
+ # conditions are met:
+ # * The entity-body is single-part.
+ #
+ # * The entity-body follows the encoding requirements of the
+ # "application/x-www-form-urlencoded" content-type as defined by
+ # `W3C.REC-html40-19980424`_.
+
+ # * The HTTP request entity-header includes the "Content-Type"
+ # header field set to "application/x-www-form-urlencoded".
+ #
+ # .._`W3C.REC-html40-19980424`: http://tools.ietf.org/html/rfc5849#ref-W3C.REC-html40-19980424
+
+ # TODO: enforce header param inclusion conditions
+ bodyparams = extract_params(body) or []
+ params.extend(bodyparams)
+
+ # ensure all oauth params are unescaped
+ unescaped_params = []
+ for k, v in params:
+ if k.startswith(u'oauth_'):
+ v = utils.unescape(v)
+ unescaped_params.append((k, v))
+
+ # The "oauth_signature" parameter MUST be excluded from the signature
+ # base string if present.
+ if exclude_oauth_signature:
+ unescaped_params = filter(lambda i: i[0] != u'oauth_signature',
+ unescaped_params)
+
+ return unescaped_params
+
+
+def normalize_parameters(params):
+ """**Parameters Normalization**
+ Per `section 3.4.1.3.2`_ of the spec.
+
+ For example, the list of parameters from the previous section would
+ be normalized as follows:
+
+ Encoded::
+
+ +------------------------+------------------+
+ | Name | Value |
+ +------------------------+------------------+
+ | b5 | %3D%253D |
+ | a3 | a |
+ | c%40 | |
+ | a2 | r%20b |
+ | oauth_consumer_key | 9djdj82h48djs9d2 |
+ | oauth_token | kkk9d7dh3k39sjv7 |
+ | oauth_signature_method | HMAC-SHA1 |
+ | oauth_timestamp | 137131201 |
+ | oauth_nonce | 7d8f3e4a |
+ | c2 | |
+ | a3 | 2%20q |
+ +------------------------+------------------+
+
+ Sorted::
+
+ +------------------------+------------------+
+ | Name | Value |
+ +------------------------+------------------+
+ | a2 | r%20b |
+ | a3 | 2%20q |
+ | a3 | a |
+ | b5 | %3D%253D |
+ | c%40 | |
+ | c2 | |
+ | oauth_consumer_key | 9djdj82h48djs9d2 |
+ | oauth_nonce | 7d8f3e4a |
+ | oauth_signature_method | HMAC-SHA1 |
+ | oauth_timestamp | 137131201 |
+ | oauth_token | kkk9d7dh3k39sjv7 |
+ +------------------------+------------------+
+
+ Concatenated Pairs::
+
+ +-------------------------------------+
+ | Name=Value |
+ +-------------------------------------+
+ | a2=r%20b |
+ | a3=2%20q |
+ | a3=a |
+ | b5=%3D%253D |
+ | c%40= |
+ | c2= |
+ | oauth_consumer_key=9djdj82h48djs9d2 |
+ | oauth_nonce=7d8f3e4a |
+ | oauth_signature_method=HMAC-SHA1 |
+ | oauth_timestamp=137131201 |
+ | oauth_token=kkk9d7dh3k39sjv7 |
+ +-------------------------------------+
+
+ and concatenated together into a single string (line breaks are for
+ display purposes only)::
+
+ a2=r%20b&a3=2%20q&a3=a&b5=%3D%253D&c%40=&c2=&oauth_consumer_key=9dj
+ dj82h48djs9d2&oauth_nonce=7d8f3e4a&oauth_signature_method=HMAC-SHA1
+ &oauth_timestamp=137131201&oauth_token=kkk9d7dh3k39sjv7
+
+ .. _`section 3.4.1.3.2`: http://tools.ietf.org/html/rfc5849#section-3.4.1.3.2
+ """
+
+ # The parameters collected in `Section 3.4.1.3`_ are normalized into a
+ # single string as follows:
+ #
+ # .. _`Section 3.4.1.3`: http://tools.ietf.org/html/rfc5849#section-3.4.1.3
+
+ # 1. First, the name and value of each parameter are encoded
+ # (`Section 3.6`_).
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ key_values = [(utils.escape(k), utils.escape(v)) for k, v in params]
+
+ # 2. The parameters are sorted by name, using ascending byte value
+ # ordering. If two or more parameters share the same name, they
+ # are sorted by their value.
+ key_values.sort()
+
+ # 3. The name of each parameter is concatenated to its corresponding
+ # value using an "=" character (ASCII code 61) as a separator, even
+ # if the value is empty.
+ parameter_parts = [u'{0}={1}'.format(k, v) for k, v in key_values]
+
+ # 4. The sorted name/value pairs are concatenated together into a
+ # single string by using an "&" character (ASCII code 38) as
+ # separator.
+ return u'&'.join(parameter_parts)
+
+
+def sign_hmac_sha1(base_string, client_secret, resource_owner_secret):
+ """**HMAC-SHA1**
+
+ The "HMAC-SHA1" signature method uses the HMAC-SHA1 signature
+ algorithm as defined in `RFC2104`_::
+
+ digest = HMAC-SHA1 (key, text)
+
+ Per `section 3.4.2`_ of the spec.
+
+ .. _`RFC2104`: http://tools.ietf.org/html/rfc2104
+ .. _`section 3.4.2`: http://tools.ietf.org/html/rfc5849#section-3.4.2
+ """
+
+ # The HMAC-SHA1 function variables are used in following way:
+
+ # text is set to the value of the signature base string from
+ # `Section 3.4.1.1`_.
+ #
+ # .. _`Section 3.4.1.1`: http://tools.ietf.org/html/rfc5849#section-3.4.1.1
+ text = base_string
+
+ # key is set to the concatenated values of:
+ # 1. The client shared-secret, after being encoded (`Section 3.6`_).
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ key = utils.escape(client_secret or u'')
+
+ # 2. An "&" character (ASCII code 38), which MUST be included
+ # even when either secret is empty.
+ key += u'&'
+
+ # 3. The token shared-secret, after being encoded (`Section 3.6`_).
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ key += utils.escape(resource_owner_secret or u'')
+
+ # FIXME: HMAC does not support unicode!
+ key_utf8 = key.encode('utf-8')
+ text_utf8 = text.encode('utf-8')
+ signature = hmac.new(key_utf8, text_utf8, hashlib.sha1)
+
+ # digest is used to set the value of the "oauth_signature" protocol
+ # parameter, after the result octet string is base64-encoded
+ # per `RFC2045, Section 6.8`.
+ #
+ # .. _`RFC2045, Section 6.8`: http://tools.ietf.org/html/rfc2045#section-6.8
+ return binascii.b2a_base64(signature.digest())[:-1].decode('utf-8')
+
+
+def sign_rsa_sha1(base_string, rsa_private_key):
+ """**RSA-SHA1**
+
+ Per `section 3.4.3`_ of the spec.
+
+ The "RSA-SHA1" signature method uses the RSASSA-PKCS1-v1_5 signature
+ algorithm as defined in `RFC3447, Section 8.2`_ (also known as
+ PKCS#1), using SHA-1 as the hash function for EMSA-PKCS1-v1_5. To
+ use this method, the client MUST have established client credentials
+ with the server that included its RSA public key (in a manner that is
+ beyond the scope of this specification).
+
+ NOTE: this method requires the python-rsa library.
+
+ .. _`section 3.4.3`: http://tools.ietf.org/html/rfc5849#section-3.4.3
+ .. _`RFC3447, Section 8.2`: http://tools.ietf.org/html/rfc3447#section-8.2
+
+ """
+
+ # TODO: finish RSA documentation
+
+ import rsa
+ key = rsa.PrivateKey.load_pkcs1(rsa_private_key)
+ sig = rsa.sign(base_string, key, 'SHA-1')
+ return binascii.b2a_base64(sig)[:-1]
+
+
+def sign_plaintext(client_secret, resource_owner_secret):
+ """Sign a request using plaintext.
+
+ Per `section 3.4.4`_ of the spec.
+
+ The "PLAINTEXT" method does not employ a signature algorithm. It
+ MUST be used with a transport-layer mechanism such as TLS or SSL (or
+ sent over a secure channel with equivalent protections). It does not
+ utilize the signature base string or the "oauth_timestamp" and
+ "oauth_nonce" parameters.
+
+ .. _`section 3.4.4`: http://tools.ietf.org/html/rfc5849#section-3.4.4
+
+ """
+
+ # The "oauth_signature" protocol parameter is set to the concatenated
+ # value of:
+
+ # 1. The client shared-secret, after being encoded (`Section 3.6`_).
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ signature = utils.escape(client_secret or u'')
+
+ # 2. An "&" character (ASCII code 38), which MUST be included even
+ # when either secret is empty.
+ signature += u'&'
+
+ # 3. The token shared-secret, after being encoded (`Section 3.6`_).
+ #
+ # .. _`Section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+ signature += utils.escape(resource_owner_secret or u'')
+
+ return signature
+
diff --git a/libs/oauthlib/oauth1/rfc5849/utils.py b/libs/oauthlib/oauth1/rfc5849/utils.py
new file mode 100644
index 00000000..6db446fd
--- /dev/null
+++ b/libs/oauthlib/oauth1/rfc5849/utils.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+"""
+oauthlib.utils
+~~~~~~~~~~~~~~
+
+This module contains utility methods used by various parts of the OAuth
+spec.
+"""
+
+import string
+import time
+import urllib2
+from random import getrandbits, choice
+
+from oauthlib.common import quote, unquote
+
+UNICODE_ASCII_CHARACTER_SET = (string.ascii_letters.decode('ascii') +
+ string.digits.decode('ascii'))
+
+
+def filter_params(target):
+ """Decorator which filters params to remove non-oauth_* parameters
+
+ Assumes the decorated method takes a params dict or list of tuples as its
+ first argument.
+ """
+ def wrapper(params, *args, **kwargs):
+ params = filter_oauth_params(params)
+ return target(params, *args, **kwargs)
+
+ wrapper.__doc__ = target.__doc__
+ return wrapper
+
+
+def filter_oauth_params(params):
+ """Removes all non oauth parameters from a dict or a list of params."""
+ is_oauth = lambda kv: kv[0].startswith(u"oauth_")
+ if isinstance(params, dict):
+ return filter(is_oauth, params.items())
+ else:
+ return filter(is_oauth, params)
+
+
+def generate_timestamp():
+ """Get seconds since epoch (UTC).
+
+ Per `section 3.3`_ of the spec.
+
+ .. _`section 3.3`: http://tools.ietf.org/html/rfc5849#section-3.3
+ """
+ return unicode(int(time.time()))
+
+
+def generate_nonce():
+ """Generate pseudorandom nonce that is unlikely to repeat.
+
+ Per `section 3.3`_ of the spec.
+
+ A random 64-bit number is appended to the epoch timestamp for both
+ randomness and to decrease the likelihood of collisions.
+
+ .. _`section 3.3`: http://tools.ietf.org/html/rfc5849#section-3.3
+ """
+ return unicode(getrandbits(64)) + generate_timestamp()
+
+
+def generate_token(length=20, chars=UNICODE_ASCII_CHARACTER_SET):
+ """Generates a generic OAuth token
+
+ According to `section 2`_ of the spec, the method of token
+ construction is undefined. This implementation is simply a random selection
+ of `length` choices from `chars`.
+
+ Credit to Ignacio Vazquez-Abrams for his excellent `Stackoverflow answer`_
+
+ .. _`Stackoverflow answer` : http://stackoverflow.com/questions/2257441/
+ python-random-string-generation-with-upper-case-letters-and-digits
+
+ """
+ return u''.join(choice(chars) for x in range(length))
+
+
+def escape(u):
+ """Escape a unicode string in an OAuth-compatible fashion.
+
+ Per `section 3.6`_ of the spec.
+
+ .. _`section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+
+ """
+ if not isinstance(u, unicode):
+ raise ValueError('Only unicode objects are escapable.')
+ # Letters, digits, and the characters '_.-' are already treated as safe
+ # by urllib.quote(). We need to add '~' to fully support rfc5849.
+ return quote(u, safe='~')
+
+
+def unescape(u):
+ if not isinstance(u, unicode):
+ raise ValueError('Only unicode objects are unescapable.')
+ return unquote(u)
+
+
+def urlencode(query):
+ """Encode a sequence of two-element tuples or dictionary into a URL query string.
+
+ Operates using an OAuth-safe escape() method, in contrast to urllib.urlencode.
+ """
+ # Convert dictionaries to list of tuples
+ if isinstance(query, dict):
+ query = query.items()
+ return u"&".join([u'='.join([escape(k), escape(v)]) for k, v in query])
+
+
+def parse_keqv_list(l):
+ """A unicode-safe version of urllib2.parse_keqv_list"""
+ encoded_list = [u.encode('utf-8') for u in l]
+ encoded_parsed = urllib2.parse_keqv_list(encoded_list)
+ return dict((k.decode('utf-8'),
+ v.decode('utf-8')) for k,v in encoded_parsed.items())
+
+
+def parse_http_list(u):
+ """A unicode-safe version of urllib2.parse_http_list"""
+ encoded_str = u.encode('utf-8')
+ encoded_list = urllib2.parse_http_list(encoded_str)
+ return [s.decode('utf-8') for s in encoded_list]
+
+
+def parse_authorization_header(authorization_header):
+ """Parse an OAuth authorization header into a list of 2-tuples"""
+ auth_scheme = u'OAuth '
+ if authorization_header.startswith(auth_scheme):
+ authorization_header = authorization_header.replace(auth_scheme, u'', 1)
+ items = parse_http_list(authorization_header)
+ try:
+ return parse_keqv_list(items).items()
+ except ValueError:
+ raise ValueError('Malformed authorization header')
+
diff --git a/libs/oauthlib/oauth2/__init__.py b/libs/oauthlib/oauth2/__init__.py
new file mode 100644
index 00000000..0e8933cf
--- /dev/null
+++ b/libs/oauthlib/oauth2/__init__.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+"""
+oauthlib.oauth2
+~~~~~~~~~~~~~~
+
+This module is a wrapper for the most recent implementation of OAuth 2.0 Client
+and Server classes.
+"""
+
+from .draft25 import Client, Server
+
diff --git a/libs/oauthlib/oauth2/draft25/__init__.py b/libs/oauthlib/oauth2/draft25/__init__.py
new file mode 100644
index 00000000..3e50a18f
--- /dev/null
+++ b/libs/oauthlib/oauth2/draft25/__init__.py
@@ -0,0 +1,14 @@
+"""
+oauthlib.oauth2.draft_25
+~~~~~~~~~~~~~~
+
+This module is an implementation of various logic needed
+for signing and checking OAuth 2.0 draft 25 requests.
+"""
+
+class Client(object):
+ pass
+
+class Server(object):
+ pass
+
diff --git a/libs/oauthlib/oauth2/draft25/tokens.py b/libs/oauthlib/oauth2/draft25/tokens.py
new file mode 100644
index 00000000..9b5f5868
--- /dev/null
+++ b/libs/oauthlib/oauth2/draft25/tokens.py
@@ -0,0 +1,131 @@
+from __future__ import absolute_import
+"""
+oauthlib.oauth2.draft25.tokens
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module contains methods for adding two types of access tokens to requests.
+
+- Bearer http://tools.ietf.org/html/draft-ietf-oauth-saml2-bearer-08
+- MAC http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-00
+
+"""
+from binascii import b2a_base64
+import hashlib
+import hmac
+from urlparse import urlparse
+
+from . import utils
+
+
+def prepare_mac_header(token, uri, key, http_method, nonce=None, headers=None,
+ body=None, ext=u'', hash_algorithm=u'hmac-sha-1'):
+ """Add an `MAC Access Authentication`_ signature to headers.
+
+ Unlike OAuth 1, this HMAC signature does not require inclusion of the request
+ payload/body, neither does it use a combination of client_secret and
+ token_secret but rather a mac_key provided together with the access token.
+
+ Currently two algorithms are supported, "hmac-sha-1" and "hmac-sha-256",
+ `extension algorithms`_ are not supported.
+
+ Example MAC Authorization header, linebreaks added for clarity
+
+ Authorization: MAC id="h480djs93hd8",
+ nonce="1336363200:dj83hs9s",
+ mac="bhCQXTVyfj5cmA9uKkPFx1zeOXM="
+
+ .. _`MAC Access Authentication`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01
+ .. _`extension algorithms`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-7.1
+
+ :param uri: Request URI.
+ :param headers: Request headers as a dictionary.
+ :param http_method: HTTP Request method.
+ :param key: MAC given provided by token endpoint.
+ :param algorithm: HMAC algorithm provided by token endpoint.
+ :return: headers dictionary with the authorization field added.
+ """
+ http_method = http_method.upper()
+ host, port = utils.host_from_uri(uri)
+
+ if hash_algorithm.lower() == u'hmac-sha-1':
+ h = hashlib.sha1
+ else:
+ h = hashlib.sha256
+
+ nonce = nonce or u'{0}:{1}'.format(utils.generate_nonce(), utils.generate_timestamp())
+ sch, net, path, par, query, fra = urlparse(uri)
+
+ if query:
+ request_uri = path + u'?' + query
+ else:
+ request_uri = path
+
+ # Hash the body/payload
+ if body is not None:
+ bodyhash = b2a_base64(h(body).digest())[:-1].decode('utf-8')
+ else:
+ bodyhash = u''
+
+ # Create the normalized base string
+ base = []
+ base.append(nonce)
+ base.append(http_method.upper())
+ base.append(request_uri)
+ base.append(host)
+ base.append(port)
+ base.append(bodyhash)
+ base.append(ext)
+ base_string = '\n'.join(base) + u'\n'
+
+ # hmac struggles with unicode strings - http://bugs.python.org/issue5285
+ if isinstance(key, unicode):
+ key = key.encode('utf-8')
+ sign = hmac.new(key, base_string, h)
+ sign = b2a_base64(sign.digest())[:-1].decode('utf-8')
+
+ header = []
+ header.append(u'MAC id="%s"' % token)
+ header.append(u'nonce="%s"' % nonce)
+ if bodyhash:
+ header.append(u'bodyhash="%s"' % bodyhash)
+ if ext:
+ header.append(u'ext="%s"' % ext)
+ header.append(u'mac="%s"' % sign)
+
+ headers = headers or {}
+ headers[u'Authorization'] = u', '.join(header)
+ return headers
+
+
+def prepare_bearer_uri(token, uri):
+ """Add a `Bearer Token`_ to the request URI.
+ Not recommended, use only if client can't use authorization header or body.
+
+ http://www.example.com/path?access_token=h480djs93hd8
+
+ .. _`Bearer Token`: http://tools.ietf.org/html/draft-ietf-oauth-v2-bearer-18
+ """
+ return utils.add_params_to_uri(uri, [((u'access_token', token))])
+
+
+def prepare_bearer_headers(token, headers=None):
+ """Add a `Bearer Token`_ to the request URI.
+ Recommended method of passing bearer tokens.
+
+ Authorization: Bearer h480djs93hd8
+
+ .. _`Bearer Token`: http://tools.ietf.org/html/draft-ietf-oauth-v2-bearer-18
+ """
+ headers = headers or {}
+ headers[u'Authorization'] = u'Bearer %s' % token
+ return headers
+
+
+def prepare_bearer_body(token, body=u''):
+ """Add a `Bearer Token`_ to the request body.
+
+ access_token=h480djs93hd8
+
+ .. _`Bearer Token`: http://tools.ietf.org/html/draft-ietf-oauth-v2-bearer-18
+ """
+ return utils.add_params_to_qs(body, [((u'access_token', token))])
diff --git a/libs/oauthlib/oauth2/draft25/utils.py b/libs/oauthlib/oauth2/draft25/utils.py
new file mode 100644
index 00000000..48b4ea1d
--- /dev/null
+++ b/libs/oauthlib/oauth2/draft25/utils.py
@@ -0,0 +1,128 @@
+"""
+oauthlib.utils
+~~~~~~~~~~~~~~
+
+This module contains utility methods used by various parts of the OAuth 2 spec.
+"""
+
+import random
+import string
+import time
+import urllib
+from urlparse import urlparse, urlunparse, parse_qsl
+
+UNICODE_ASCII_CHARACTER_SET = (string.ascii_letters.decode('ascii') +
+ string.digits.decode('ascii'))
+
+def add_params_to_qs(query, params):
+ """Extend a query with a list of two-tuples.
+
+ :param query: Query string.
+ :param params: List of two-tuples.
+ :return: extended query
+ """
+ queryparams = parse_qsl(query, keep_blank_values=True)
+ queryparams.extend(params)
+ return urlencode(queryparams)
+
+
+def add_params_to_uri(uri, params):
+ """Add a list of two-tuples to the uri query components.
+
+ :param uri: Full URI.
+ :param params: List of two-tuples.
+ :return: uri with extended query
+ """
+ sch, net, path, par, query, fra = urlparse(uri)
+ query = add_params_to_qs(query, params)
+ return urlunparse((sch, net, path, par, query, fra))
+
+
+def escape(u):
+ """Escape a string in an OAuth-compatible fashion.
+
+ Per `section 3.6`_ of the spec.
+
+ .. _`section 3.6`: http://tools.ietf.org/html/rfc5849#section-3.6
+
+ """
+ if not isinstance(u, unicode):
+ raise ValueError('Only unicode objects are escapable.')
+ return urllib.quote(u.encode('utf-8'), safe='~')
+
+
+def generate_nonce():
+ """Generate pseudorandom nonce that is unlikely to repeat.
+
+ Per `section 3.2.1`_ of the MAC Access Authentication spec.
+
+ A random 64-bit number is appended to the epoch timestamp for both
+ randomness and to decrease the likelihood of collisions.
+
+ .. _`section 3.2.1`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-3.2.1
+ """
+ return unicode(unicode(random.getrandbits(64)) + generate_timestamp())
+
+
+def generate_timestamp():
+ """Get seconds since epoch (UTC).
+
+ Per `section 3.2.1`_ of the MAC Access Authentication spec.
+
+ .. _`section 3.2.1`: http://tools.ietf.org/html/draft-ietf-oauth-v2-http-mac-01#section-3.2.1
+ """
+ return unicode(int(time.time()))
+
+
+def generate_token(length=20, chars=UNICODE_ASCII_CHARACTER_SET):
+ """Generates a generic OAuth 2 token
+
+ According to `section 1.4`_ and `section 1.5` of the spec, the method of token
+ construction is undefined. This implementation is simply a random selection
+ of `length` choices from `chars`. SystemRandom is used since it provides
+ higher entropy than random.choice.
+
+ .. _`section 1.4`: http://tools.ietf.org/html/draft-ietf-oauth-v2-25#section-1.4
+ .. _`section 1.5`: http://tools.ietf.org/html/draft-ietf-oauth-v2-25#section-1.5
+ """
+ rand = random.SystemRandom()
+ return u''.join(rand.choice(chars) for x in range(length))
+
+
+def host_from_uri(uri):
+ """Extract hostname and port from URI.
+
+ Will use default port for HTTP and HTTPS if none is present in the URI.
+
+ >>> host_from_uri(u'https://www.example.com/path?query')
+ u'www.example.com', u'443'
+ >>> host_from_uri(u'http://www.example.com:8080/path?query')
+ u'www.example.com', u'8080'
+
+ :param uri: Full URI.
+ :param http_method: HTTP request method.
+ :return: hostname, port
+ """
+ default_ports = {
+ u'HTTP' : u'80',
+ u'HTTPS' : u'443',
+ }
+
+ sch, netloc, path, par, query, fra = urlparse(uri)
+ if u':' in netloc:
+ netloc, port = netloc.split(u':', 1)
+ else:
+ port = default_ports.get(sch.upper())
+
+ return netloc, port
+
+
+def urlencode(query):
+ """Encode a sequence of two-element tuples or dictionary into a URL query string.
+
+ Operates using an OAuth-safe escape() method, in contrast to urllib.urlenocde.
+ """
+ # Convert dictionaries to list of tuples
+ if isinstance(query, dict):
+ query = query.items()
+ return "&".join(['='.join([escape(k), escape(v)]) for k, v in query])
diff --git a/libs/subliminal/api.py b/libs/subliminal/api.py
index baff5f1a..a7baeab6 100755
--- a/libs/subliminal/api.py
+++ b/libs/subliminal/api.py
@@ -18,7 +18,8 @@
from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
group_by_video, key_subtitles)
-from .languages import list_languages
+import guessit
+from guessit.language import ALL_LANGUAGES
import logging
@@ -26,7 +27,7 @@ __all__ = ['list_subtitles', 'download_subtitles']
logger = logging.getLogger(__name__)
-def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
+def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
"""List subtitles in given paths according to the criteria
:param paths: path(s) to video file or folder
@@ -37,19 +38,20 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
:param bool multi: search multiple languages for the same video
:param string cache_dir: path to the cache directory to use
:param int max_depth: maximum depth for scanning entries
+ :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
:return: found subtitles
:rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
"""
services = services or SERVICES
- languages = set(languages or list_languages(1))
+ languages = set(map(guessit.Language, languages or []) or ALL_LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
logger.warning(u'Not all entries are unicode')
results = []
service_instances = {}
- tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
+ tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
for task in tasks:
try:
result = consume_task(task, service_instances)
@@ -61,7 +63,7 @@ def list_subtitles(paths, languages=None, services=None, force=True, multi=False
return group_by_video(results)
-def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, order=None):
+def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
"""Download subtitles in given paths according to the criteria
:param paths: path(s) to video file or folder
@@ -72,6 +74,7 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
:param bool multi: search multiple languages for the same video
:param string cache_dir: path to the cache directory to use
:param int max_depth: maximum depth for scanning entries
+ :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
:param order: preferred order for subtitles sorting
:type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
:return: found subtitles
@@ -79,11 +82,11 @@ def download_subtitles(paths, languages=None, services=None, force=True, multi=F
"""
services = services or SERVICES
- languages = languages or list_languages(1)
+ languages = map(guessit.Language, languages or []) or list(ALL_LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
- subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
+ subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth, scan_filter)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
results = []
diff --git a/libs/subliminal/async.py b/libs/subliminal/async.py
index ce18a278..e125bbf9 100755
--- a/libs/subliminal/async.py
+++ b/libs/subliminal/async.py
@@ -18,7 +18,7 @@
from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
create_download_tasks, group_by_video, key_subtitles)
-from .languages import list_languages
+from guessit.language import ALL_LANGUAGES
from .tasks import StopTask
import Queue
import logging
@@ -108,29 +108,29 @@ class Pool(object):
break
return results
- def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
+ def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None):
"""See :meth:`subliminal.list_subtitles`"""
services = services or SERVICES
- languages = set(languages or list_languages(1))
+ languages = set(languages or ALL_LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
if any([not isinstance(p, unicode) for p in paths]):
logger.warning(u'Not all entries are unicode')
- tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
+ tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter)
for task in tasks:
self.tasks.put(task)
self.join()
results = self.collect()
return group_by_video(results)
- def download_subtitles(self, paths, languages=None, services=None, cache_dir=None, max_depth=3, force=True, multi=False, order=None):
+ def download_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, scan_filter=None, order=None):
"""See :meth:`subliminal.download_subtitles`"""
services = services or SERVICES
- languages = languages or list_languages(1)
+ languages = languages or list(ALL_LANGUAGES)
if isinstance(paths, basestring):
paths = [paths]
order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
- subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
+ subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth, scan_filter)
for video, subtitles in subtitles_by_video.iteritems():
subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
tasks = create_download_tasks(subtitles_by_video, multi)
diff --git a/libs/subliminal/cache.py b/libs/subliminal/cache.py
new file mode 100755
index 00000000..f4d7d947
--- /dev/null
+++ b/libs/subliminal/cache.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Nicolas Wack
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+import os.path
+from collections import defaultdict
+import threading
+from functools import wraps
+import logging
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+
+logger = logging.getLogger(__name__)
+
+
+class Cache(object):
+ """A Cache object contains cached values for methods. It can have
+ separate internal caches, one for each service.
+ """
+
+ def __init__(self, cache_dir):
+ self.cache_dir = cache_dir
+ self.cache = defaultdict(dict)
+ self.lock = threading.RLock()
+
+ def __del__(self):
+ for service_name in self.cache:
+ self.save(service_name)
+
+ def cache_location(self, service_name):
+ return os.path.join(self.cache_dir, 'subliminal_%s.cache' % service_name)
+
+ def load(self, service_name):
+ with self.lock:
+ if service_name in self.cache:
+ # already loaded
+ return
+
+ self.cache[service_name] = defaultdict(dict)
+ filename = self.cache_location(service_name)
+ logger.debug(u'Cache: loading cache from %s' % filename)
+ try:
+ self.cache[service_name] = pickle.load(open(filename, 'rb'))
+ except IOError:
+ logger.info('Cache: Cache file "%s" doesn\'t exist, creating it' % filename)
+ except EOFError:
+ logger.error('Cache: cache file "%s" is corrupted... Removing it.' % filename)
+ os.remove(filename)
+
+ def save(self, service_name):
+ filename = self.cache_location(service_name)
+ logger.debug(u'Cache: saving cache to %s' % filename)
+ with self.lock:
+ pickle.dump(self.cache[service_name], open(filename, 'wb'))
+
+ def clear(self, service_name):
+ try:
+ os.remove(self.cache_location(service_name))
+ except OSError:
+ pass
+ self.cache[service_name] = defaultdict(dict)
+
+ def cached_func_key(self, func, cls=None):
+ try:
+ cls = func.im_class
+ except:
+ pass
+ return ('%s.%s' % (cls.__module__, cls.__name__), func.__name__)
+
+ def function_cache(self, service_name, func):
+ func_key = self.cached_func_key(func)
+ return self.cache[service_name][func_key]
+
+ def cache_for(self, service_name, func, args, result):
+ # no need to lock here, dict ops are atomic
+ self.function_cache(service_name, func)[args] = result
+
+ def cached_value(self, service_name, func, args):
+ """Raises KeyError if not found"""
+ # no need to lock here, dict ops are atomic
+ return self.function_cache(service_name, func)[args]
+
+
+def cachedmethod(function):
+ """Decorator to make a method use the cache.
+
+ WARNING: this can NOT be used with static functions, it has to be used on
+ methods of some class."""
+
+ @wraps(function)
+ def cached(*args):
+ c = args[0].config.cache
+ service_name = args[0].__class__.__name__
+ func_key = c.cached_func_key(function, cls=args[0].__class__)
+ func_cache = c.cache[service_name][func_key]
+
+ # we need to remove the first element of args for the key, as it is the
+ # instance pointer and we don't want the cache to know which instance
+ # called it, it is shared among all instances of the same class
+ key = args[1:]
+
+ if key in func_cache:
+ result = func_cache[key]
+ logger.debug(u'Using cached value for %s(%s), returns: %s' % (func_key, key, result))
+ return result
+
+ result = function(*args)
+
+ # note: another thread could have already cached a value in the
+ # meantime, but that's ok as we prefer to keep the latest value in
+ # the cache
+ func_cache[key] = result
+
+ return result
+
+ return cached
diff --git a/libs/subliminal/core.py b/libs/subliminal/core.py
index 56f4347e..9bbc1cd8 100755
--- a/libs/subliminal/core.py
+++ b/libs/subliminal/core.py
@@ -20,6 +20,8 @@ from .services import ServiceConfig
from .tasks import DownloadTask, ListTask
from .utils import get_keywords
from .videos import Episode, Movie, scan
+from guessit.language import lang_set
+import bs4
from collections import defaultdict
from itertools import groupby
import guessit
@@ -30,11 +32,11 @@ __all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE',
'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence',
'key_subtitles', 'group_by_video']
logger = logging.getLogger(__name__)
-SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb']
+SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles']
LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
-def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth):
+def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth, scan_filter):
"""Create a list of :class:`~subliminal.tasks.ListTask` from one or more paths using the given criteria
:param paths: path(s) to video file or folder
@@ -45,18 +47,20 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
:param bool multi: search multiple languages for the same video
:param string cache_dir: path to the cache directory to use
:param int max_depth: maximum depth for scanning entries
+ :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
:return: the created tasks
:rtype: list of :class:`~subliminal.tasks.ListTask`
"""
scan_result = []
for p in paths:
- scan_result.extend(scan(p, max_depth))
+ scan_result.extend(scan(p, max_depth, scan_filter))
logger.debug(u'Found %d videos in %r with maximum depth %d' % (len(scan_result), paths, max_depth))
tasks = []
config = ServiceConfig(multi, cache_dir)
+ services = filter_services(services)
for video, detected_subtitles in scan_result:
- detected_languages = set([s.language for s in detected_subtitles])
+ detected_languages = set(s.language for s in detected_subtitles)
wanted_languages = languages.copy()
if not force and multi:
wanted_languages -= detected_languages
@@ -70,14 +74,9 @@ def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_d
for service_name in services:
mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
service = mod.Service
- service_languages = wanted_languages & service.available_languages()
- if not service_languages:
- logger.debug(u'Skipping %r: none of wanted languages %r available for service %s' % (video, wanted_languages, service_name))
+ if not service.check_validity(video, wanted_languages):
continue
- if not service.is_valid_video(video):
- logger.debug(u'Skipping %r: not part of supported videos %r for service %s' % (video, service.videos, service_name))
- continue
- task = ListTask(video, service_languages, service_name, config)
+ task = ListTask(video, wanted_languages & service.languages, service_name, config)
logger.debug(u'Created task %r' % task)
tasks.append(task)
return tasks
@@ -128,25 +127,19 @@ def consume_task(task, services=None):
logger.info(u'Consuming %r' % task)
result = None
if isinstance(task, ListTask):
- if task.service not in services:
- mod = __import__('services.' + task.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
- services[task.service] = mod.Service(task.config)
- services[task.service].init()
- subtitles = services[task.service].list(task.video, task.languages)
- result = subtitles
+ service = get_service(services, task.service, config=task.config)
+ result = service.list(task.video, task.languages)
elif isinstance(task, DownloadTask):
for subtitle in task.subtitles:
- if subtitle.service not in services:
- mod = __import__('services.' + subtitle.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
- services[subtitle.service] = mod.Service()
- services[subtitle.service].init()
+ service = get_service(services, subtitle.service)
try:
- services[subtitle.service].download(subtitle)
+ service.download(subtitle)
result = subtitle
break
except DownloadFailedError:
logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
continue
+
if result is None:
logger.error(u'No subtitles could be downloaded for video %r' % task.video)
return result
@@ -193,6 +186,26 @@ def matching_confidence(video, subtitle):
return confidence
+def get_service(services, service_name, config=None):
+ """Get a service from its name in the service dict with the specified config.
+ If the service does not exist in the service dict, it is created and added to the dict.
+
+ :param dict services: dict where to get existing services or put created ones
+ :param string service_name: name of the service to get
+ :param config: config to use for the service
+ :type config: :class:`~subliminal.services.ServiceConfig` or None
+ :return: the corresponding service
+ :rtype: :class:`~subliminal.services.ServiceBase`
+
+ """
+ if service_name not in services:
+ mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+ services[service_name] = mod.Service()
+ services[service_name].init()
+ services[service_name].config = config
+ return services[service_name]
+
+
def key_subtitles(subtitle, video, languages, services, order):
"""Create a key to sort subtitle using the given order
@@ -238,3 +251,21 @@ def group_by_video(list_results):
for video, subtitles in list_results:
result[video] += subtitles
return result
+
+
+def filter_services(services):
+ """Filter out services that are not available because of a missing feature
+
+ :param list services: service names to filter
+ :return: a copy of the initial list of service names without unavailable ones
+ :rtype: list
+
+ """
+ filtered_services = services[:]
+ for service_name in services:
+ mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+ service = mod.Service
+ if service.required_features is not None and bs4.builder_registry.lookup(*service.required_features) is None:
+ logger.warning(u'Service %s not available: none of available features could be used. One of %r required' % (service_name, service.required_features))
+ filtered_services.remove(service_name)
+ return filtered_services
diff --git a/libs/subliminal/infos.py b/libs/subliminal/infos.py
index b28fda00..9958234e 100755
--- a/libs/subliminal/infos.py
+++ b/libs/subliminal/infos.py
@@ -15,4 +15,4 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
-__version__ = '0.5.1'
+__version__ = '0.6.0'
diff --git a/libs/subliminal/languages.py b/libs/subliminal/languages.py
deleted file mode 100755
index f743953f..00000000
--- a/libs/subliminal/languages.py
+++ /dev/null
@@ -1,547 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2011-2012 Antoine Bertin
-#
-# This file is part of subliminal.
-#
-# subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with subliminal. If not, see .
-__all__ = ['convert_language', 'list_languages', 'LANGUAGES']
-
-
-def convert_language(language, to_iso, from_iso=None):
- """Convert a language into another format
-
- :param string language: language
- :param int to_iso: convert language to ISO-639-x
- :param int from_iso: convert language from ISO-639-x
- :return: converted language
- :rtype: string
-
- """
- if from_iso == None: # if no from_iso is given, try to guess it
- if language.startswith(language[:1].upper()):
- from_iso = 0
- elif len(language) == 2:
- from_iso = 1
- elif len(language) == 3:
- from_iso = 2
- else:
- raise ValueError('Invalid input language format')
- if isinstance(language, unicode):
- language = language.encode('utf-8')
- converted_language = None
- for language_tuple in LANGUAGES:
- if language_tuple[from_iso] == language and language_tuple[to_iso]:
- converted_language = language_tuple[to_iso]
- break
- return converted_language
-
-
-def list_languages(iso):
- """List languages in the given ISO-639-x format
-
- :param int iso: ISO-639-x format to list
- :return: languages in the requested format
- :rtype: list
-
- """
- return [l[iso] for l in LANGUAGES if l[iso]]
-
-#: ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-#: + ('Brazilian', 'po', 'pob')
-LANGUAGES = [('Afar', 'aa', 'aar'),
- ('Abkhazian', 'ab', 'abk'),
- ('Achinese', '', 'ace'),
- ('Acoli', '', 'ach'),
- ('Adangme', '', 'ada'),
- ('Adyghe; Adygei', '', 'ady'),
- ('Afro-Asiatic languages', '', 'afa'),
- ('Afrihili', '', 'afh'),
- ('Afrikaans', 'af', 'afr'),
- ('Ainu', '', 'ain'),
- ('Akan', 'ak', 'aka'),
- ('Akkadian', '', 'akk'),
- ('Albanian', 'sq', 'alb'),
- ('Aleut', '', 'ale'),
- ('Algonquian languages', '', 'alg'),
- ('Southern Altai', '', 'alt'),
- ('Amharic', 'am', 'amh'),
- ('English, Old (ca.450-1100)', '', 'ang'),
- ('Angika', '', 'anp'),
- ('Apache languages', '', 'apa'),
- ('Arabic', 'ar', 'ara'),
- ('Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)', '', 'arc'),
- ('Aragonese', 'an', 'arg'),
- ('Armenian', 'hy', 'arm'),
- ('Mapudungun; Mapuche', '', 'arn'),
- ('Arapaho', '', 'arp'),
- ('Artificial languages', '', 'art'),
- ('Arawak', '', 'arw'),
- ('Assamese', 'as', 'asm'),
- ('Asturian; Bable; Leonese; Asturleonese', '', 'ast'),
- ('Athapascan languages', '', 'ath'),
- ('Australian languages', '', 'aus'),
- ('Avaric', 'av', 'ava'),
- ('Avestan', 'ae', 'ave'),
- ('Awadhi', '', 'awa'),
- ('Aymara', 'ay', 'aym'),
- ('Azerbaijani', 'az', 'aze'),
- ('Banda languages', '', 'bad'),
- ('Bamileke languages', '', 'bai'),
- ('Bashkir', 'ba', 'bak'),
- ('Baluchi', '', 'bal'),
- ('Bambara', 'bm', 'bam'),
- ('Balinese', '', 'ban'),
- ('Basque', 'eu', 'baq'),
- ('Basa', '', 'bas'),
- ('Baltic languages', '', 'bat'),
- ('Beja; Bedawiyet', '', 'bej'),
- ('Belarusian', 'be', 'bel'),
- ('Bemba', '', 'bem'),
- ('Bengali', 'bn', 'ben'),
- ('Berber languages', '', 'ber'),
- ('Bhojpuri', '', 'bho'),
- ('Bihari languages', 'bh', 'bih'),
- ('Bikol', '', 'bik'),
- ('Bini; Edo', '', 'bin'),
- ('Bislama', 'bi', 'bis'),
- ('Siksika', '', 'bla'),
- ('Bantu (Other)', '', 'bnt'),
- ('Bosnian', 'bs', 'bos'),
- ('Braj', '', 'bra'),
- ('Breton', 'br', 'bre'),
- ('Batak languages', '', 'btk'),
- ('Buriat', '', 'bua'),
- ('Buginese', '', 'bug'),
- ('Bulgarian', 'bg', 'bul'),
- ('Burmese', 'my', 'bur'),
- ('Blin; Bilin', '', 'byn'),
- ('Caddo', '', 'cad'),
- ('Central American Indian languages', '', 'cai'),
- ('Galibi Carib', '', 'car'),
- ('Catalan; Valencian', 'ca', 'cat'),
- ('Caucasian languages', '', 'cau'),
- ('Cebuano', '', 'ceb'),
- ('Celtic languages', '', 'cel'),
- ('Chamorro', 'ch', 'cha'),
- ('Chibcha', '', 'chb'),
- ('Chechen', 'ce', 'che'),
- ('Chagatai', '', 'chg'),
- ('Chinese', 'zh', 'chi'),
- ('Chuukese', '', 'chk'),
- ('Mari', '', 'chm'),
- ('Chinook jargon', '', 'chn'),
- ('Choctaw', '', 'cho'),
- ('Chipewyan; Dene Suline', '', 'chp'),
- ('Cherokee', '', 'chr'),
- ('Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic', 'cu', 'chu'),
- ('Chuvash', 'cv', 'chv'),
- ('Cheyenne', '', 'chy'),
- ('Chamic languages', '', 'cmc'),
- ('Coptic', '', 'cop'),
- ('Cornish', 'kw', 'cor'),
- ('Corsican', 'co', 'cos'),
- ('Creoles and pidgins, English based', '', 'cpe'),
- ('Creoles and pidgins, French-based ', '', 'cpf'),
- ('Creoles and pidgins, Portuguese-based ', '', 'cpp'),
- ('Cree', 'cr', 'cre'),
- ('Crimean Tatar; Crimean Turkish', '', 'crh'),
- ('Creoles and pidgins ', '', 'crp'),
- ('Kashubian', '', 'csb'),
- ('Cushitic languages', '', 'cus'),
- ('Czech', 'cs', 'cze'),
- ('Dakota', '', 'dak'),
- ('Danish', 'da', 'dan'),
- ('Dargwa', '', 'dar'),
- ('Land Dayak languages', '', 'day'),
- ('Delaware', '', 'del'),
- ('Slave (Athapascan)', '', 'den'),
- ('Dogrib', '', 'dgr'),
- ('Dinka', '', 'din'),
- ('Divehi; Dhivehi; Maldivian', 'dv', 'div'),
- ('Dogri', '', 'doi'),
- ('Dravidian languages', '', 'dra'),
- ('Lower Sorbian', '', 'dsb'),
- ('Duala', '', 'dua'),
- ('Dutch, Middle (ca.1050-1350)', '', 'dum'),
- ('Dutch; Flemish', 'nl', 'dut'),
- ('Dyula', '', 'dyu'),
- ('Dzongkha', 'dz', 'dzo'),
- ('Efik', '', 'efi'),
- ('Egyptian (Ancient)', '', 'egy'),
- ('Ekajuk', '', 'eka'),
- ('Elamite', '', 'elx'),
- ('English', 'en', 'eng'),
- ('English, Middle (1100-1500)', '', 'enm'),
- ('Esperanto', 'eo', 'epo'),
- ('Estonian', 'et', 'est'),
- ('Ewe', 'ee', 'ewe'),
- ('Ewondo', '', 'ewo'),
- ('Fang', '', 'fan'),
- ('Faroese', 'fo', 'fao'),
- ('Fanti', '', 'fat'),
- ('Fijian', 'fj', 'fij'),
- ('Filipino; Pilipino', '', 'fil'),
- ('Finnish', 'fi', 'fin'),
- ('Finno-Ugrian languages', '', 'fiu'),
- ('Fon', '', 'fon'),
- ('French', 'fr', 'fre'),
- ('French, Middle (ca.1400-1600)', '', 'frm'),
- ('French, Old (842-ca.1400)', '', 'fro'),
- ('Northern Frisian', '', 'frr'),
- ('Eastern Frisian', '', 'frs'),
- ('Western Frisian', 'fy', 'fry'),
- ('Fulah', 'ff', 'ful'),
- ('Friulian', '', 'fur'),
- ('Ga', '', 'gaa'),
- ('Gayo', '', 'gay'),
- ('Gbaya', '', 'gba'),
- ('Germanic languages', '', 'gem'),
- ('Georgian', 'ka', 'geo'),
- ('German', 'de', 'ger'),
- ('Geez', '', 'gez'),
- ('Gilbertese', '', 'gil'),
- ('Gaelic; Scottish Gaelic', 'gd', 'gla'),
- ('Irish', 'ga', 'gle'),
- ('Galician', 'gl', 'glg'),
- ('Manx', 'gv', 'glv'),
- ('German, Middle High (ca.1050-1500)', '', 'gmh'),
- ('German, Old High (ca.750-1050)', '', 'goh'),
- ('Gondi', '', 'gon'),
- ('Gorontalo', '', 'gor'),
- ('Gothic', '', 'got'),
- ('Grebo', '', 'grb'),
- ('Greek, Ancient (to 1453)', '', 'grc'),
- ('Greek, Modern (1453-)', 'el', 'gre'),
- ('Guarani', 'gn', 'grn'),
- ('Swiss German; Alemannic; Alsatian', '', 'gsw'),
- ('Gujarati', 'gu', 'guj'),
- ('Gwich\'in', '', 'gwi'),
- ('Haida', '', 'hai'),
- ('Haitian; Haitian Creole', 'ht', 'hat'),
- ('Hausa', 'ha', 'hau'),
- ('Hawaiian', '', 'haw'),
- ('Hebrew', 'he', 'heb'),
- ('Herero', 'hz', 'her'),
- ('Hiligaynon', '', 'hil'),
- ('Himachali languages; Western Pahari languages', '', 'him'),
- ('Hindi', 'hi', 'hin'),
- ('Hittite', '', 'hit'),
- ('Hmong; Mong', '', 'hmn'),
- ('Hiri Motu', 'ho', 'hmo'),
- ('Croatian', 'hr', 'hrv'),
- ('Upper Sorbian', '', 'hsb'),
- ('Hungarian', 'hu', 'hun'),
- ('Hupa', '', 'hup'),
- ('Iban', '', 'iba'),
- ('Igbo', 'ig', 'ibo'),
- ('Icelandic', 'is', 'ice'),
- ('Ido', 'io', 'ido'),
- ('Sichuan Yi; Nuosu', 'ii', 'iii'),
- ('Ijo languages', '', 'ijo'),
- ('Inuktitut', 'iu', 'iku'),
- ('Interlingue; Occidental', 'ie', 'ile'),
- ('Iloko', '', 'ilo'),
- ('Interlingua (International Auxiliary Language Association)', 'ia', 'ina'),
- ('Indic languages', '', 'inc'),
- ('Indonesian', 'id', 'ind'),
- ('Indo-European languages', '', 'ine'),
- ('Ingush', '', 'inh'),
- ('Inupiaq', 'ik', 'ipk'),
- ('Iranian languages', '', 'ira'),
- ('Iroquoian languages', '', 'iro'),
- ('Italian', 'it', 'ita'),
- ('Javanese', 'jv', 'jav'),
- ('Lojban', '', 'jbo'),
- ('Japanese', 'ja', 'jpn'),
- ('Judeo-Persian', '', 'jpr'),
- ('Judeo-Arabic', '', 'jrb'),
- ('Kara-Kalpak', '', 'kaa'),
- ('Kabyle', '', 'kab'),
- ('Kachin; Jingpho', '', 'kac'),
- ('Kalaallisut; Greenlandic', 'kl', 'kal'),
- ('Kamba', '', 'kam'),
- ('Kannada', 'kn', 'kan'),
- ('Karen languages', '', 'kar'),
- ('Kashmiri', 'ks', 'kas'),
- ('Kanuri', 'kr', 'kau'),
- ('Kawi', '', 'kaw'),
- ('Kazakh', 'kk', 'kaz'),
- ('Kabardian', '', 'kbd'),
- ('Khasi', '', 'kha'),
- ('Khoisan languages', '', 'khi'),
- ('Central Khmer', 'km', 'khm'),
- ('Khotanese; Sakan', '', 'kho'),
- ('Kikuyu; Gikuyu', 'ki', 'kik'),
- ('Kinyarwanda', 'rw', 'kin'),
- ('Kirghiz; Kyrgyz', 'ky', 'kir'),
- ('Kimbundu', '', 'kmb'),
- ('Konkani', '', 'kok'),
- ('Komi', 'kv', 'kom'),
- ('Kongo', 'kg', 'kon'),
- ('Korean', 'ko', 'kor'),
- ('Kosraean', '', 'kos'),
- ('Kpelle', '', 'kpe'),
- ('Karachay-Balkar', '', 'krc'),
- ('Karelian', '', 'krl'),
- ('Kru languages', '', 'kro'),
- ('Kurukh', '', 'kru'),
- ('Kuanyama; Kwanyama', 'kj', 'kua'),
- ('Kumyk', '', 'kum'),
- ('Kurdish', 'ku', 'kur'),
- ('Kutenai', '', 'kut'),
- ('Ladino', '', 'lad'),
- ('Lahnda', '', 'lah'),
- ('Lamba', '', 'lam'),
- ('Lao', 'lo', 'lao'),
- ('Latin', 'la', 'lat'),
- ('Latvian', 'lv', 'lav'),
- ('Lezghian', '', 'lez'),
- ('Limburgan; Limburger; Limburgish', 'li', 'lim'),
- ('Lingala', 'ln', 'lin'),
- ('Lithuanian', 'lt', 'lit'),
- ('Mongo', '', 'lol'),
- ('Lozi', '', 'loz'),
- ('Luxembourgish; Letzeburgesch', 'lb', 'ltz'),
- ('Luba-Lulua', '', 'lua'),
- ('Luba-Katanga', 'lu', 'lub'),
- ('Ganda', 'lg', 'lug'),
- ('Luiseno', '', 'lui'),
- ('Lunda', '', 'lun'),
- ('Luo (Kenya and Tanzania)', '', 'luo'),
- ('Lushai', '', 'lus'),
- ('Macedonian', 'mk', 'mac'),
- ('Madurese', '', 'mad'),
- ('Magahi', '', 'mag'),
- ('Marshallese', 'mh', 'mah'),
- ('Maithili', '', 'mai'),
- ('Makasar', '', 'mak'),
- ('Malayalam', 'ml', 'mal'),
- ('Mandingo', '', 'man'),
- ('Maori', 'mi', 'mao'),
- ('Austronesian languages', '', 'map'),
- ('Marathi', 'mr', 'mar'),
- ('Masai', '', 'mas'),
- ('Malay', 'ms', 'may'),
- ('Moksha', '', 'mdf'),
- ('Mandar', '', 'mdr'),
- ('Mende', '', 'men'),
- ('Irish, Middle (900-1200)', '', 'mga'),
- ('Mi\'kmaq; Micmac', '', 'mic'),
- ('Minangkabau', '', 'min'),
- ('Uncoded languages', '', 'mis'),
- ('Mon-Khmer languages', '', 'mkh'),
- ('Malagasy', 'mg', 'mlg'),
- ('Maltese', 'mt', 'mlt'),
- ('Manchu', '', 'mnc'),
- ('Manipuri', '', 'mni'),
- ('Manobo languages', '', 'mno'),
- ('Mohawk', '', 'moh'),
- ('Mongolian', 'mn', 'mon'),
- ('Mossi', '', 'mos'),
- ('Multiple languages', '', 'mul'),
- ('Munda languages', '', 'mun'),
- ('Creek', '', 'mus'),
- ('Mirandese', '', 'mwl'),
- ('Marwari', '', 'mwr'),
- ('Mayan languages', '', 'myn'),
- ('Erzya', '', 'myv'),
- ('Nahuatl languages', '', 'nah'),
- ('North American Indian languages', '', 'nai'),
- ('Neapolitan', '', 'nap'),
- ('Nauru', 'na', 'nau'),
- ('Navajo; Navaho', 'nv', 'nav'),
- ('Ndebele, South; South Ndebele', 'nr', 'nbl'),
- ('Ndebele, North; North Ndebele', 'nd', 'nde'),
- ('Ndonga', 'ng', 'ndo'),
- ('Low German; Low Saxon; German, Low; Saxon, Low', '', 'nds'),
- ('Nepali', 'ne', 'nep'),
- ('Nepal Bhasa; Newari', '', 'new'),
- ('Nias', '', 'nia'),
- ('Niger-Kordofanian languages', '', 'nic'),
- ('Niuean', '', 'niu'),
- ('Norwegian Nynorsk; Nynorsk, Norwegian', 'nn', 'nno'),
- ('Bokmål, Norwegian; Norwegian Bokmål', 'nb', 'nob'),
- ('Nogai', '', 'nog'),
- ('Norse, Old', '', 'non'),
- ('Norwegian', 'no', 'nor'),
- ('N\'Ko', '', 'nqo'),
- ('Pedi; Sepedi; Northern Sotho', '', 'nso'),
- ('Nubian languages', '', 'nub'),
- ('Classical Newari; Old Newari; Classical Nepal Bhasa', '', 'nwc'),
- ('Chichewa; Chewa; Nyanja', 'ny', 'nya'),
- ('Nyamwezi', '', 'nym'),
- ('Nyankole', '', 'nyn'),
- ('Nyoro', '', 'nyo'),
- ('Nzima', '', 'nzi'),
- ('Occitan (post 1500); Provençal', 'oc', 'oci'),
- ('Ojibwa', 'oj', 'oji'),
- ('Oriya', 'or', 'ori'),
- ('Oromo', 'om', 'orm'),
- ('Osage', '', 'osa'),
- ('Ossetian; Ossetic', 'os', 'oss'),
- ('Turkish, Ottoman (1500-1928)', '', 'ota'),
- ('Otomian languages', '', 'oto'),
- ('Papuan languages', '', 'paa'),
- ('Pangasinan', '', 'pag'),
- ('Pahlavi', '', 'pal'),
- ('Pampanga; Kapampangan', '', 'pam'),
- ('Panjabi; Punjabi', 'pa', 'pan'),
- ('Papiamento', '', 'pap'),
- ('Palauan', '', 'pau'),
- ('Persian, Old (ca.600-400 B.C.)', '', 'peo'),
- ('Persian', 'fa', 'per'),
- ('Philippine languages', '', 'phi'),
- ('Phoenician', '', 'phn'),
- ('Pali', 'pi', 'pli'),
- ('Polish', 'pl', 'pol'),
- ('Pohnpeian', '', 'pon'),
- ('Portuguese', 'pt', 'por'),
- ('Prakrit languages', '', 'pra'),
- ('Provençal, Old (to 1500)', '', 'pro'),
- ('Pushto; Pashto', 'ps', 'pus'),
- ('Reserved for local use', '', 'qaa-qtz'),
- ('Quechua', 'qu', 'que'),
- ('Rajasthani', '', 'raj'),
- ('Rapanui', '', 'rap'),
- ('Rarotongan; Cook Islands Maori', '', 'rar'),
- ('Romance languages', '', 'roa'),
- ('Romansh', 'rm', 'roh'),
- ('Romany', '', 'rom'),
- ('Romanian; Moldavian; Moldovan', 'ro', 'rum'),
- ('Rundi', 'rn', 'run'),
- ('Aromanian; Arumanian; Macedo-Romanian', '', 'rup'),
- ('Russian', 'ru', 'rus'),
- ('Sandawe', '', 'sad'),
- ('Sango', 'sg', 'sag'),
- ('Yakut', '', 'sah'),
- ('South American Indian (Other)', '', 'sai'),
- ('Salishan languages', '', 'sal'),
- ('Samaritan Aramaic', '', 'sam'),
- ('Sanskrit', 'sa', 'san'),
- ('Sasak', '', 'sas'),
- ('Santali', '', 'sat'),
- ('Sicilian', '', 'scn'),
- ('Scots', '', 'sco'),
- ('Selkup', '', 'sel'),
- ('Semitic languages', '', 'sem'),
- ('Irish, Old (to 900)', '', 'sga'),
- ('Sign Languages', '', 'sgn'),
- ('Shan', '', 'shn'),
- ('Sidamo', '', 'sid'),
- ('Sinhala; Sinhalese', 'si', 'sin'),
- ('Siouan languages', '', 'sio'),
- ('Sino-Tibetan languages', '', 'sit'),
- ('Slavic languages', '', 'sla'),
- ('Slovak', 'sk', 'slo'),
- ('Slovenian', 'sl', 'slv'),
- ('Southern Sami', '', 'sma'),
- ('Northern Sami', 'se', 'sme'),
- ('Sami languages', '', 'smi'),
- ('Lule Sami', '', 'smj'),
- ('Inari Sami', '', 'smn'),
- ('Samoan', 'sm', 'smo'),
- ('Skolt Sami', '', 'sms'),
- ('Shona', 'sn', 'sna'),
- ('Sindhi', 'sd', 'snd'),
- ('Soninke', '', 'snk'),
- ('Sogdian', '', 'sog'),
- ('Somali', 'so', 'som'),
- ('Songhai languages', '', 'son'),
- ('Sotho, Southern', 'st', 'sot'),
- ('Spanish; Castilian', 'es', 'spa'),
- ('Sardinian', 'sc', 'srd'),
- ('Sranan Tongo', '', 'srn'),
- ('Serbian', 'sr', 'srp'),
- ('Serer', '', 'srr'),
- ('Nilo-Saharan languages', '', 'ssa'),
- ('Swati', 'ss', 'ssw'),
- ('Sukuma', '', 'suk'),
- ('Sundanese', 'su', 'sun'),
- ('Susu', '', 'sus'),
- ('Sumerian', '', 'sux'),
- ('Swahili', 'sw', 'swa'),
- ('Swedish', 'sv', 'swe'),
- ('Classical Syriac', '', 'syc'),
- ('Syriac', '', 'syr'),
- ('Tahitian', 'ty', 'tah'),
- ('Tai languages', '', 'tai'),
- ('Tamil', 'ta', 'tam'),
- ('Tatar', 'tt', 'tat'),
- ('Telugu', 'te', 'tel'),
- ('Timne', '', 'tem'),
- ('Tereno', '', 'ter'),
- ('Tetum', '', 'tet'),
- ('Tajik', 'tg', 'tgk'),
- ('Tagalog', 'tl', 'tgl'),
- ('Thai', 'th', 'tha'),
- ('Tibetan', 'bo', 'tib'),
- ('Tigre', '', 'tig'),
- ('Tigrinya', 'ti', 'tir'),
- ('Tiv', '', 'tiv'),
- ('Tokelau', '', 'tkl'),
- ('Klingon; tlhIngan-Hol', '', 'tlh'),
- ('Tlingit', '', 'tli'),
- ('Tamashek', '', 'tmh'),
- ('Tonga (Nyasa)', '', 'tog'),
- ('Tonga (Tonga Islands)', 'to', 'ton'),
- ('Tok Pisin', '', 'tpi'),
- ('Tsimshian', '', 'tsi'),
- ('Tswana', 'tn', 'tsn'),
- ('Tsonga', 'ts', 'tso'),
- ('Turkmen', 'tk', 'tuk'),
- ('Tumbuka', '', 'tum'),
- ('Tupi languages', '', 'tup'),
- ('Turkish', 'tr', 'tur'),
- ('Altaic languages', '', 'tut'),
- ('Tuvalu', '', 'tvl'),
- ('Twi', 'tw', 'twi'),
- ('Tuvinian', '', 'tyv'),
- ('Udmurt', '', 'udm'),
- ('Ugaritic', '', 'uga'),
- ('Uighur; Uyghur', 'ug', 'uig'),
- ('Ukrainian', 'uk', 'ukr'),
- ('Umbundu', '', 'umb'),
- ('Undetermined', '', 'und'),
- ('Urdu', 'ur', 'urd'),
- ('Uzbek', 'uz', 'uzb'),
- ('Vai', '', 'vai'),
- ('Venda', 've', 'ven'),
- ('Vietnamese', 'vi', 'vie'),
- ('Volapük', 'vo', 'vol'),
- ('Votic', '', 'vot'),
- ('Wakashan languages', '', 'wak'),
- ('Walamo', '', 'wal'),
- ('Waray', '', 'war'),
- ('Washo', '', 'was'),
- ('Welsh', 'cy', 'wel'),
- ('Sorbian languages', '', 'wen'),
- ('Walloon', 'wa', 'wln'),
- ('Wolof', 'wo', 'wol'),
- ('Kalmyk; Oirat', '', 'xal'),
- ('Xhosa', 'xh', 'xho'),
- ('Yao', '', 'yao'),
- ('Yapese', '', 'yap'),
- ('Yiddish', 'yi', 'yid'),
- ('Yoruba', 'yo', 'yor'),
- ('Yupik languages', '', 'ypk'),
- ('Zapotec', '', 'zap'),
- ('Blissymbols; Blissymbolics; Bliss', '', 'zbl'),
- ('Zenaga', '', 'zen'),
- ('Zhuang; Chuang', 'za', 'zha'),
- ('Zande languages', '', 'znd'),
- ('Zulu', 'zu', 'zul'),
- ('Zuni', '', 'zun'),
- ('No linguistic content; Not applicable', '', 'zxx'),
- ('Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki', '', 'zza'),
- ('Brazilian', 'po', 'pob')]
diff --git a/libs/subliminal/services/__init__.py b/libs/subliminal/services/__init__.py
index 67e457fe..e1aea2d4 100755
--- a/libs/subliminal/services/__init__.py
+++ b/libs/subliminal/services/__init__.py
@@ -15,11 +15,15 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
-from ..exceptions import MissingLanguageError, DownloadFailedError
+from .. import cache
+from ..exceptions import MissingLanguageError, DownloadFailedError, ServiceError
+from ..subtitles import EXTENSIONS
+from guessit.language import lang_set, UNDETERMINED
import logging
import os
import requests
import threading
+import zipfile
__all__ = ['ServiceBase', 'ServiceConfig']
@@ -37,7 +41,7 @@ class ServiceBase(object):
server_url = ''
#: User Agent for any HTTP-based requests
- user_agent = 'subliminal v0.5'
+ user_agent = 'subliminal v0.6'
#: Whether based on an API or not
api_based = False
@@ -45,21 +49,18 @@ class ServiceBase(object):
#: Timeout for web requests
timeout = 5
- #: Lock for cache interactions
- lock = threading.Lock()
-
#: Mapping to Service's language codes and subliminal's
languages = {}
- #: Whether the mapping is reverted or not
- reverted_languages = False
-
#: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
videos = []
#: Whether the video has to exist or not
require_video = False
+ #: List of required features for BeautifulSoup
+ required_features = None
+
def __init__(self, config=None):
self.config = config or ServiceConfig()
@@ -75,6 +76,30 @@ class ServiceBase(object):
logger.debug(u'Initializing %s' % self.__class__.__name__)
self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent})
+ def init_cache(self):
+ """Initialize cache, make sure it is loaded from disk"""
+ if not self.config or not self.config.cache:
+ raise ServiceError('Cache directory is required')
+
+ service_name = self.__class__.__name__
+ self.config.cache.load(service_name)
+
+ def save_cache(self):
+ service_name = self.__class__.__name__
+ self.config.cache.save(service_name)
+
+ def clear_cache(self):
+ service_name = self.__class__.__name__
+ self.config.cache.clear(service_name)
+
+ def cache_for(self, func, args, result):
+ service_name = self.__class__.__name__
+ return self.config.cache.cache_for(service_name, func, args, result)
+
+ def cached_value(self, func, args):
+ service_name = self.__class__.__name__
+ return self.config.cache.cached_value(service_name, func, args)
+
def terminate(self):
"""Terminate connection"""
logger.debug(u'Terminating %s' % self.__class__.__name__)
@@ -84,25 +109,20 @@ class ServiceBase(object):
pass
def list(self, video, languages):
- """List subtitles"""
- pass
+ """List subtitles
+
+ As a service writer, you can either override this method or implement
+ :meth:`list_checked` instead to have the languages pre-filtered for you
+
+ """
+ if not self.check_validity(video, languages):
+ return []
+ return self.list_checked(video, languages)
def download(self, subtitle):
"""Download a subtitle"""
self.download_file(subtitle.link, subtitle.path)
- @classmethod
- def available_languages(cls):
- """Available languages in the Service
-
- :return: available languages
- :rtype: set
-
- """
- if not cls.reverted_languages:
- return set(cls.languages.keys())
- return set(cls.languages.values())
-
@classmethod
def check_validity(cls, video, languages):
"""Check for video and languages validity in the Service
@@ -113,72 +133,15 @@ class ServiceBase(object):
:rtype: bool
"""
- languages &= cls.available_languages()
+ languages = (lang_set(languages) & cls.languages) - set([UNDETERMINED])
if not languages:
- logger.debug(u'No language available for service %s' % cls.__class__.__name__.lower())
+ logger.debug(u'No language available for service %s' % cls.__name__.lower())
return False
- if not cls.is_valid_video(video):
- logger.debug(u'%r is not valid for service %s' % (video, cls.__class__.__name__.lower()))
+ if cls.require_video and not video.exists or not isinstance(video, tuple(cls.videos)):
+ logger.debug(u'%r is not valid for service %s' % (video, cls.__name__.lower()))
return False
return True
- @classmethod
- def is_valid_video(cls, video):
- """Check if video is valid in the Service
-
- :param video: the video to check
- :type video: :class:`~subliminal.videos.Video`
- :rtype: bool
-
- """
- if cls.require_video and not video.exists:
- return False
- if not isinstance(video, tuple(cls.videos)):
- return False
- return True
-
- @classmethod
- def is_valid_language(cls, language):
- """Check if language is valid in the Service
-
- :param string language: the language to check
- :rtype: bool
-
- """
- if language in cls.available_languages():
- return True
- return False
-
- @classmethod
- def get_revert_language(cls, language):
- """ISO-639-1 language code from service language code
-
- :param string language: service language code
- :return: ISO-639-1 language code
- :rtype: string
-
- """
- if not cls.reverted_languages and language in cls.languages.values():
- return [k for k, v in cls.languages.iteritems() if v == language][0]
- if cls.reverted_languages and language in cls.languages.keys():
- return cls.languages[language]
- raise MissingLanguageError(language)
-
- @classmethod
- def get_language(cls, language):
- """Service language code from ISO-639-1 language code
-
- :param string language: ISO-639-1 language code
- :return: service language code
- :rtype: string
-
- """
- if not cls.reverted_languages and language in cls.languages.keys():
- return cls.languages[language]
- if cls.reverted_languages and language in cls.languages.values():
- return [k for k, v in cls.languages.iteritems() if v == language][0]
- raise MissingLanguageError(language)
-
def download_file(self, url, filepath):
"""Attempt to download a file and remove it in case of failure
@@ -198,6 +161,43 @@ class ServiceBase(object):
raise DownloadFailedError(str(e))
logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+ def download_zip_file(self, url, filepath):
+ """Attempt to download a zip file and extract any subtitle file from it, if any.
+ This cleans up after itself if anything fails.
+
+ :param string url: URL of the zip file to download
+ :param string filepath: destination path for the subtitle
+
+ """
+ logger.info(u'Downloading %s' % url)
+ try:
+ zippath = filepath + '.zip'
+ r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
+ with open(zippath, 'wb') as f:
+ f.write(r.content)
+ if not zipfile.is_zipfile(zippath):
+ # TODO: could check if maybe we already have a text file and
+ # download it directly
+ raise DownloadFailedError('Downloaded file is not a zip file')
+ zipsub = zipfile.ZipFile(zippath)
+ for subfile in zipsub.namelist():
+ if os.path.splitext(subfile)[1] in EXTENSIONS:
+ open(filepath, 'w').write(zipsub.open(subfile).read())
+ break
+ else:
+ logger.debug(u'No subtitles found in zip file')
+ raise DownloadFailedError('No subtitles found in zip file')
+ os.remove(zippath)
+ logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+ return
+ except Exception as e:
+ logger.error(u'Download %s failed: %s' % (url, e))
+ if os.path.exists(zippath):
+ os.remove(zippath)
+ if os.path.exists(filepath):
+ os.remove(filepath)
+ raise DownloadFailedError(str(e))
+
class ServiceConfig(object):
"""Configuration for any :class:`Service`
@@ -209,6 +209,9 @@ class ServiceConfig(object):
def __init__(self, multi=False, cache_dir=None):
self.multi = multi
self.cache_dir = cache_dir
+ self.cache = None
+ if cache_dir is not None:
+ self.cache = cache.Cache(cache_dir)
def __repr__(self):
- return 'ServiceConfig(%r, %s)' % (self.multi, self.cache_dir)
+ return 'ServiceConfig(%r, %s)' % (self.multi, self.cache.cache_dir)
diff --git a/libs/subliminal/services/addic7ed.py b/libs/subliminal/services/addic7ed.py
new file mode 100755
index 00000000..c754de55
--- /dev/null
+++ b/libs/subliminal/services/addic7ed.py
@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Olivier Leveau
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..cache import cachedmethod
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode
+from bs4 import BeautifulSoup
+from guessit.language import lang_set
+from subliminal.utils import get_keywords
+import guessit
+import logging
+import re
+
+
+logger = logging.getLogger(__name__)
+
+
+def match(pattern, string):
+ try:
+ return re.search(pattern, string).group(1)
+ except AttributeError:
+ logger.debug(u'Could not match %r on %r' % (pattern, string))
+ return None
+
+
+def matches(pattern, string):
+ try:
+ return re.search(pattern, string).group(1, 2)
+ except AttributeError:
+ logger.debug(u'Could not match %r on %r' % (pattern, string))
+ return None
+
+
+class Addic7ed(ServiceBase):
+ server_url = 'http://www.addic7ed.com'
+ api_based = False
+ languages = lang_set([u'English', u'Italian', u'Portuguese',
+ u'Portuguese (Brazilian)', u'Romanian',
+ u'Spanish', u'French', u'Greek', u'Arabic',
+ u'German', u'Croatian', u'Indonesian', u'Hebrew',
+ u'Russian', u'Turkish', u'Swedish', u'Czech',
+ u'Dutch', u'Hungarian', u'Norwegian', u'Polish',
+ u'Persian'], strict=True)
+ videos = [Episode]
+ require_video = False
+ required_features = ['permissive']
+
+ @cachedmethod
+ def get_likely_series_id(self, name):
+ r = self.session.get('%s/shows.php' % self.server_url)
+ soup = BeautifulSoup(r.content, self.required_features)
+ for elem in soup.find_all('h3'):
+ show_name = elem.a.text.lower()
+ show_id = int(match('show/([0-9]+)', elem.a['href']))
+ # we could just return the id of the queried show, but as we
+ # already downloaded the whole page we might as well fill in the
+ # information for all the shows
+ self.cache_for(self.get_likely_series_id, args=(show_name,), result=show_id)
+ return self.cached_value(self.get_likely_series_id, args=(name,))
+
+ @cachedmethod
+ def get_episode_url(self, series_id, season, number):
+ """Get the Addic7ed id for the given episode. Raises KeyError if none
+ could be found
+
+ """
+ # download the page of the show, contains ids for all episodes all seasons
+ r = self.session.get('%s/show/%d' % (self.server_url, series_id))
+ soup = BeautifulSoup(r.content, self.required_features)
+ form = soup.find('form', attrs={'name': 'multidl'})
+ for table in form.find_all('table'):
+ for row in table.find_all('tr'):
+ cell = row.find('td', 'MultiDldS')
+ if not cell:
+ continue
+ m = matches('/serie/.+/([0-9]+)/([0-9]+)/', cell.a['href'])
+ if not m:
+ continue
+ episode_url = cell.a['href']
+ season_number = int(m[0])
+ episode_number = int(m[1])
+ # we could just return the url of the queried episode, but as we
+ # already downloaded the whole page we might as well fill in the
+ # information for all the episodes of the show
+ self.cache_for(self.get_episode_url, args=(series_id, season_number, episode_number), result=episode_url)
+ # raises KeyError if not found
+ return self.cached_value(self.get_episode_url, args=(series_id, season, number))
+
+ # Do not cache this method in order to always check for the most recent
+ # subtitles
+ def get_sub_urls(self, episode_url):
+ suburls = []
+ r = self.session.get('%s/%s' % (self.server_url, episode_url))
+ epsoup = BeautifulSoup(r.content, self.required_features)
+ for releaseTable in epsoup.find_all('table', 'tabel95'):
+ releaseRow = releaseTable.find('td', 'NewsTitle')
+ if not releaseRow:
+ continue
+ release = releaseRow.text.strip()
+ for row in releaseTable.find_all('tr'):
+ link = row.find('a', 'buttonDownload')
+ if not link:
+ continue
+ if 'href' not in link.attrs or not (link['href'].startswith('/original') or link['href'].startswith('/updated')):
+ continue
+ suburl = link['href']
+ lang = guessit.Language(row.find('td', 'language').text.strip())
+ result = {'suburl': suburl, 'language': lang, 'release': release}
+ suburls.append(result)
+ return suburls
+
+ def list_checked(self, video, languages):
+ return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
+
+ def query(self, filepath, languages, keywords, series, season, episode):
+ logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+ self.init_cache()
+ try:
+ sid = self.get_likely_series_id(series.lower())
+ except KeyError:
+ logger.debug(u'Could not find series id for %s' % series)
+ return []
+
+ try:
+ ep_url = self.get_episode_url(sid, season, episode)
+ except KeyError:
+ logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
+ return []
+ suburls = self.get_sub_urls(ep_url)
+
+ # filter the subtitles with our queried languages
+ subtitles = []
+ for suburl in suburls:
+ language = suburl['language']
+ if language not in languages:
+ continue
+
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
+ '%s/%s' % (self.server_url, suburl['suburl']),
+ keywords=[suburl['release']])
+ subtitles.append(subtitle)
+ return subtitles
+
+
+Service = Addic7ed
diff --git a/libs/subliminal/services/bierdopje.py b/libs/subliminal/services/bierdopje.py
index 15401ada..ab6c577c 100755
--- a/libs/subliminal/services/bierdopje.py
+++ b/libs/subliminal/services/bierdopje.py
@@ -16,13 +16,14 @@
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
from . import ServiceBase
+from ..cache import cachedmethod
from ..exceptions import ServiceError
from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..videos import Episode
from ..utils import to_unicode
-import BeautifulSoup
+from ..videos import Episode
+from bs4 import BeautifulSoup
+from guessit.language import lang_set
import logging
-import os.path
import urllib
try:
import cPickle as pickle
@@ -36,30 +37,23 @@ logger = logging.getLogger(__name__)
class BierDopje(ServiceBase):
server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
api_based = True
- languages = {'en': 'en', 'nl': 'nl'}
- reverted_languages = False
+ languages = lang_set(['en', 'nl'])
videos = [Episode]
require_video = False
+ required_features = ['xml']
- def __init__(self, config=None):
- super(BierDopje, self).__init__(config)
- self.showids = {}
- if self.config and self.config.cache_dir:
- self.init_cache()
+ @cachedmethod
+ def get_show_id(self, series):
+ r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
+ if r.status_code != 200:
+ logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+ return None
+ soup = BeautifulSoup(r.content, self.required_features)
+ if soup.status.contents[0] == 'false':
+ logger.debug(u'Could not find show %s' % series)
+ return None
- def init_cache(self):
- logger.debug(u'Initializing cache...')
- if not self.config or not self.config.cache_dir:
- raise ServiceError('Cache directory is required')
- self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache')
- if not os.path.exists(self.showids_cache):
- self.save_cache()
-
- def save_cache(self):
- logger.debug(u'Saving showids to cache...')
- with self.lock:
- with open(self.showids_cache, 'w') as f:
- pickle.dump(self.showids, f)
+ return int(soup.showid.contents[0])
def load_cache(self):
logger.debug(u'Loading showids from cache...')
@@ -67,25 +61,12 @@ class BierDopje(ServiceBase):
with open(self.showids_cache, 'r') as f:
self.showids = pickle.load(f)
- def query(self, season, episode, languages, filepath, tvdbid=None, series=None):
- self.load_cache()
+ def query(self, filepath, season, episode, languages, tvdbid=None, series=None):
+ self.init_cache()
if series:
- if series.lower() in self.showids: # from cache
- request_id = self.showids[series.lower()]
- logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series))
- else: # query to get showid
- logger.debug(u'Getting showid from show name %s...' % series)
- r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
- if r.status_code != 200:
- logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
- return []
- soup = BeautifulSoup.BeautifulStoneSoup(r.content)
- if soup.status.contents[0] == 'false':
- logger.debug(u'Could not find show %s' % series)
- return []
- request_id = int(soup.showid.contents[0])
- self.showids[series.lower()] = request_id
- self.save_cache()
+ request_id = self.get_show_id(series.lower())
+ if request_id is None:
+ return []
request_source = 'showid'
request_is_tvdbid = 'false'
elif tvdbid:
@@ -96,14 +77,14 @@ class BierDopje(ServiceBase):
raise ServiceError('One or more parameter missing')
subtitles = []
for language in languages:
- logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
- r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid))
+ logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language.alpha2))
+ r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language.alpha2, request_is_tvdbid))
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
- soup = BeautifulSoup.BeautifulStoneSoup(r.content)
+ soup = BeautifulSoup(r.content, self.required_features)
if soup.status.contents[0] == 'false':
- logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
+ logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language.alpha2))
continue
path = get_subtitle_path(filepath, language, self.config.multi)
for result in soup.results('result'):
@@ -112,11 +93,8 @@ class BierDopje(ServiceBase):
subtitles.append(subtitle)
return subtitles
- def list(self, video, languages):
- if not self.check_validity(video, languages):
- return []
- results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series)
- return results
+ def list_checked(self, video, languages):
+ return self.query(video.path or video.release, video.season, video.episode, languages, video.tvdbid, video.series)
Service = BierDopje
diff --git a/libs/subliminal/services/opensubtitles.py b/libs/subliminal/services/opensubtitles.py
index 9dee27b9..cf493805 100755
--- a/libs/subliminal/services/opensubtitles.py
+++ b/libs/subliminal/services/opensubtitles.py
@@ -18,8 +18,10 @@
from . import ServiceBase
from ..exceptions import ServiceError, DownloadFailedError
from ..subtitles import get_subtitle_path, ResultSubtitle
-from ..videos import Episode, Movie
from ..utils import to_unicode
+from ..videos import Episode, Movie
+from guessit.language import lang_set
+import guessit
import gzip
import logging
import os.path
@@ -32,34 +34,71 @@ logger = logging.getLogger(__name__)
class OpenSubtitles(ServiceBase):
server_url = 'http://api.opensubtitles.org/xml-rpc'
api_based = True
- languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara',
- 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze',
- 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis',
- 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che',
- 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze',
- 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
- 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful',
- 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
- 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin',
- 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii',
- 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav',
- 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm',
- 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
- 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit',
- 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao',
- 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau',
- 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob',
- 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss',
- 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que',
- 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin',
- 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
- 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe',
- 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
- 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur',
- 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie',
- 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor',
- 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'}
- reverted_languages = False
+ # language list fetched from:
+ # http://www.opensubtitles.org/addons/export_languages.php
+ languages = lang_set(['aar', 'abk', 'ace', 'ach', 'ada', 'ady', 'afa', 'afh',
+ 'afr', 'ain', 'aka', 'akk', 'alb', 'ale', 'alg', 'alt',
+ 'amh', 'ang', 'apa', 'ara', 'arc', 'arg', 'arm', 'arn',
+ 'arp', 'art', 'arw', 'asm', 'ast', 'ath', 'aus', 'ava',
+ 'ave', 'awa', 'aym', 'aze', 'bad', 'bai', 'bak', 'bal',
+ 'bam', 'ban', 'baq', 'bas', 'bat', 'bej', 'bel', 'bem',
+ 'ben', 'ber', 'bho', 'bih', 'bik', 'bin', 'bis', 'bla',
+ 'bnt', 'bod', 'bos', 'bra', 'bre', 'btk', 'bua', 'bug',
+ 'bul', 'bur', 'byn', 'cad', 'cai', 'car', 'cat', 'cau',
+ 'ceb', 'cel', 'cha', 'chb', 'che', 'chg', 'chi', 'chk',
+ 'chm', 'chn', 'cho', 'chp', 'chr', 'chu', 'chv', 'chy',
+ 'cmc', 'cop', 'cor', 'cos', 'cpe', 'cpf', 'cpp', 'cre',
+ 'crh', 'crp', 'csb', 'cus', 'cym', 'cze', 'dak', 'dan',
+ 'dar', 'day', 'del', 'den', 'deu', 'dgr', 'din', 'div',
+ 'doi', 'dra', 'dua', 'dum', 'dut', 'dyu', 'dzo', 'efi',
+ 'egy', 'eka', 'elx', 'eng', 'enm', 'epo', 'est', 'eus',
+ 'ewe', 'ewo', 'fan', 'fao', 'fas', 'fat', 'fij', 'fil',
+ 'fin', 'fiu', 'fon', 'fra', 'fre', 'frm', 'fro', 'fry',
+ 'ful', 'fur', 'gaa', 'gay', 'gba', 'gem', 'geo', 'ger',
+ 'gez', 'gil', 'gla', 'gle', 'glg', 'glv', 'gmh', 'goh',
+ 'gon', 'gor', 'got', 'grb', 'grc', 'ell', 'grn', 'guj',
+ 'gwi', 'hai', 'hat', 'hau', 'haw', 'heb', 'her', 'hil',
+ 'him', 'hin', 'hit', 'hmn', 'hmo', 'hrv', 'hun', 'hup',
+ 'hye', 'iba', 'ibo', 'ice', 'ido', 'iii', 'ijo', 'iku',
+ 'ile', 'ilo', 'ina', 'inc', 'ind', 'ine', 'inh', 'ipk',
+ 'ira', 'iro', 'isl', 'ita', 'jav', 'jpn', 'jpr', 'jrb',
+ 'kaa', 'kab', 'kac', 'kal', 'kam', 'kan', 'kar', 'kas',
+ 'kat', 'kau', 'kaw', 'kaz', 'kbd', 'kha', 'khi', 'khm',
+ 'kho', 'kik', 'kin', 'kir', 'kmb', 'kok', 'kom', 'kon',
+ 'kor', 'kos', 'kpe', 'krc', 'kro', 'kru', 'kua', 'kum',
+ 'kur', 'kut', 'lad', 'lah', 'lam', 'lao', 'lat', 'lav',
+ 'lez', 'lim', 'lin', 'lit', 'lol', 'loz', 'ltz', 'lua',
+ 'lub', 'lug', 'lui', 'lun', 'luo', 'lus', 'mac', 'mad',
+ 'mag', 'mah', 'mai', 'mak', 'mal', 'man', 'mao', 'map',
+ 'mar', 'mas', 'may', 'mdf', 'mdr', 'men', 'mga', 'mic',
+ 'min', 'mis', 'mkd', 'mkh', 'mlg', 'mlt', 'mnc', 'mni',
+ 'mno', 'moh', 'mol', 'mon', 'mos', 'mri', 'msa', 'mwl',
+ 'mul', 'mun', 'mus', 'mwr', 'mya', 'myn', 'myv', 'nah',
+ 'nai', 'nap', 'nau', 'nav', 'nbl', 'nde', 'ndo', 'nds',
+ 'nep', 'new', 'nia', 'nic', 'niu', 'nld', 'nno', 'nob',
+ 'nog', 'non', 'nor', 'nso', 'nub', 'nwc', 'nya', 'nym',
+ 'nyn', 'nyo', 'nzi', 'oci', 'oji', 'ori', 'orm', 'osa',
+ 'oss', 'ota', 'oto', 'paa', 'pag', 'pal', 'pam', 'pan',
+ 'pap', 'pau', 'peo', 'per', 'phi', 'phn', 'pli', 'pol',
+ 'pon', 'por', 'pra', 'pro', 'pus', 'que', 'raj', 'rap',
+ 'rar', 'roa', 'roh', 'rom', 'ron', 'run', 'rup', 'rus',
+ 'sad', 'sag', 'sah', 'sai', 'sal', 'sam', 'san', 'sas',
+ 'sat', 'scc', 'scn', 'sco', 'scr', 'sel', 'sem', 'sga',
+ 'sgn', 'shn', 'sid', 'sin', 'sio', 'sit', 'sla', 'slk',
+ 'slo', 'slv', 'sma', 'sme', 'smi', 'smj', 'smn', 'smo',
+ 'sms', 'sna', 'snd', 'snk', 'sog', 'som', 'son', 'sot',
+ 'spa', 'sqi', 'srd', 'srp', 'srr', 'ssa', 'ssw', 'suk',
+ 'sun', 'sus', 'sux', 'swa', 'swe', 'syr', 'tah', 'tai',
+ 'tam', 'tat', 'tel', 'tem', 'ter', 'tet', 'tgk', 'tgl',
+ 'tha', 'tib', 'tig', 'tir', 'tiv', 'tkl', 'tlh', 'tli',
+ 'tmh', 'tog', 'ton', 'tpi', 'tsi', 'tsn', 'tso', 'tuk',
+ 'tum', 'tup', 'tur', 'tut', 'tvl', 'twi', 'tyv', 'udm',
+ 'uga', 'uig', 'ukr', 'umb', 'und', 'urd', 'uzb', 'vai',
+ 'ven', 'vie', 'vol', 'vot', 'wak', 'wal', 'war', 'was',
+ 'wel', 'wen', 'wln', 'wol', 'xal', 'xho', 'yao', 'yap',
+ 'yid', 'yor', 'ypk', 'zap', 'zen', 'zha', 'zho', 'znd',
+ 'zul', 'zun', 'rum', 'pob', 'unk', 'ass'])
+
videos = [Episode, Movie]
require_video = False
confidence_order = ['moviehash', 'imdbid', 'fulltext']
@@ -92,7 +131,7 @@ class OpenSubtitles(ServiceBase):
if not searches:
raise ServiceError('One or more parameter missing')
for search in searches:
- search['sublanguageid'] = ','.join([self.get_language(l) for l in languages])
+ search['sublanguageid'] = ','.join(l.opensubtitles for l in languages)
logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
results = self.server.SearchSubtitles(self.token, searches)
if not results['data']:
@@ -100,7 +139,7 @@ class OpenSubtitles(ServiceBase):
return []
subtitles = []
for result in results['data']:
- language = self.get_revert_language(result['SubLanguageID'])
+ language = guessit.Language(result['SubLanguageID'])
path = get_subtitle_path(filepath, language, self.config.multi)
confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
@@ -108,9 +147,7 @@ class OpenSubtitles(ServiceBase):
subtitles.append(subtitle)
return subtitles
- def list(self, video, languages):
- if not self.check_validity(video, languages):
- return []
+ def list_checked(self, video, languages):
results = []
if video.exists:
results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))
diff --git a/libs/subliminal/services/podnapisi.py b/libs/subliminal/services/podnapisi.py
new file mode 100755
index 00000000..0e12fa00
--- /dev/null
+++ b/libs/subliminal/services/podnapisi.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..exceptions import ServiceError, DownloadFailedError
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..utils import to_unicode
+from ..videos import Episode, Movie
+from guessit.language import lang_set
+from hashlib import md5, sha256
+import guessit
+import logging
+import xmlrpclib
+
+
+logger = logging.getLogger(__name__)
+
+
+class Podnapisi(ServiceBase):
+ server_url = 'http://ssp.podnapisi.net:8000'
+ api_based = True
+ languages = lang_set(['sl', 'en', 'nn', 'ko', 'de', 'is', 'cs', 'fr', 'it', 'bs', 'jp', 'ar', 'ro',
+ 'hu', 'gr', 'zh', 'lt', 'et', 'lv', 'he', 'nl', 'da', 'sv', 'pl', 'ru', 'es',
+ 'sq', 'tr', 'fi', 'pt', 'bg', 'mk', 'sr', 'sk', 'hr', 'hi', 'th', 'ca', 'uk',
+ 'pb', 'ga', 'be', 'vi', 'fa', 'ca', 'id', 'ms'])
+ #FIXME: ag and cyr not recognized by guessit
+ videos = [Episode, Movie]
+ require_video = True
+
+ def __init__(self, config=None):
+ super(Podnapisi, self).__init__(config)
+ self.server = xmlrpclib.ServerProxy(self.server_url)
+ self.token = None
+
+ def init(self):
+ super(Podnapisi, self).init()
+ result = self.server.initiate(self.user_agent)
+ if result['status'] != 200:
+ raise ServiceError('Initiate failed')
+ username = 'python_subliminal'
+ password = sha256(md5('XWFXQ6gE5Oe12rv4qxXX').hexdigest() + result['nonce']).hexdigest()
+ self.token = result['session']
+ result = self.server.authenticate(self.token, username, password)
+ if result['status'] != 200:
+ raise ServiceError('Authenticate failed')
+
+ def terminate(self):
+ super(Podnapisi, self).terminate()
+
+ def query(self, filepath, languages, moviehash):
+ results = self.server.search(self.token, [moviehash])
+ if results['status'] != 200:
+ logger.error('Search failed with error code %d' % results['status'])
+ return []
+ if not results['results'] or not results['results'][moviehash]['subtitles']:
+ logger.debug(u'Could not find subtitles for %r with token %s' % (moviehash, self.token))
+ return []
+ subtitles = []
+ for result in results['results'][moviehash]['subtitles']:
+ language = guessit.Language(result['lang'])
+ if language == guessit.language.UNDETERMINED or language not in languages:
+ continue
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['id'],
+ release=to_unicode(result['release']), confidence=result['weight'])
+ subtitles.append(subtitle)
+ if not subtitles:
+ return []
+ # Convert weight to confidence
+ max_weight = float(max([s.confidence for s in subtitles]))
+ min_weight = float(min([s.confidence for s in subtitles]))
+ for subtitle in subtitles:
+ if max_weight == 0 and min_weight == 0:
+ subtitle.confidence = 1.0
+ else:
+ subtitle.confidence = (subtitle.confidence - min_weight) / (max_weight - min_weight)
+ return subtitles
+
+ def list_checked(self, video, languages):
+ results = self.query(video.path, languages, video.hashes['OpenSubtitles'])
+ return results
+
+ def download(self, subtitle):
+ results = self.server.download(self.token, [subtitle.link])
+ if results['status'] != 200:
+ raise DownloadFailedError()
+ subtitle.link = 'http://www.podnapisi.net/static/podnapisi/' + results['names'][0]['filename']
+ self.download_file(subtitle.link, subtitle.path)
+ return subtitle
+
+
+Service = Podnapisi
diff --git a/libs/subliminal/services/subswiki.py b/libs/subliminal/services/subswiki.py
index c5670c1c..6671a8de 100755
--- a/libs/subliminal/services/subswiki.py
+++ b/libs/subliminal/services/subswiki.py
@@ -19,8 +19,10 @@ from . import ServiceBase
from ..exceptions import ServiceError
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie
+from bs4 import BeautifulSoup
+from guessit.language import lang_set
from subliminal.utils import get_keywords, split_keyword
-import BeautifulSoup
+import guessit
import logging
import re
import urllib
@@ -32,17 +34,15 @@ logger = logging.getLogger(__name__)
class SubsWiki(ServiceBase):
server_url = 'http://www.subswiki.com'
api_based = False
- languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
- u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
- u'Italian': 'it', u'Català': 'ca'}
- reverted_languages = True
+ languages = lang_set([u'English (US)', u'English (UK)', u'English', u'French', u'Brazilian',
+ u'Portuguese', u'Español (Latinoamérica)', u'Español (España)',
+ u'Español', u'Italian', u'Català'], strict=True)
videos = [Episode, Movie]
require_video = False
release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
+ required_features = ['permissive']
- def list(self, video, languages):
- if not self.check_validity(video, languages):
- return []
+ def list_checked(self, video, languages):
results = []
if isinstance(video, Episode):
results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
@@ -74,7 +74,7 @@ class SubsWiki(ServiceBase):
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
- soup = BeautifulSoup.BeautifulSoup(r.content)
+ soup = BeautifulSoup(r.content, self.required_features)
subtitles = []
for sub in soup('td', {'class': 'NewsTitle'}):
sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
@@ -82,8 +82,8 @@ class SubsWiki(ServiceBase):
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
- language = self.get_revert_language(html_language.string.strip())
- if not language in languages:
+ language = guessit.Language(html_language.string.strip())
+ if language not in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
html_status = html_language.findNextSibling('td')
@@ -96,4 +96,5 @@ class SubsWiki(ServiceBase):
subtitles.append(subtitle)
return subtitles
+
Service = SubsWiki
diff --git a/libs/subliminal/services/subtitulos.py b/libs/subliminal/services/subtitulos.py
index 44888e70..8154aaaf 100755
--- a/libs/subliminal/services/subtitulos.py
+++ b/libs/subliminal/services/subtitulos.py
@@ -18,8 +18,10 @@
from . import ServiceBase
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode
+from bs4 import BeautifulSoup
+from guessit.language import lang_set
from subliminal.utils import get_keywords, split_keyword
-import BeautifulSoup
+import guessit
import logging
import re
import unicodedata
@@ -32,19 +34,19 @@ logger = logging.getLogger(__name__)
class Subtitulos(ServiceBase):
server_url = 'http://www.subtitulos.es'
api_based = False
- languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
- u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
- u'Italian': 'it', u'Català': 'ca'}
- reverted_languages = True
+ languages = lang_set([u'English (US)', u'English (UK)', u'English', u'French', u'Brazilian',
+ u'Portuguese', u'Español (Latinoamérica)', u'Español (España)', u'Español',
+ u'Italian', u'Català'], strict=True)
videos = [Episode]
require_video = False
- release_pattern = re.compile('Versión (.+) ([0-9]+).([0-9])+ megabytes')
+ required_features = ['permissive']
+ # the '.+' in the pattern for Version allows us to match both 'ó'
+ # and the 'ó' char directly. This is because now BS4 converts the html
+ # code chars into their equivalent unicode char
+ release_pattern = re.compile('Versi.+n (.+) ([0-9]+).([0-9])+ megabytes')
- def list(self, video, languages):
- if not self.check_validity(video, languages):
- return []
- results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
- return results
+ def list_checked(self, video, languages):
+ return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
def query(self, filepath, languages, keywords, series, season, episode):
request_series = series.lower().replace(' ', '_')
@@ -58,7 +60,7 @@ class Subtitulos(ServiceBase):
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
- soup = BeautifulSoup.BeautifulSoup(r.content)
+ soup = BeautifulSoup(r.content, self.required_features)
subtitles = []
for sub in soup('div', {'id': 'version'}):
sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
@@ -66,8 +68,8 @@ class Subtitulos(ServiceBase):
logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
continue
for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
- language = self.get_revert_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
- if not language in languages:
+ language = guessit.Language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
+ if language not in languages:
logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
continue
html_status = html_language.findNext('li', {'class': 'li-estado green'})
@@ -80,4 +82,5 @@ class Subtitulos(ServiceBase):
subtitles.append(subtitle)
return subtitles
+
Service = Subtitulos
diff --git a/libs/subliminal/services/thesubdb.py b/libs/subliminal/services/thesubdb.py
index cccddd40..1800dcf7 100755
--- a/libs/subliminal/services/thesubdb.py
+++ b/libs/subliminal/services/thesubdb.py
@@ -18,6 +18,8 @@
from . import ServiceBase
from ..subtitles import get_subtitle_path, ResultSubtitle
from ..videos import Episode, Movie, UnknownVideo
+from guessit.language import lang_set
+import guessit
import logging
@@ -26,21 +28,17 @@ logger = logging.getLogger(__name__)
class TheSubDB(ServiceBase):
server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
- user_agent = 'SubDB/1.0 (subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API
+ user_agent = 'SubDB/1.0 (subliminal/0.6; https://github.com/Diaoul/subliminal)' # defined by the API
api_based = True
- languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
- 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
- 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
- 'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages
- reverted_languages = False
+ languages = lang_set(['af', 'cs', 'da', 'de', 'en', 'es', 'fi',
+ 'fr', 'hu', 'id', 'it', 'la', 'nl', 'no',
+ 'oc', 'pl', 'pt', 'ro', 'ru', 'sl', 'sr',
+ 'sv', 'tr'], strict=True) # list available with the API at http://sandbox.thesubdb.com/?action=languages
videos = [Movie, Episode, UnknownVideo]
require_video = True
- def list(self, video, languages):
- if not self.check_validity(video, languages):
- return []
- results = self.query(video.path, video.hashes['TheSubDB'], languages)
- return results
+ def list_checked(self, video, languages):
+ return self.query(video.path, video.hashes['TheSubDB'], languages)
def query(self, filepath, moviehash, languages):
r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
@@ -50,7 +48,7 @@ class TheSubDB(ServiceBase):
if r.status_code != 200:
logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
return []
- available_languages = set([self.get_revert_language(l) for l in r.content.split(',')])
+ available_languages = set(guessit.Language(l) for l in r.content.split(','))
languages &= available_languages
if not languages:
logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
@@ -58,8 +56,9 @@ class TheSubDB(ServiceBase):
subtitles = []
for language in languages:
path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.get_language(language)))
+ subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(), '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, language.alpha2))
subtitles.append(subtitle)
return subtitles
+
Service = TheSubDB
diff --git a/libs/subliminal/services/tvsubtitles.py b/libs/subliminal/services/tvsubtitles.py
new file mode 100755
index 00000000..5c74cc1e
--- /dev/null
+++ b/libs/subliminal/services/tvsubtitles.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Nicolas Wack
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..cache import cachedmethod
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode
+from bs4 import BeautifulSoup
+from guessit.language import lang_set
+from subliminal.utils import get_keywords
+import guessit
+import logging
+import re
+
+
+logger = logging.getLogger(__name__)
+
+
+def match(pattern, string):
+ try:
+ return re.search(pattern, string).group(1)
+ except AttributeError:
+ logger.debug(u'Could not match %r on %r' % (pattern, string))
+ return None
+
+
+class TvSubtitles(ServiceBase):
+ server_url = 'http://www.tvsubtitles.net'
+ api_based = False
+ languages = lang_set([u'English', u'Español', u'French', u'German',
+ u'Brazilian', u'Russian', u'Ukrainian', u'Italian',
+ u'Greek', u'Arabic', u'Hungarian', u'Polish',
+ u'Turkish', u'Dutch', u'Portuguese', u'Swedish',
+ u'Danish', u'Finnish', u'Korean', u'Chinese',
+ u'Japanese', u'Bulgarian', u'Czech', u'Romanian'], strict=True)
+ videos = [Episode]
+ require_video = False
+ required_features = ['permissive']
+
+ @cachedmethod
+ def get_likely_series_id(self, name):
+ r = self.session.post('%s/search.php' % self.server_url, data={'q': name})
+ soup = BeautifulSoup(r.content, self.required_features)
+ maindiv = soup.find('div', 'left')
+ results = []
+ for elem in maindiv.find_all('li'):
+ sid = int(match('tvshow-([0-9]+)\.html', elem.a['href']))
+ show_name = match('(.*) \(', elem.a.text)
+ results.append((show_name, sid))
+ #TODO: pick up the best one in a smart way
+ result = results[0]
+ return result[1]
+
+ @cachedmethod
+ def get_episode_id(self, series_id, season, number):
+ """Get the TvSubtitles id for the given episode. Raises KeyError if none
+ could be found."""
+ # download the page of the season, contains ids for all episodes
+ episode_id = None
+ r = self.session.get('%s/tvshow-%d-%d.html' % (self.server_url, series_id, season))
+ soup = BeautifulSoup(r.content, self.required_features)
+ table = soup.find('table', id='table5')
+ for row in table.find_all('tr'):
+ cells = row.find_all('td')
+ if not cells:
+ continue
+
+ episode_number = match('x([0-9]+)', cells[0].text)
+ if not episode_number:
+ continue
+
+ episode_number = int(episode_number)
+ episode_id = int(match('episode-([0-9]+)', cells[1].a['href']))
+ # we could just return the id of the queried episode, but as we
+ # already downloaded the whole page we might as well fill in the
+ # information for all the episodes of the season
+ self.cache_for(self.get_episode_id, args=(series_id, season, episode_number), result=episode_id)
+ # raises KeyError if not found
+ return self.cached_value(self.get_episode_id, args=(series_id, season, number))
+
+ # Do not cache this method in order to always check for the most recent
+ # subtitles
+ def get_sub_ids(self, episode_id):
+ subids = []
+ r = self.session.get('%s/episode-%d.html' % (self.server_url, episode_id))
+ epsoup = BeautifulSoup(r.content, self.required_features)
+ for subdiv in epsoup.find_all('a'):
+ if 'href' not in subdiv.attrs or not subdiv['href'].startswith('/subtitle'):
+ continue
+ subid = int(match('([0-9]+)', subdiv['href']))
+ lang = guessit.Language(match('flags/(.*).gif', subdiv.img['src']))
+ result = {'subid': subid, 'language': lang}
+ for p in subdiv.find_all('p'):
+ if 'alt' in p.attrs and p['alt'] == 'rip':
+ result['rip'] = p.text.strip()
+ if 'alt' in p.attrs and p['alt'] == 'release':
+ result['release'] = p.text.strip()
+
+ subids.append(result)
+ return subids
+
+ def list_checked(self, video, languages):
+ return self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
+
+ def query(self, filepath, languages, keywords, series, season, episode):
+ logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+ self.init_cache()
+ sid = self.get_likely_series_id(series.lower())
+ try:
+ ep_id = self.get_episode_id(sid, season, episode)
+ except KeyError:
+ logger.debug(u'Could not find episode id for %s season %d episode %d' % (series, season, episode))
+ return []
+ subids = self.get_sub_ids(ep_id)
+ # filter the subtitles with our queried languages
+ subtitles = []
+ for subid in subids:
+ language = subid['language']
+ if language not in languages:
+ continue
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ subtitle = ResultSubtitle(path, language, self.__class__.__name__.lower(),
+ '%s/download-%d.html' % (self.server_url, subid['subid']),
+ keywords=[subid['rip'], subid['release']])
+ subtitles.append(subtitle)
+ return subtitles
+
+ def download(self, subtitle):
+ self.download_zip_file(subtitle.link, subtitle.path)
+
+
+Service = TvSubtitles
diff --git a/libs/subliminal/subtitles.py b/libs/subliminal/subtitles.py
index 355046dc..a33c35c3 100755
--- a/libs/subliminal/subtitles.py
+++ b/libs/subliminal/subtitles.py
@@ -15,13 +15,13 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
-from .languages import list_languages, convert_language
import os.path
+import guessit
+from guessit.language import is_language
__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
-
#: Subtitles extensions
EXTENSIONS = ['.srt', '.sub', '.txt']
@@ -30,7 +30,8 @@ class Subtitle(object):
"""Base class for subtitles
:param string path: path to the subtitle
- :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+ :param language: language of the subtitle
+ :type language: :class:`guessit.Language`
"""
def __init__(self, path, language):
@@ -49,7 +50,8 @@ class EmbeddedSubtitle(Subtitle):
"""Subtitle embedded in a container
:param string path: path to the subtitle
- :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+ :param language: language of the subtitle
+ :type language: :class:`guessit.Language`
:param int track_id: id of the subtitle track in the container
"""
@@ -59,7 +61,7 @@ class EmbeddedSubtitle(Subtitle):
@classmethod
def from_enzyme(cls, path, subtitle):
- language = convert_language(subtitle.language, 1, 2)
+ language = guessit.Language(subtitle.language) or None
return cls(path, language, subtitle.trackno)
@@ -76,8 +78,8 @@ class ExternalSubtitle(Subtitle):
if not extension:
raise ValueError('Not a supported subtitle extension')
language = os.path.splitext(path[:len(path) - len(extension)])[1][1:]
- if not language in list_languages(1):
- language = None
+ language = guessit.Language(language) or None
+
return cls(path, language)
@@ -85,7 +87,8 @@ class ResultSubtitle(ExternalSubtitle):
"""Subtitle found using :mod:`~subliminal.services`
:param string path: path to the subtitle
- :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+ :param language: language of the subtitle
+ :type language: :class:`guessit.Language`
:param string service: name of the service
:param string link: download link for the subtitle
:param string release: release name of the video
@@ -111,20 +114,27 @@ class ResultSubtitle(ExternalSubtitle):
"""
extension = os.path.splitext(self.path)[0]
language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
- if not language in list_languages(1):
- return True
- return False
+ return not is_language(language)
def __repr__(self):
return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
def get_subtitle_path(video_path, language, multi):
- """Create the subtitle path from the given video path using language if multi"""
+ """Create the subtitle path from the given video path using language if multi
+
+ :param string video_path: path to the video
+ :param language: language of the subtitle
+ :type language: :class:`guessit.Language`
+ :param bool multi: whether to use multi language naming or not
+ :return: path of the subtitle
+ :rtype: string
+
+ """
if not os.path.exists(video_path):
path = os.path.splitext(os.path.basename(video_path))[0]
else:
path = os.path.splitext(video_path)[0]
if multi and language:
- return path + '.%s%s' % (language, EXTENSIONS[0])
+ return path + '.%s%s' % (language.alpha2, EXTENSIONS[0])
return path + '%s' % EXTENSIONS[0]
diff --git a/libs/subliminal/videos.py b/libs/subliminal/videos.py
index 79276b14..adfcafe7 100755
--- a/libs/subliminal/videos.py
+++ b/libs/subliminal/videos.py
@@ -16,7 +16,6 @@
# You should have received a copy of the GNU Lesser General Public License
# along with subliminal. If not, see .
from . import subtitles
-from .languages import list_languages
import enzyme
import guessit
import hashlib
@@ -130,14 +129,23 @@ class Video(object):
logger.debug(u'Failed parsing %s with enzyme' % self.path)
if isinstance(video_infos, enzyme.core.AVContainer):
results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
- for l in list_languages(1):
- for e in subtitles.EXTENSIONS:
- single_path = basepath + '%s' % e
- if os.path.exists(single_path):
- results.append(subtitles.ExternalSubtitle(single_path, None))
- multi_path = basepath + '.%s%s' % (l, e)
- if os.path.exists(multi_path):
- results.append(subtitles.ExternalSubtitle(multi_path, l))
+
+ # cannot use glob here because it chokes if there are any square
+ # brackets inside the filename, so we have to use basic string
+ # startswith/endswith comparisons
+ folder, basename = os.path.split(basepath)
+ existing = [f for f in os.listdir(folder) if f.startswith(basename)]
+ for path in existing:
+ for ext in subtitles.EXTENSIONS:
+ if path.endswith(ext):
+ possible_lang = path[len(basename) + 1:-len(ext)]
+ if possible_lang == '':
+ results.append(subtitles.ExternalSubtitle(path, None))
+ else:
+ lang = guessit.Language(possible_lang)
+ if lang:
+ results.append(subtitles.ExternalSubtitle(path, lang))
+
return results
def __repr__(self):
@@ -189,11 +197,12 @@ class UnknownVideo(Video):
pass
-def scan(entry, max_depth=3, depth=0):
+def scan(entry, max_depth=3, scan_filter=None, depth=0):
"""Scan a path for videos and subtitles
:param string entry: path
:param int max_depth: maximum folder depth
+ :param function scan_filter: filter function that takes a path as argument and returns a boolean indicating whether it has to be filtered out (``True``) or not (``False``)
:param int depth: starting depth
:return: found videos and subtitles
:rtype: list of (:class:`Video`, [:class:`~subliminal.subtitles.Subtitle`])
@@ -207,13 +216,15 @@ def scan(entry, max_depth=3, depth=0):
logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
result = []
for e in os.listdir(entry):
- result.extend(scan(os.path.join(entry, e), max_depth, depth + 1))
+ result.extend(scan(os.path.join(entry, e), max_depth, scan_filter, depth + 1))
return result
if os.path.isfile(entry) or depth == 0:
logger.debug(u'Scanning file %s with depth %d/%d' % (entry, depth, max_depth))
if depth != 0: # trust the user: only check for valid format if recursing
if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
return []
+ if scan_filter is not None and scan_filter(entry):
+ return []
video = Video.from_path(entry)
return [(video, video.scan())]
logger.warning(u'Scanning entry %s failed with depth %d/%d' % (entry, depth, max_depth))