from __future__ import absolute_import, division, unicode_literals import os import json import pytest from .support import get_data_files from html5lib import constants from html5lib.filters.lint import Filter as Lint from html5lib.serializer import HTMLSerializer, serialize from html5lib.treewalkers.base import TreeWalker # pylint:disable=wrong-import-position optionals_loaded = [] try: from lxml import etree optionals_loaded.append("lxml") except ImportError: pass # pylint:enable=wrong-import-position default_namespace = constants.namespaces["html"] class JsonWalker(TreeWalker): def __iter__(self): for token in self.tree: type = token[0] if type == "StartTag": if len(token) == 4: namespace, name, attrib = token[1:4] else: namespace = default_namespace name, attrib = token[1:3] yield self.startTag(namespace, name, self._convertAttrib(attrib)) elif type == "EndTag": if len(token) == 3: namespace, name = token[1:3] else: namespace = default_namespace name = token[1] yield self.endTag(namespace, name) elif type == "EmptyTag": if len(token) == 4: namespace, name, attrib = token[1:] else: namespace = default_namespace name, attrib = token[1:] for token in self.emptyTag(namespace, name, self._convertAttrib(attrib)): yield token elif type == "Comment": yield self.comment(token[1]) elif type in ("Characters", "SpaceCharacters"): for token in self.text(token[1]): yield token elif type == "Doctype": if len(token) == 4: yield self.doctype(token[1], token[2], token[3]) elif len(token) == 3: yield self.doctype(token[1], token[2]) else: yield self.doctype(token[1]) else: raise ValueError("Unknown token type: " + type) def _convertAttrib(self, attribs): """html5lib tree-walkers use a dict of (namespace, name): value for attributes, but JSON cannot represent this. Convert from the format in the serializer tests (a list of dicts with "namespace", "name", and "value" as keys) to html5lib's tree-walker format.""" attrs = {} for attrib in attribs: name = (attrib["namespace"], attrib["name"]) assert(name not in attrs) attrs[name] = attrib["value"] return attrs def serialize_html(input, options): options = {str(k): v for k, v in options.items()} encoding = options.get("encoding", None) if "encoding" in options: del options["encoding"] stream = Lint(JsonWalker(input), False) serializer = HTMLSerializer(alphabetical_attributes=True, **options) return serializer.render(stream, encoding) def throwsWithLatin1(input): with pytest.raises(UnicodeEncodeError): serialize_html(input, {"encoding": "iso-8859-1"}) def testDoctypeName(): throwsWithLatin1([["Doctype", "\u0101"]]) def testDoctypePublicId(): throwsWithLatin1([["Doctype", "potato", "\u0101"]]) def testDoctypeSystemId(): throwsWithLatin1([["Doctype", "potato", "potato", "\u0101"]]) def testCdataCharacters(): test_serializer([["StartTag", "http://www.w3.org/1999/xhtml", "style", {}], ["Characters", "\u0101"]], ["