Skip to content

Allow Elements to be passed to parse_*() #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
levic wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Allow Elements to be passed to parse_*() #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c4a4d96
Fix minor typo
levic Feb 6, 2023
266ed59
Add test cases for passing an etree Element as an extractor
levic Feb 6, 2023
e4e38e6
Allow etree Elements to be passed as extractors to parse_*()
levic Feb 6, 2023
edb6dbf
Raise an error if the user passes an etree Element to parse() [they s…
levic Feb 6, 2023
bed4af0
Slight simplification of test case
levic Feb 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions tests/test_parsers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
from urllib.parse import urlparse
import copy
import unittest
from unittest.mock import patch

from lxml import etree

from xextract.extractors import HtmlXPathExtractor, XmlXPathExtractor
from xextract.parsers import (
ParserError, ParsingError, BaseParser, BaseNamedParser,
Prefix, Group, Element, String, Url, DateTime, Date)
Expand Down Expand Up @@ -276,6 +278,135 @@ def test_text_extract(self):
self.assertEqual(val, 'nice')


class TestElementAsExtractor(unittest.TestCase):
def test_element_as_parser(self):
"""
we can pass an Element as the extractor to parse_*()
"""
html = '''
<div><span>Hello world!</span></div>
<div></div>
<div><span>Hello mars!</span></div>
'''

# take only the first containers so we can verify that the correct descendant is chosen
container = Element(css='div', count=3).parse(html)[2]

val = Element(css='span', count=1).parse_html(container)
self.assertEqual(val.tag, 'span')
self.assertEqual(val.text, 'Hello mars!')

def test_element_as_parser_html(self):
"""
passing Element as extractor for parse_html should use the correct parser
"""

html = '''
<html>
<div>
<script>console.log(" &lt; > ");</script>
<span>a&lt;a&lsquo;</span>
</div>
</html>
'''

original_tostring = etree.tostring
original_parser = HtmlXPathExtractor._parser

# etree.HtmlParser is immutable so we can't patch it
# instead we patch HtmlXPathExtractor._parser which references it
with patch.object(HtmlXPathExtractor, attribute='_parser', autospec=True) as mock_parser:
mock_parser.side_effect = original_parser
with patch('lxml.etree.tostring', autospec=True) as mock_tostring:
mock_tostring.side_effect = original_tostring

ancestor = Element(css='div', count=1).parse_html(html)
mock_parser.assert_called_once()
mock_parser.reset_mock()
mock_tostring.assert_not_called()

script_val = String(css='script', count=1).parse_html(ancestor)
mock_parser.assert_not_called()
mock_tostring.assert_called_once()
self.assertEqual(mock_tostring.call_args.kwargs['method'], 'html')
mock_tostring.reset_mock()

span_val = String(css='span', count=1).parse_html(ancestor)
mock_parser.assert_not_called()
mock_tostring.assert_called_once()
self.assertEqual(mock_tostring.call_args.kwargs['method'], 'html')
mock_tostring.reset_mock()

# HTML has different entity processing from XML:
# <script> contents are automatically CDATA
# &lt; is an entity only inside <span>
# &lsquo; is a valid entity
self.assertEqual(script_val, 'console.log(" &lt; > ");')
self.assertEqual(span_val, 'a<a‘')

with self.assertRaises(ParserError):
# .parse() is missing the html/xml specifier
String(css='script', count=1).parse(ancestor)

def test_element_as_parser_xml(self):
"""
passing Element as extractor for parse_xml should use the correct parser
"""

xml = '''
<?xml version="1.0" encoding="UTF-8"?>
<body xmlns="http://test.com/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<ul>
<xsi:li>
<script>console.log(" &lt; > ");</script>
<span>a&lt;a&lsquo;</span>
</xsi:li>
</ul>
</body>
'''
namespaces = {'a': 'http://test.com/', 'b': 'http://www.w3.org/2001/XMLSchema-instance'}

original_tostring = etree.tostring
original_parser = XmlXPathExtractor._parser

# etree.XMLParser is immutable so we can't patch it
# instead we patch XmlXPathExtractor._parser which references it
with patch.object(XmlXPathExtractor, attribute='_parser', autospec=True) as mock_parser:

mock_parser.side_effect = original_parser
with patch('lxml.etree.tostring', autospec=True) as mock_tostring:
mock_tostring.side_effect = original_tostring

ancestor = Element(xpath='//a:body', count=1, namespaces=namespaces).parse_xml(xml)
mock_parser.assert_called_once()
mock_parser.reset_mock()
mock_tostring.assert_not_called()

script_val = String(xpath='//a:script', count=1, namespaces=namespaces).parse_xml(ancestor)
mock_parser.assert_not_called()
mock_tostring.assert_called_once()
self.assertEqual(mock_tostring.call_args.kwargs['method'], 'xml')
mock_tostring.reset_mock()

span_val = String(xpath='//a:span', count=1, namespaces=namespaces).parse_xml(ancestor)
mock_parser.assert_not_called()
mock_tostring.assert_called_once()
self.assertEqual(mock_tostring.call_args.kwargs['method'], 'xml')
mock_tostring.reset_mock()


# XML has different entity processing from HTML:
# <script> contents are not automatically CDATA
# &lt; is a valid entity inside both <span> and <script>
# &lsquo; is not a valid entity (with libxml it also happens to break subsequent entity parsing)
self.assertEqual(script_val, 'console.log(" < > ");')
self.assertEqual(span_val, 'a<a')

with self.assertRaises(ParserError):
# .parse() is missing the html/xml specifier
String(xpath='//a:script', count=1, namespaces=namespaces).parse(ancestor)


class TestGroup(TestBaseNamedParser):
parser_class = Group
parser_kwargs = {'name': 'val', 'children': []}
Expand Down
3 changes: 3 additions & 0 deletions xextract/extractors/lxml_extractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ def __init__(self, body=None, namespaces=None, _root=None):
self._root = _root

def _get_root(self, body, encoding=None):
if isinstance(body, etree._Element):
return body

body = body.strip() or self._empty_doc
if isinstance(body, str):
body = body.encode('utf-8')
Expand Down
5 changes: 4 additions & 1 deletion xextract/parsers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class ParserError(Exception):


class ParsingError(Exception):
'''Numebr of parsed elements doesn't match the expected quantity.'''
'''Number of parsed elements doesn't match the expected quantity.'''


class BaseParser(object):
Expand All @@ -39,6 +39,9 @@ def __call__(self, body, url=None):
return self.parse(body, url)

def parse(self, body, url=None):
if isinstance(body, etree._Element):
raise ParserError('You must use parse_html() or parse_xml() if passing an Element as the extractor')

if isinstance(body, XPathExtractor):
extractor = body
else:
Expand Down