Ensure HTML is parsed as UTF-8

refack · refack · commit fe934342f90b · 2025-09-23T18:20:22.000-04:00
Fixes: #44
diff --git a/commands/preprocess.py b/commands/preprocess.py
@@ -386,7 +386,7 @@ def remove_unused_external(html):
 
 
 def preprocess_html_file(root, fn, rename_map):
-    parser = etree.HTMLParser()
+    parser = etree.HTMLParser(encoding='utf-8')
     html = etree.parse(fn, parser)
     output = io.StringIO()
 
diff --git a/commands/preprocess_cssless.py b/commands/preprocess_cssless.py
@@ -31,7 +31,7 @@
 def preprocess_html_merge_cssless(src_path, dst_path):
     with open(src_path, 'r') as a_file:
         content = a_file.read()
-        parser = etree.HTMLParser()
+        parser = etree.HTMLParser(encoding='utf-8')
         stripped = content.strip()
         root = etree.fromstring(stripped, parser)
 
diff --git a/index2ddg.py b/index2ddg.py
@@ -582,7 +582,7 @@ def main():
         # i+=1
 
         root = e.parse(os.path.join(args.reference, fn),
-                       parser=html.HTMLParser())
+                       parser=html.HTMLParser(encoding='utf-8'))
 
         for ident in idents:
 
diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py
@@ -258,7 +258,7 @@ def setUp(self):
         self.testdata = os.path.join(os.path.dirname(__file__),
                                      'preprocess_data')
         infile = os.path.join(self.testdata, "fabs.html")
-        self.parser = etree.HTMLParser()
+        self.parser = etree.HTMLParser(encoding='utf-8')
         self.html = etree.parse(infile, self.parser)
 
     # Check whether the HTML matches the contents of the specified test data
diff --git a/tests/test_preprocess_cssless.py b/tests/test_preprocess_cssless.py
@@ -83,7 +83,7 @@ def assert_converts_html(self, input, expected_output, function):
         expected_output = \
             '<html><body>{0}</body></html>'.format(expected_output)
 
-        parser = etree.HTMLParser()
+        parser = etree.HTMLParser(encoding='utf-8')
         root = etree.fromstring(input, parser)
 
         root = function(root)
