Make EncodingTester usable in testing parsed state

This change updates EncodingTester to make it test the result for cases
when the expected character encoding is not limited to what can be
determined by checking only the first 1024 bytes of the input stream.

Otherwise, without this change, EncodingTester is limited to only being
useful for testing the output of the meta prescan.

This change also allows EncodingTester to be given a directory name
rather than a list of files (or pathname with a shell wildcard). And
when given a directory name, it recurses the directory looking for *.dat
files, and then run the tests from those files.

Without that change, we can’t easily run EncodingTester from AntRun in
Maven — because we can’t use shell wildcards in the “arg” value for the
Ant “java” task, and any list of files we otherwise construct within
Maven ends up getting put into the java arg value as a single string
(single argument) — including the spaces between filenames.

Author: sideshowbarker

test-src/nu/validator/htmlparser/test/EncodingTester.java

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2007 Henri Sivonen
- * Copyright (c) 2008 Mozilla Foundation
+ * Copyright (c) 2008-2020 Mozilla Foundation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a 
  * copy of this software and associated documentation files (the "Software"), 
@@ -23,6 +23,7 @@
 
 package nu.validator.htmlparser.test;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -36,6 +37,8 @@
 
 public class EncodingTester {
 
+    protected static int SNIFFING_LIMIT = 16384;
+
     private final InputStream aggregateStream;
 
     private final StringBuilder builder = new StringBuilder();
@@ -47,6 +50,13 @@ public EncodingTester(InputStream aggregateStream) {
         this.aggregateStream = aggregateStream;
     }
 
+    /**
+     * @param aggregateStream
+     */
+    public EncodingTester() {
+        this.aggregateStream = null;
+    }
+
     private void runTests() throws IOException, SAXException {
         while (runTest()) {
             // spin
@@ -59,7 +69,7 @@ private boolean runTest() throws IOException, SAXException {
         }
         UntilHashInputStream stream = new UntilHashInputStream(aggregateStream);
         HtmlInputStreamReader reader = new HtmlInputStreamReader(stream, null,
-                null, null, Heuristics.NONE);
+                null, null, Heuristics.NONE, SNIFFING_LIMIT);
         Charset charset = reader.getCharset();
         stream.close();
         if (skipLabel()) {
@@ -107,16 +117,45 @@ private boolean skipLabel() throws IOException {
         }
     }
 
+    private void recurseDirectory(File directory) throws Throwable {
+        if ("scripted".equals(directory.getName())) {
+            return;
+        }
+        if (directory.canRead()) {
+            File[] files = directory.listFiles();
+            for (File file : files) {
+                if (file.isDirectory()) {
+                    recurseDirectory(file);
+                } else {
+                    if (!file.getName().endsWith(".dat")) {
+                        continue;
+                    }
+                    EncodingTester tester = new EncodingTester(
+                            new FileInputStream(file.getPath().toString()));
+                    tester.runTests();
+                }
+            }
+        }
+    }
+
     /**
      * @param args
-     * @throws SAXException
-     * @throws IOException
+     * @throws Throwable
      */
-    public static void main(String[] args) throws IOException, SAXException {
+    public static void main(String[] args) throws Throwable {
         for (int i = 0; i < args.length; i++) {
-            EncodingTester tester = new EncodingTester(new FileInputStream(
-                    args[i]));
-            tester.runTests();
+            File file = new File(args[i]);
+            if (file.isDirectory()) {
+                EncodingTester tester = new EncodingTester();
+                tester.recurseDirectory(file);
+            } else {
+                if (!file.getName().endsWith(".dat")) {
+                    return;
+                }
+                EncodingTester tester = new EncodingTester(
+                        new FileInputStream(file.getPath().toString()));
+                tester.runTests();
+            }
         }
     }