Add DNS resolver abstraction and mock for testable hostname resolution

src/main/java/de/rub/nds/crawler/data/ScanTarget.java

@@ -10,8 +10,9 @@
 
 import de.rub.nds.crawler.constant.JobStatus;
 import de.rub.nds.crawler.denylist.IDenylistProvider;
+import de.rub.nds.crawler.dns.DefaultDnsResolver;
+import de.rub.nds.crawler.dns.DnsResolver;
 import java.io.Serializable;
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.validator.routines.InetAddressValidator;
@@ -36,6 +37,25 @@ public class ScanTarget implements Serializable {
      */
     public static Pair<ScanTarget, JobStatus> fromTargetString(
             String targetString, int defaultPort, IDenylistProvider denylistProvider) {
+        return fromTargetString(
+                targetString, defaultPort, denylistProvider, new DefaultDnsResolver());
+    }
+
+    /**
+     * Initializes a ScanTarget object from a string that potentially contains a hostname, an ip, a
+     * port, the tranco rank.
+     *
+     * @param targetString from which to create the ScanTarget object
+     * @param defaultPort that used if no port is present in targetString
+     * @param denylistProvider which provides info if a host is denylisted
+     * @param dnsResolver the DNS resolver to use for hostname resolution
+     * @return ScanTarget object
+     */
+    public static Pair<ScanTarget, JobStatus> fromTargetString(
+            String targetString,
+            int defaultPort,
+            IDenylistProvider denylistProvider,
+            DnsResolver dnsResolver) {
         ScanTarget target = new ScanTarget();
 
         // check if targetString contains rank (e.g. "1,example.com")
@@ -55,8 +75,7 @@ public static Pair<ScanTarget, JobStatus> fromTargetString(
                     targetString = parts[1];
                     if (targetString.trim().isEmpty()) {
                         try {
-                            target.setIp(
-                                    InetAddress.getByName(target.getHostname()).getHostAddress());
+                            target.setIp(dnsResolver.resolveHostname(target.getHostname()));
                         } catch (UnknownHostException e) {
                             return Pair.of(target, JobStatus.UNRESOLVABLE);
                         }
@@ -124,7 +143,7 @@ public static Pair<ScanTarget, JobStatus> fromTargetString(
             try {
                 // TODO this only allows one IP per hostname; it may be interesting to scan all IPs
                 // for a domain, or at least one v4 and one v6
-                target.setIp(InetAddress.getByName(targetString).getHostAddress());
+                target.setIp(dnsResolver.resolveHostname(targetString));
             } catch (UnknownHostException e) {
                 LOGGER.error(
                         "Host {} is unknown or can not be reached with error {}.", targetString, e);

src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java

@@ -0,0 +1,26 @@
+/*
+ * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner
+ *
+ * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH
+ *
+ * Licensed under Apache License, Version 2.0
+ * http://www.apache.org/licenses/LICENSE-2.0.txt
+ */
+package de.rub.nds.crawler.dns;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Default DNS resolver implementation that uses the standard Java DNS resolution via InetAddress.
+ */
+public class DefaultDnsResolver implements DnsResolver {
+
+    /** Creates a new default DNS resolver instance. */
+    public DefaultDnsResolver() {}
+
+    @Override
+    public String resolveHostname(String hostname) throws UnknownHostException {
+        return InetAddress.getByName(hostname).getHostAddress();
+    }
+}

src/main/java/de/rub/nds/crawler/dns/DnsResolver.java

@@ -0,0 +1,23 @@
+/*
+ * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner
+ *
+ * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH
+ *
+ * Licensed under Apache License, Version 2.0
+ * http://www.apache.org/licenses/LICENSE-2.0.txt
+ */
+package de.rub.nds.crawler.dns;
+
+import java.net.UnknownHostException;
+
+/** Interface for DNS resolution. Allows for mocking and testing of DNS lookups. */
+public interface DnsResolver {
+    /**
+     * Resolves a hostname to its IP address.
+     *
+     * @param hostname the hostname to resolve
+     * @return the IP address as a string
+     * @throws UnknownHostException if the hostname cannot be resolved
+     */
+    String resolveHostname(String hostname) throws UnknownHostException;
+}

src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java

@@ -11,6 +11,7 @@
 import static org.junit.jupiter.api.Assertions.*;
 
 import de.rub.nds.crawler.constant.JobStatus;
+import de.rub.nds.crawler.dns.MockDnsResolver;
 import org.apache.commons.lang3.tuple.Pair;
 import org.junit.jupiter.api.Test;
 
@@ -76,22 +77,28 @@ void testIPv6CompressedAddress() {
 
     @Test
     void testHostnameWithPort() {
+        MockDnsResolver mockDnsResolver = new MockDnsResolver();
+        mockDnsResolver.addMapping("example.com", "93.184.216.34");
+
         Pair<ScanTarget, JobStatus> result =
-                ScanTarget.fromTargetString("example.com:8080", 443, null);
+                ScanTarget.fromTargetString("example.com:8080", 443, null, mockDnsResolver);
         assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight());
         assertEquals("example.com", result.getLeft().getHostname());
         assertEquals(8080, result.getLeft().getPort());
-        // IP will be resolved by DNS lookup, so we can't test the exact value
-        assertNotNull(result.getLeft().getIp());
+        assertEquals("93.184.216.34", result.getLeft().getIp());
     }
 
     @Test
     void testHostnameWithoutPort() {
-        Pair<ScanTarget, JobStatus> result = ScanTarget.fromTargetString("example.com", 443, null);
+        MockDnsResolver mockDnsResolver = new MockDnsResolver();
+        mockDnsResolver.addMapping("example.com", "93.184.216.34");
+
+        Pair<ScanTarget, JobStatus> result =
+                ScanTarget.fromTargetString("example.com", 443, null, mockDnsResolver);
         assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight());
         assertEquals("example.com", result.getLeft().getHostname());
         assertEquals(443, result.getLeft().getPort());
-        assertNotNull(result.getLeft().getIp());
+        assertEquals("93.184.216.34", result.getLeft().getIp());
     }
 
     @Test
@@ -143,10 +150,14 @@ void testTrancoRankWithIPv6() {
 
     @Test
     void testUnknownHost() {
+        MockDnsResolver mockDnsResolver = new MockDnsResolver();
+        mockDnsResolver.addUnresolvableHost("this-host-should-not-exist.invalid");
+
         Pair<ScanTarget, JobStatus> result =
-                ScanTarget.fromTargetString("this-host-should-not-exist-12345.com", 443, null);
+                ScanTarget.fromTargetString(
+                        "this-host-should-not-exist.invalid", 443, null, mockDnsResolver);
         assertEquals(JobStatus.UNRESOLVABLE, result.getRight());
-        assertEquals("this-host-should-not-exist-12345.com", result.getLeft().getHostname());
+        assertEquals("this-host-should-not-exist.invalid", result.getLeft().getHostname());
         assertNull(result.getLeft().getIp());
     }
 

src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java

@@ -0,0 +1,58 @@
+/*
+ * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner
+ *
+ * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH
+ *
+ * Licensed under Apache License, Version 2.0
+ * http://www.apache.org/licenses/LICENSE-2.0.txt
+ */
+package de.rub.nds.crawler.dns;
+
+import java.net.UnknownHostException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Mock DNS resolver for testing purposes. Allows configuring hostname-to-IP mappings and
+ * controlling whether a hostname should throw an UnknownHostException.
+ */
+public class MockDnsResolver implements DnsResolver {
+
+    private final Map<String, String> hostnameToIpMap = new HashMap<>();
+    private final Map<String, Boolean> hostnameToUnresolvableMap = new HashMap<>();
+
+    /**
+     * Adds a mapping from hostname to IP address.
+     *
+     * @param hostname the hostname to map
+     * @param ipAddress the IP address to return for this hostname
+     */
+    public void addMapping(String hostname, String ipAddress) {
+        hostnameToIpMap.put(hostname, ipAddress);
+        hostnameToUnresolvableMap.put(hostname, false);
+    }
+
+    /**
+     * Configures a hostname to throw UnknownHostException when resolved.
+     *
+     * @param hostname the hostname that should be unresolvable
+     */
+    public void addUnresolvableHost(String hostname) {
+        hostnameToUnresolvableMap.put(hostname, true);
+    }
+
+    @Override
+    public String resolveHostname(String hostname) throws UnknownHostException {
+        if (hostnameToUnresolvableMap.getOrDefault(hostname, false)) {
+            throw new UnknownHostException("Mock: hostname is unresolvable: " + hostname);
+        }
+
+        String ipAddress = hostnameToIpMap.get(hostname);
+        if (ipAddress != null) {
+            return ipAddress;
+        }
+
+        // If no mapping is found, throw UnknownHostException
+        throw new UnknownHostException("Mock: no mapping configured for hostname: " + hostname);
+    }
+}