From 2b58307e51c6f7f3988ccf964dff59ab556092fa Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Thu, 13 Nov 2025 08:39:04 +0400 Subject: [PATCH 1/4] Use .invalid TLD which is reserved by RFC 6761 and guaranteed to never resolve --- src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java index f20c6cd..c68dec9 100644 --- a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java +++ b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java @@ -144,9 +144,9 @@ void testTrancoRankWithIPv6() { @Test void testUnknownHost() { Pair result = - ScanTarget.fromTargetString("this-host-should-not-exist-12345.com", 443, null); + ScanTarget.fromTargetString("this-host-should-not-exist.invalid", 443, null); assertEquals(JobStatus.UNRESOLVABLE, result.getRight()); - assertEquals("this-host-should-not-exist-12345.com", result.getLeft().getHostname()); + assertEquals("this-host-should-not-exist.invalid", result.getLeft().getHostname()); assertNull(result.getLeft().getIp()); } From 675a666c1728497e8ad6e0776f7631eea43b82d3 Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Fri, 21 Nov 2025 08:38:03 +0400 Subject: [PATCH 2/4] Add DNS resolver abstraction and mock for testable hostname resolution --- .../de/rub/nds/crawler/data/ScanTarget.java | 23 ++++++-- .../nds/crawler/dns/DefaultDnsResolver.java | 23 ++++++++ .../de/rub/nds/crawler/dns/DnsResolver.java | 25 ++++++++ .../rub/nds/crawler/data/ScanTargetTest.java | 21 +++++-- .../rub/nds/crawler/dns/MockDnsResolver.java | 58 +++++++++++++++++++ 5 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java create mode 100644 src/main/java/de/rub/nds/crawler/dns/DnsResolver.java create mode 100644 src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java diff --git a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java index 0ef2142..d794557 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java @@ -10,8 +10,9 @@ import de.rub.nds.crawler.constant.JobStatus; import de.rub.nds.crawler.denylist.IDenylistProvider; +import de.rub.nds.crawler.dns.DefaultDnsResolver; +import de.rub.nds.crawler.dns.DnsResolver; import java.io.Serializable; -import java.net.InetAddress; import java.net.UnknownHostException; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.validator.routines.InetAddressValidator; @@ -36,6 +37,21 @@ public class ScanTarget implements Serializable { */ public static Pair fromTargetString( String targetString, int defaultPort, IDenylistProvider denylistProvider) { + return fromTargetString(targetString, defaultPort, denylistProvider, new DefaultDnsResolver()); + } + + /** + * Initializes a ScanTarget object from a string that potentially contains a hostname, an ip, a + * port, the tranco rank. + * + * @param targetString from which to create the ScanTarget object + * @param defaultPort that used if no port is present in targetString + * @param denylistProvider which provides info if a host is denylisted + * @param dnsResolver the DNS resolver to use for hostname resolution + * @return ScanTarget object + */ + public static Pair fromTargetString( + String targetString, int defaultPort, IDenylistProvider denylistProvider, DnsResolver dnsResolver) { ScanTarget target = new ScanTarget(); // check if targetString contains rank (e.g. "1,example.com") @@ -55,8 +71,7 @@ public static Pair fromTargetString( targetString = parts[1]; if (targetString.trim().isEmpty()) { try { - target.setIp( - InetAddress.getByName(target.getHostname()).getHostAddress()); + target.setIp(dnsResolver.resolveHostname(target.getHostname())); } catch (UnknownHostException e) { return Pair.of(target, JobStatus.UNRESOLVABLE); } @@ -124,7 +139,7 @@ public static Pair fromTargetString( try { // TODO this only allows one IP per hostname; it may be interesting to scan all IPs // for a domain, or at least one v4 and one v6 - target.setIp(InetAddress.getByName(targetString).getHostAddress()); + target.setIp(dnsResolver.resolveHostname(targetString)); } catch (UnknownHostException e) { LOGGER.error( "Host {} is unknown or can not be reached with error {}.", targetString, e); diff --git a/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java b/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java new file mode 100644 index 0000000..11c02e4 --- /dev/null +++ b/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java @@ -0,0 +1,23 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.dns; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Default DNS resolver implementation that uses the standard Java DNS resolution via InetAddress. + */ +public class DefaultDnsResolver implements DnsResolver { + + @Override + public String resolveHostname(String hostname) throws UnknownHostException { + return InetAddress.getByName(hostname).getHostAddress(); + } +} diff --git a/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java b/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java new file mode 100644 index 0000000..dc0a586 --- /dev/null +++ b/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java @@ -0,0 +1,25 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.dns; + +import java.net.UnknownHostException; + +/** + * Interface for DNS resolution. Allows for mocking and testing of DNS lookups. + */ +public interface DnsResolver { + /** + * Resolves a hostname to its IP address. + * + * @param hostname the hostname to resolve + * @return the IP address as a string + * @throws UnknownHostException if the hostname cannot be resolved + */ + String resolveHostname(String hostname) throws UnknownHostException; +} diff --git a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java index c68dec9..676f705 100644 --- a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java +++ b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java @@ -11,6 +11,7 @@ import static org.junit.jupiter.api.Assertions.*; import de.rub.nds.crawler.constant.JobStatus; +import de.rub.nds.crawler.dns.MockDnsResolver; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; @@ -76,22 +77,27 @@ void testIPv6CompressedAddress() { @Test void testHostnameWithPort() { + MockDnsResolver mockDnsResolver = new MockDnsResolver(); + mockDnsResolver.addMapping("example.com", "93.184.216.34"); + Pair result = - ScanTarget.fromTargetString("example.com:8080", 443, null); + ScanTarget.fromTargetString("example.com:8080", 443, null, mockDnsResolver); assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); assertEquals("example.com", result.getLeft().getHostname()); assertEquals(8080, result.getLeft().getPort()); - // IP will be resolved by DNS lookup, so we can't test the exact value - assertNotNull(result.getLeft().getIp()); + assertEquals("93.184.216.34", result.getLeft().getIp()); } @Test void testHostnameWithoutPort() { - Pair result = ScanTarget.fromTargetString("example.com", 443, null); + MockDnsResolver mockDnsResolver = new MockDnsResolver(); + mockDnsResolver.addMapping("example.com", "93.184.216.34"); + + Pair result = ScanTarget.fromTargetString("example.com", 443, null, mockDnsResolver); assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); assertEquals("example.com", result.getLeft().getHostname()); assertEquals(443, result.getLeft().getPort()); - assertNotNull(result.getLeft().getIp()); + assertEquals("93.184.216.34", result.getLeft().getIp()); } @Test @@ -143,8 +149,11 @@ void testTrancoRankWithIPv6() { @Test void testUnknownHost() { + MockDnsResolver mockDnsResolver = new MockDnsResolver(); + mockDnsResolver.addUnresolvableHost("this-host-should-not-exist.invalid"); + Pair result = - ScanTarget.fromTargetString("this-host-should-not-exist.invalid", 443, null); + ScanTarget.fromTargetString("this-host-should-not-exist.invalid", 443, null, mockDnsResolver); assertEquals(JobStatus.UNRESOLVABLE, result.getRight()); assertEquals("this-host-should-not-exist.invalid", result.getLeft().getHostname()); assertNull(result.getLeft().getIp()); diff --git a/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java b/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java new file mode 100644 index 0000000..4465201 --- /dev/null +++ b/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java @@ -0,0 +1,58 @@ +/* + * TLS-Crawler - A TLS scanning tool to perform large scale scans with the TLS-Scanner + * + * Copyright 2018-2022 Ruhr University Bochum, Paderborn University, and Hackmanit GmbH + * + * Licensed under Apache License, Version 2.0 + * http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package de.rub.nds.crawler.dns; + +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; + +/** + * Mock DNS resolver for testing purposes. Allows configuring hostname-to-IP mappings and + * controlling whether a hostname should throw an UnknownHostException. + */ +public class MockDnsResolver implements DnsResolver { + + private final Map hostnameToIpMap = new HashMap<>(); + private final Map hostnameToUnresolvableMap = new HashMap<>(); + + /** + * Adds a mapping from hostname to IP address. + * + * @param hostname the hostname to map + * @param ipAddress the IP address to return for this hostname + */ + public void addMapping(String hostname, String ipAddress) { + hostnameToIpMap.put(hostname, ipAddress); + hostnameToUnresolvableMap.put(hostname, false); + } + + /** + * Configures a hostname to throw UnknownHostException when resolved. + * + * @param hostname the hostname that should be unresolvable + */ + public void addUnresolvableHost(String hostname) { + hostnameToUnresolvableMap.put(hostname, true); + } + + @Override + public String resolveHostname(String hostname) throws UnknownHostException { + if (hostnameToUnresolvableMap.getOrDefault(hostname, false)) { + throw new UnknownHostException("Mock: hostname is unresolvable: " + hostname); + } + + String ipAddress = hostnameToIpMap.get(hostname); + if (ipAddress != null) { + return ipAddress; + } + + // If no mapping is found, throw UnknownHostException + throw new UnknownHostException("Mock: no mapping configured for hostname: " + hostname); + } +} From 3abb5caa3c1e1c84b5af0ff68ebd7f16046599e8 Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Fri, 21 Nov 2025 08:38:38 +0400 Subject: [PATCH 3/4] Applying mvn spotless:apply --- .../java/de/rub/nds/crawler/data/ScanTarget.java | 8 ++++++-- .../de/rub/nds/crawler/dns/DefaultDnsResolver.java | 2 +- .../java/de/rub/nds/crawler/dns/DnsResolver.java | 4 +--- .../java/de/rub/nds/crawler/data/ScanTargetTest.java | 12 +++++++----- .../java/de/rub/nds/crawler/dns/MockDnsResolver.java | 12 ++++++------ 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java index d794557..1d05c8a 100644 --- a/src/main/java/de/rub/nds/crawler/data/ScanTarget.java +++ b/src/main/java/de/rub/nds/crawler/data/ScanTarget.java @@ -37,7 +37,8 @@ public class ScanTarget implements Serializable { */ public static Pair fromTargetString( String targetString, int defaultPort, IDenylistProvider denylistProvider) { - return fromTargetString(targetString, defaultPort, denylistProvider, new DefaultDnsResolver()); + return fromTargetString( + targetString, defaultPort, denylistProvider, new DefaultDnsResolver()); } /** @@ -51,7 +52,10 @@ public static Pair fromTargetString( * @return ScanTarget object */ public static Pair fromTargetString( - String targetString, int defaultPort, IDenylistProvider denylistProvider, DnsResolver dnsResolver) { + String targetString, + int defaultPort, + IDenylistProvider denylistProvider, + DnsResolver dnsResolver) { ScanTarget target = new ScanTarget(); // check if targetString contains rank (e.g. "1,example.com") diff --git a/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java b/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java index 11c02e4..a6765ff 100644 --- a/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java +++ b/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java @@ -15,7 +15,7 @@ * Default DNS resolver implementation that uses the standard Java DNS resolution via InetAddress. */ public class DefaultDnsResolver implements DnsResolver { - + @Override public String resolveHostname(String hostname) throws UnknownHostException { return InetAddress.getByName(hostname).getHostAddress(); diff --git a/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java b/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java index dc0a586..f46711e 100644 --- a/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java +++ b/src/main/java/de/rub/nds/crawler/dns/DnsResolver.java @@ -10,9 +10,7 @@ import java.net.UnknownHostException; -/** - * Interface for DNS resolution. Allows for mocking and testing of DNS lookups. - */ +/** Interface for DNS resolution. Allows for mocking and testing of DNS lookups. */ public interface DnsResolver { /** * Resolves a hostname to its IP address. diff --git a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java index 676f705..ec27027 100644 --- a/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java +++ b/src/test/java/de/rub/nds/crawler/data/ScanTargetTest.java @@ -79,7 +79,7 @@ void testIPv6CompressedAddress() { void testHostnameWithPort() { MockDnsResolver mockDnsResolver = new MockDnsResolver(); mockDnsResolver.addMapping("example.com", "93.184.216.34"); - + Pair result = ScanTarget.fromTargetString("example.com:8080", 443, null, mockDnsResolver); assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); @@ -92,8 +92,9 @@ void testHostnameWithPort() { void testHostnameWithoutPort() { MockDnsResolver mockDnsResolver = new MockDnsResolver(); mockDnsResolver.addMapping("example.com", "93.184.216.34"); - - Pair result = ScanTarget.fromTargetString("example.com", 443, null, mockDnsResolver); + + Pair result = + ScanTarget.fromTargetString("example.com", 443, null, mockDnsResolver); assertEquals(JobStatus.TO_BE_EXECUTED, result.getRight()); assertEquals("example.com", result.getLeft().getHostname()); assertEquals(443, result.getLeft().getPort()); @@ -151,9 +152,10 @@ void testTrancoRankWithIPv6() { void testUnknownHost() { MockDnsResolver mockDnsResolver = new MockDnsResolver(); mockDnsResolver.addUnresolvableHost("this-host-should-not-exist.invalid"); - + Pair result = - ScanTarget.fromTargetString("this-host-should-not-exist.invalid", 443, null, mockDnsResolver); + ScanTarget.fromTargetString( + "this-host-should-not-exist.invalid", 443, null, mockDnsResolver); assertEquals(JobStatus.UNRESOLVABLE, result.getRight()); assertEquals("this-host-should-not-exist.invalid", result.getLeft().getHostname()); assertNull(result.getLeft().getIp()); diff --git a/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java b/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java index 4465201..84594ea 100644 --- a/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java +++ b/src/test/java/de/rub/nds/crawler/dns/MockDnsResolver.java @@ -17,10 +17,10 @@ * controlling whether a hostname should throw an UnknownHostException. */ public class MockDnsResolver implements DnsResolver { - + private final Map hostnameToIpMap = new HashMap<>(); private final Map hostnameToUnresolvableMap = new HashMap<>(); - + /** * Adds a mapping from hostname to IP address. * @@ -31,7 +31,7 @@ public void addMapping(String hostname, String ipAddress) { hostnameToIpMap.put(hostname, ipAddress); hostnameToUnresolvableMap.put(hostname, false); } - + /** * Configures a hostname to throw UnknownHostException when resolved. * @@ -40,18 +40,18 @@ public void addMapping(String hostname, String ipAddress) { public void addUnresolvableHost(String hostname) { hostnameToUnresolvableMap.put(hostname, true); } - + @Override public String resolveHostname(String hostname) throws UnknownHostException { if (hostnameToUnresolvableMap.getOrDefault(hostname, false)) { throw new UnknownHostException("Mock: hostname is unresolvable: " + hostname); } - + String ipAddress = hostnameToIpMap.get(hostname); if (ipAddress != null) { return ipAddress; } - + // If no mapping is found, throw UnknownHostException throw new UnknownHostException("Mock: no mapping configured for hostname: " + hostname); } From a9657d6e6adc52b2e2e70ef61e0c4c63c633a170 Mon Sep 17 00:00:00 2001 From: Juan del Carmen Grados Vasquez Date: Fri, 21 Nov 2025 09:29:15 +0400 Subject: [PATCH 4/4] Adding doc for DefaultDnsResolver constructor --- src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java b/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java index a6765ff..d46fba5 100644 --- a/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java +++ b/src/main/java/de/rub/nds/crawler/dns/DefaultDnsResolver.java @@ -16,6 +16,9 @@ */ public class DefaultDnsResolver implements DnsResolver { + /** Creates a new default DNS resolver instance. */ + public DefaultDnsResolver() {} + @Override public String resolveHostname(String hostname) throws UnknownHostException { return InetAddress.getByName(hostname).getHostAddress();