feat: add --auto-wildcard flag for automatic wildcard DNS detection

internal/runner/autowildcard.go

@@ -0,0 +1,129 @@
+package runner
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/retryabledns"
+	"github.com/rs/xid"
+)
+
+const autoWildcardProbes = 3
+
+// extractRootDomain extracts the root domain from a subdomain.
+// For example, "www.sub.example.com" with known root "example.com" returns "example.com".
+// Without a known root, it takes the last two labels.
+func extractBaseDomain(host string) string {
+	host = strings.TrimSuffix(host, ".")
+	parts := strings.Split(host, ".")
+	if len(parts) >= 2 {
+		return strings.Join(parts[len(parts)-2:], ".")
+	}
+	return host
+}
+
+// AutoWildcardFilter automatically detects wildcard domains and filters results
+func (r *Runner) AutoWildcardFilter() {
+	// Collect all resolved hosts grouped by base domain
+	baseDomainIPs := make(map[string]map[string]struct{}) // baseDomain -> ip -> set(hosts)
+	baseDomainHosts := make(map[string][]string)           // baseDomain -> hosts
+
+	// Collect all A records per host and group by base domain
+	hostARecords := make(map[string][]string)
+	r.hm.Scan(func(k, v []byte) error {
+		var dnsdata retryabledns.DNSData
+		if err := dnsdata.Unmarshal(v); err != nil {
+			return nil
+		}
+		host := string(k)
+		if len(dnsdata.A) > 0 {
+			hostARecords[host] = dnsdata.A
+			baseDomain := extractBaseDomain(host)
+			if _, ok := baseDomainIPs[baseDomain]; !ok {
+				baseDomainIPs[baseDomain] = make(map[string]struct{})
+			}
+			for _, a := range dnsdata.A {
+				baseDomainIPs[baseDomain][a] = struct{}{}
+			}
+			baseDomainHosts[baseDomain] = append(baseDomainHosts[baseDomain], host)
+		}
+		return nil
+	})
+
+	if len(baseDomainHosts) == 0 {
+		return
+	}
+
+	gologger.Print().Msgf("Starting auto-wildcard detection for %d domain(s)\n", len(baseDomainHosts))
+
+	// For each base domain, probe random subdomains to detect wildcards
+	wildcardIPs := make(map[string]map[string]struct{}) // baseDomain -> set of wildcard IPs
+
+	for baseDomain := range baseDomainHosts {
+		ips := r.probeBaseDomain(baseDomain)
+		if len(ips) > 0 {
+			wildcardIPs[baseDomain] = ips
+			gologger.Verbose().Msgf("Auto-wildcard detected for %s\n", baseDomain)
+		}
+	}
+
+	if len(wildcardIPs) == 0 {
+		gologger.Print().Msgf("No wildcard domains detected\n")
+		return
+	}
+
+	gologger.Print().Msgf("Detected %d wildcard domain(s), starting to filter\n", len(wildcardIPs))
+
+	// Filter hosts: mark hosts whose IPs match wildcard IPs
+	numRemoved := 0
+	for baseDomain, wIPs := range wildcardIPs {
+		hosts, ok := baseDomainHosts[baseDomain]
+		if !ok {
+			continue
+		}
+		for _, host := range hosts {
+			aRecords := hostARecords[host]
+			allWildcard := len(aRecords) > 0
+			for _, ip := range aRecords {
+				if _, ok := wIPs[ip]; !ok {
+					allWildcard = false
+					break
+				}
+			}
+			if allWildcard {
+				_ = r.wildcards.Set(host, struct{}{})
+				numRemoved++
+			}
+		}
+	}
+
+	gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemoved)
+}
+
+// probeBaseDomain probes a base domain with random subdomains to detect if it's a wildcard
+func (r *Runner) probeBaseDomain(baseDomain string) map[string]struct{} {
+	ipCount := make(map[string]int)
+
+	for i := 0; i < autoWildcardProbes; i++ {
+		probeHost := fmt.Sprintf("%s.%s", xid.New().String(), baseDomain)
+		resp, err := r.dnsx.QueryOne(probeHost)
+		if err != nil || resp == nil || len(resp.A) == 0 {
+			// If any probe doesn't resolve, not a wildcard
+			return nil
+		}
+		for _, a := range resp.A {
+			ipCount[a]++
+		}
+	}
+
+	// A domain is considered wildcard if all probes resolved to at least one common IP
+	wildcardIPs := make(map[string]struct{})
+	for ip, count := range ipCount {
+		if count >= autoWildcardProbes {
+			wildcardIPs[ip] = struct{}{}
+		}
+	}
+
+	return wildcardIPs
+}

internal/runner/options.go

@@ -59,6 +59,7 @@ type Options struct {
 	TraceMaxRecursion     int
 	WildcardThreshold     int
 	WildcardDomain        string
+	AutoWildcard          bool
 	ShowStatistics        bool
 	rcodes                map[int]struct{}
 	RCode                 string
@@ -189,6 +190,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"),
 		flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"),
 		flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"),
+		flagSet.BoolVar(&options.AutoWildcard, "auto-wildcard", false, "automatically detect and filter wildcard DNS responses across all domains"),
 		flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"),
 	)
 

internal/runner/runner.go

@@ -115,7 +115,7 @@ func New(options *Options) (*Runner, error) {
 	}
 
 	// If no option is specified or wildcard filter has been requested use query type A
-	if len(questionTypes) == 0 || options.WildcardDomain != "" {
+	if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard {
 		options.A = true
 		questionTypes = append(questionTypes, dns.TypeA)
 	}
@@ -467,8 +467,13 @@ func (r *Runner) run() error {
 	close(r.outputchan)
 	r.wgoutputworker.Wait()
 
-	if r.options.WildcardDomain != "" {
-		gologger.Print().Msgf("Starting to filter wildcard subdomains\n")
+	if r.options.WildcardDomain != "" || r.options.AutoWildcard {
+		if r.options.AutoWildcard {
+			// Auto-wildcard detection and filtering
+			r.AutoWildcardFilter()
+		} else {
+			gologger.Print().Msgf("Starting to filter wildcard subdomains\n")
+		}
 		ipDomain := make(map[string]map[string]struct{})
 		listIPs := []string{}
 		// prepare in memory structure similarly to shuffledns
@@ -491,61 +496,79 @@ func (r *Runner) run() error {
 			return nil
 		})
 
-		gologger.Debug().Msgf("Found %d unique IPs:%s\n", len(listIPs), strings.Join(listIPs, ", "))
-		// wildcard workers
-		numThreads := r.options.Threads
-		if numThreads > len(listIPs) {
-			numThreads = len(listIPs)
-		}
-		for i := 0; i < numThreads; i++ {
-			r.wgwildcardworker.Add(1)
-			go r.wildcardWorker()
-		}
-
-		seen := make(map[string]struct{})
-		for _, a := range listIPs {
-			hosts := ipDomain[a]
-			if len(hosts) >= r.options.WildcardThreshold {
-				for host := range hosts {
-					if _, ok := seen[host]; !ok {
-						seen[host] = struct{}{}
-						r.wildcardworkerchan <- host
+		if !r.options.AutoWildcard {
+			gologger.Debug().Msgf("Found %d unique IPs:%s\n", len(listIPs), strings.Join(listIPs, ", "))
+			// wildcard workers
+			numThreads := r.options.Threads
+			if numThreads > len(listIPs) {
+				numThreads = len(listIPs)
+			}
+			for i := 0; i < numThreads; i++ {
+				r.wgwildcardworker.Add(1)
+				go r.wildcardWorker()
+			}
+
+			seen := make(map[string]struct{})
+			for _, a := range listIPs {
+				hosts := ipDomain[a]
+				if len(hosts) >= r.options.WildcardThreshold {
+					for host := range hosts {
+						if _, ok := seen[host]; !ok {
+							seen[host] = struct{}{}
+							r.wildcardworkerchan <- host
+						}
 					}
 				}
 			}
+			close(r.wildcardworkerchan)
+			r.wgwildcardworker.Wait()
 		}
-		close(r.wildcardworkerchan)
-		r.wgwildcardworker.Wait()
 
 		// we need to restart output
 		r.startOutputWorker()
-		seen = make(map[string]struct{})
+		seen := make(map[string]struct{})
 		seenRemovedSubdomains := make(map[string]struct{})
 		numRemovedSubdomains := 0
 		for _, A := range listIPs {
 			for host := range ipDomain[A] {
-				if host == r.options.WildcardDomain {
-					if _, ok := seen[host]; !ok {
-						seen[host] = struct{}{}
-						_ = r.lookupAndOutput(host)
-					}
-				} else if !r.wildcards.Has(host) {
-					if _, ok := seen[host]; !ok {
-						seen[host] = struct{}{}
-						_ = r.lookupAndOutput(host)
+				if r.options.AutoWildcard {
+					if !r.wildcards.Has(host) {
+						if _, ok := seen[host]; !ok {
+							seen[host] = struct{}{}
+							_ = r.lookupAndOutput(host)
+						}
+					} else {
+						if _, ok := seenRemovedSubdomains[host]; !ok {
+							numRemovedSubdomains++
+							seenRemovedSubdomains[host] = struct{}{}
+						}
 					}
 				} else {
-					if _, ok := seenRemovedSubdomains[host]; !ok {
-						numRemovedSubdomains++
-						seenRemovedSubdomains[host] = struct{}{}
+					if host == r.options.WildcardDomain {
+						if _, ok := seen[host]; !ok {
+							seen[host] = struct{}{}
+							_ = r.lookupAndOutput(host)
+						}
+					} else if !r.wildcards.Has(host) {
+						if _, ok := seen[host]; !ok {
+							seen[host] = struct{}{}
+							_ = r.lookupAndOutput(host)
+						}
+					} else {
+						if _, ok := seenRemovedSubdomains[host]; !ok {
+							numRemovedSubdomains++
+							seenRemovedSubdomains[host] = struct{}{}
+						}
 					}
 				}
 			}
 		}
 		close(r.outputchan)
 		// waiting output worker
 		r.wgoutputworker.Wait()
-		gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains)
+		if !r.options.AutoWildcard {
+			gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains)
+		}
 	}
 
 	return nil
@@ -731,7 +754,7 @@ func (r *Runner) worker() {
 			}
 		}
 		// if wildcard filtering just store the data
-		if r.options.WildcardDomain != "" {
+		if r.options.WildcardDomain != "" || r.options.AutoWildcard {
 			if err := r.storeDNSData(dnsData.DNSData); err != nil {
 				gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err)
 			}