diff --git a/internal/runner/autowildcard.go b/internal/runner/autowildcard.go new file mode 100644 index 00000000..b6b03375 --- /dev/null +++ b/internal/runner/autowildcard.go @@ -0,0 +1,129 @@ +package runner + +import ( + "fmt" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/retryabledns" + "github.com/rs/xid" +) + +const autoWildcardProbes = 3 + +// extractRootDomain extracts the root domain from a subdomain. +// For example, "www.sub.example.com" with known root "example.com" returns "example.com". +// Without a known root, it takes the last two labels. +func extractBaseDomain(host string) string { + host = strings.TrimSuffix(host, ".") + parts := strings.Split(host, ".") + if len(parts) >= 2 { + return strings.Join(parts[len(parts)-2:], ".") + } + return host +} + +// AutoWildcardFilter automatically detects wildcard domains and filters results +func (r *Runner) AutoWildcardFilter() { + // Collect all resolved hosts grouped by base domain + baseDomainIPs := make(map[string]map[string]struct{}) // baseDomain -> ip -> set(hosts) + baseDomainHosts := make(map[string][]string) // baseDomain -> hosts + + // Collect all A records per host and group by base domain + hostARecords := make(map[string][]string) + r.hm.Scan(func(k, v []byte) error { + var dnsdata retryabledns.DNSData + if err := dnsdata.Unmarshal(v); err != nil { + return nil + } + host := string(k) + if len(dnsdata.A) > 0 { + hostARecords[host] = dnsdata.A + baseDomain := extractBaseDomain(host) + if _, ok := baseDomainIPs[baseDomain]; !ok { + baseDomainIPs[baseDomain] = make(map[string]struct{}) + } + for _, a := range dnsdata.A { + baseDomainIPs[baseDomain][a] = struct{}{} + } + baseDomainHosts[baseDomain] = append(baseDomainHosts[baseDomain], host) + } + return nil + }) + + if len(baseDomainHosts) == 0 { + return + } + + gologger.Print().Msgf("Starting auto-wildcard detection for %d domain(s)\n", len(baseDomainHosts)) + + // For each base domain, probe random subdomains to detect wildcards + wildcardIPs := make(map[string]map[string]struct{}) // baseDomain -> set of wildcard IPs + + for baseDomain := range baseDomainHosts { + ips := r.probeBaseDomain(baseDomain) + if len(ips) > 0 { + wildcardIPs[baseDomain] = ips + gologger.Verbose().Msgf("Auto-wildcard detected for %s\n", baseDomain) + } + } + + if len(wildcardIPs) == 0 { + gologger.Print().Msgf("No wildcard domains detected\n") + return + } + + gologger.Print().Msgf("Detected %d wildcard domain(s), starting to filter\n", len(wildcardIPs)) + + // Filter hosts: mark hosts whose IPs match wildcard IPs + numRemoved := 0 + for baseDomain, wIPs := range wildcardIPs { + hosts, ok := baseDomainHosts[baseDomain] + if !ok { + continue + } + for _, host := range hosts { + aRecords := hostARecords[host] + allWildcard := len(aRecords) > 0 + for _, ip := range aRecords { + if _, ok := wIPs[ip]; !ok { + allWildcard = false + break + } + } + if allWildcard { + _ = r.wildcards.Set(host, struct{}{}) + numRemoved++ + } + } + } + + gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemoved) +} + +// probeBaseDomain probes a base domain with random subdomains to detect if it's a wildcard +func (r *Runner) probeBaseDomain(baseDomain string) map[string]struct{} { + ipCount := make(map[string]int) + + for i := 0; i < autoWildcardProbes; i++ { + probeHost := fmt.Sprintf("%s.%s", xid.New().String(), baseDomain) + resp, err := r.dnsx.QueryOne(probeHost) + if err != nil || resp == nil || len(resp.A) == 0 { + // If any probe doesn't resolve, not a wildcard + return nil + } + for _, a := range resp.A { + ipCount[a]++ + } + } + + // A domain is considered wildcard if all probes resolved to at least one common IP + wildcardIPs := make(map[string]struct{}) + for ip, count := range ipCount { + if count >= autoWildcardProbes { + wildcardIPs[ip] = struct{}{} + } + } + + return wildcardIPs +} diff --git a/internal/runner/options.go b/internal/runner/options.go index 0e545bd5..996107ef 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -59,6 +59,7 @@ type Options struct { TraceMaxRecursion int WildcardThreshold int WildcardDomain string + AutoWildcard bool ShowStatistics bool rcodes map[int]struct{} RCode string @@ -189,6 +190,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.Resolvers, "resolver", "r", "", "list of resolvers to use (file or comma separated)"), flagSet.IntVarP(&options.WildcardThreshold, "wildcard-threshold", "wt", 5, "wildcard filter threshold"), flagSet.StringVarP(&options.WildcardDomain, "wildcard-domain", "wd", "", "domain name for wildcard filtering (other flags will be ignored - only json output is supported)"), + flagSet.BoolVar(&options.AutoWildcard, "auto-wildcard", false, "automatically detect and filter wildcard DNS responses across all domains"), flagSet.StringVar(&options.Proxy, "proxy", "", "proxy to use (eg socks5://127.0.0.1:8080)"), ) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c98e831c..49ca4283 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -115,7 +115,7 @@ func New(options *Options) (*Runner, error) { } // If no option is specified or wildcard filter has been requested use query type A - if len(questionTypes) == 0 || options.WildcardDomain != "" { + if len(questionTypes) == 0 || options.WildcardDomain != "" || options.AutoWildcard { options.A = true questionTypes = append(questionTypes, dns.TypeA) } @@ -467,8 +467,13 @@ func (r *Runner) run() error { close(r.outputchan) r.wgoutputworker.Wait() - if r.options.WildcardDomain != "" { - gologger.Print().Msgf("Starting to filter wildcard subdomains\n") + if r.options.WildcardDomain != "" || r.options.AutoWildcard { + if r.options.AutoWildcard { + // Auto-wildcard detection and filtering + r.AutoWildcardFilter() + } else { + gologger.Print().Msgf("Starting to filter wildcard subdomains\n") + } ipDomain := make(map[string]map[string]struct{}) listIPs := []string{} // prepare in memory structure similarly to shuffledns @@ -491,53 +496,69 @@ func (r *Runner) run() error { return nil }) - gologger.Debug().Msgf("Found %d unique IPs:%s\n", len(listIPs), strings.Join(listIPs, ", ")) - // wildcard workers - numThreads := r.options.Threads - if numThreads > len(listIPs) { - numThreads = len(listIPs) - } - for i := 0; i < numThreads; i++ { - r.wgwildcardworker.Add(1) - go r.wildcardWorker() - } - - seen := make(map[string]struct{}) - for _, a := range listIPs { - hosts := ipDomain[a] - if len(hosts) >= r.options.WildcardThreshold { - for host := range hosts { - if _, ok := seen[host]; !ok { - seen[host] = struct{}{} - r.wildcardworkerchan <- host + if !r.options.AutoWildcard { + gologger.Debug().Msgf("Found %d unique IPs:%s\n", len(listIPs), strings.Join(listIPs, ", ")) + // wildcard workers + numThreads := r.options.Threads + if numThreads > len(listIPs) { + numThreads = len(listIPs) + } + for i := 0; i < numThreads; i++ { + r.wgwildcardworker.Add(1) + go r.wildcardWorker() + } + + seen := make(map[string]struct{}) + for _, a := range listIPs { + hosts := ipDomain[a] + if len(hosts) >= r.options.WildcardThreshold { + for host := range hosts { + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + r.wildcardworkerchan <- host + } } } } + close(r.wildcardworkerchan) + r.wgwildcardworker.Wait() } - close(r.wildcardworkerchan) - r.wgwildcardworker.Wait() // we need to restart output r.startOutputWorker() - seen = make(map[string]struct{}) + seen := make(map[string]struct{}) seenRemovedSubdomains := make(map[string]struct{}) numRemovedSubdomains := 0 for _, A := range listIPs { for host := range ipDomain[A] { - if host == r.options.WildcardDomain { - if _, ok := seen[host]; !ok { - seen[host] = struct{}{} - _ = r.lookupAndOutput(host) - } - } else if !r.wildcards.Has(host) { - if _, ok := seen[host]; !ok { - seen[host] = struct{}{} - _ = r.lookupAndOutput(host) + if r.options.AutoWildcard { + if !r.wildcards.Has(host) { + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + _ = r.lookupAndOutput(host) + } + } else { + if _, ok := seenRemovedSubdomains[host]; !ok { + numRemovedSubdomains++ + seenRemovedSubdomains[host] = struct{}{} + } } } else { - if _, ok := seenRemovedSubdomains[host]; !ok { - numRemovedSubdomains++ - seenRemovedSubdomains[host] = struct{}{} + if host == r.options.WildcardDomain { + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + _ = r.lookupAndOutput(host) + } + } else if !r.wildcards.Has(host) { + if _, ok := seen[host]; !ok { + seen[host] = struct{}{} + _ = r.lookupAndOutput(host) + } + } else { + if _, ok := seenRemovedSubdomains[host]; !ok { + numRemovedSubdomains++ + seenRemovedSubdomains[host] = struct{}{} + } } } } @@ -545,7 +566,9 @@ func (r *Runner) run() error { close(r.outputchan) // waiting output worker r.wgoutputworker.Wait() - gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) + if !r.options.AutoWildcard { + gologger.Print().Msgf("%d wildcard subdomains removed\n", numRemovedSubdomains) + } } return nil @@ -731,7 +754,7 @@ func (r *Runner) worker() { } } // if wildcard filtering just store the data - if r.options.WildcardDomain != "" { + if r.options.WildcardDomain != "" || r.options.AutoWildcard { if err := r.storeDNSData(dnsData.DNSData); err != nil { gologger.Debug().Msgf("Failed to store DNS data for %s: %v\n", domain, err) }