diff --git a/.gitignore b/.gitignore index 2ca7b333..2293203d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ coverage.html # vendor/ /cmd/alterx/alterx /alterx +/.testing \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 9866ef7a..37315f43 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,13 @@ make run **Single test execution:** ```bash go test -v -run TestFunctionName ./path/to/package + +# Run specific test at package root +go test -v -run TestMutator +go test -v -run TestInput + +# Run with race detector +go test -race -v ./... ``` ## Architecture @@ -50,21 +57,27 @@ go test -v -run TestFunctionName ./path/to/package ### Core Components **1. Entry Point** (`cmd/alterx/main.go`) -- CLI argument parsing via `runner.ParseFlags()` -- Mode selection logic (default/discover/both) -- Pattern mining flow orchestration -- Deduplication between mined and user-defined patterns +- CLI argument parsing via `runner.ParseFlags()` using goflags library +- Mode selection logic (default/discover/both) passed to `alterx.Options` +- Pattern mining flow orchestration in `Mutator.Execute()` via goroutines +- Output writing with `getOutputWriter()` (file or stdout) +- Rules saving via `Mutator.SaveRules()` after execution completes **2. Mutator Engine** (`mutator.go`, `algo.go`) -- `Mutator` struct: Core permutation generator -- `ClusterBomb` algorithm: Nth-order payload combination using recursion +- `Mutator` struct: Core permutation generator with concurrent execution +- `Execute()` method: Runs default and/or mining modes in parallel goroutines +- `ClusterBomb` algorithm: Recursive Nth-order payload combination (cartesian product) - `IndexMap`: Maintains deterministic ordering for payload iteration -- Template replacement using variables extracted from input domains +- Template replacement using `fasttemplate` library with `{{var}}` syntax +- Deduplication via `dedupe.NewDedupe()` with configurable memory limits +- Smart optimization: Skips words already present in leftmost subdomain **3. Input Processing** (`inputs.go`) - `Input` struct: Parses domains into components (sub, suffix, tld, etld, etc.) +- Uses `publicsuffix` library to extract eTLD and root domain correctly - Variable extraction: `{{sub}}`, `{{sub1}}`, `{{suffix}}`, `{{root}}`, `{{sld}}`, etc. - Multi-level subdomain support (e.g., `cloud.api.example.com` → `sub=cloud`, `sub1=api`) +- `getNValidateRootDomain()`: Validates homogeneous domains for pattern mining **4. Pattern Mining** (`internal/patternmining/`) - **Three-phase discovery algorithm:** @@ -124,13 +137,14 @@ Generates all combinations of payloads across variables: - Early exit when no variables present in template ### Pattern Mining Workflow -1. **Validate input:** Ensure domains share common target (e.g., `.example.com`) -2. **Build distance table:** Compute pairwise Levenshtein distances -3. **Phase 1 - Edit clustering:** Group by edit distance (min to max) +1. **Validate input:** `getNValidateRootDomain()` ensures domains share common root +2. **Build distance table:** Compute pairwise Levenshtein distances with memoization +3. **Phase 1 - Edit clustering:** Group by edit distance (min to max) without prefix enforcement 4. **Phase 2 - N-grams:** Generate unigrams/bigrams, cluster by prefix -5. **Phase 3 - Prefix clustering:** Apply edit distance within prefix groups -6. **Quality validation:** Filter patterns using threshold and ratio metrics -7. **Generate subdomains:** Use DFA to produce strings from patterns +5. **Phase 3 - Prefix clustering:** Apply edit distance within prefix groups for refinement +6. **Quality validation:** `isGoodRule()` filters patterns using threshold and ratio metrics +7. **Regex generation:** Convert clusters to regex with alternations `(a|b)` and optional groups +8. **Generate subdomains:** DFA engine produces fixed-length strings from patterns ## Pattern Mining Modes @@ -155,6 +169,46 @@ Generates all combinations of payloads across variables: - `-quality-ratio 25`: Max ratio of synthetic/observed subdomains - `-save-rules output.json`: Save discovered patterns and metadata to JSON file +## Execution Flow + +### Mode-Based Execution +The `Mutator.Execute()` method orchestrates parallel execution based on mode: + +**Default Mode:** +1. Parse inputs → Extract variables → Validate patterns +2. Optionally enrich payloads from input subdomains +3. For each input × pattern combination: + - Replace input variables (e.g., `{{sub}}`, `{{suffix}}`) + - Execute ClusterBomb for payload permutations + - Skip patterns with missing variables +4. Deduplicate results and write to output + +**Discover Mode:** +1. Validate homogeneous domains (must share root) +2. Initialize `Miner` with distance/quality parameters +3. Run three-phase clustering algorithm +4. Generate regex patterns from clusters +5. Use DFA engine to produce subdomains +6. Skip input domains from output (avoid duplicates) + +**Both Mode:** +- Runs default and discover in parallel goroutines +- Deduplication happens at channel level +- Results combined before writing + +### Key Variables & Utilities + +**Variable Extraction (`util.go`):** +- `getAllVars()`: Extract variable names from template using regex +- `checkMissing()`: Validate all variables have values before execution +- `getSampleMap()`: Merge input variables with payload variables for validation +- `unsafeToBytes()`: Zero-allocation string→byte conversion for performance + +**Deduplication:** +- Enabled by default (`DedupeResults = true` in `mutator.go`) +- Uses memory-efficient dedupe from projectdiscovery/utils +- Estimates required memory: `count * maxkeyLenInBytes` + ## Common Patterns ### Adding New CLI Flags @@ -171,6 +225,14 @@ Generates all combinations of payloads across variables: - **Clustering logic:** `internal/patternmining/clustering.go` - **Tokenization rules:** `tokenize()` in `internal/patternmining/regex.go` - **Quality metrics:** `isGoodRule()` in `internal/patternmining/patternmining.go` +- **DFA operations:** `internal/dank/dank.go` (minimize, generate strings) + +### Working with Modes +When adding features that interact with modes: +1. Check `opts.Mode` in `New()` to conditionally initialize components +2. Use goroutines in `Execute()` for parallel execution (default + discover) +3. Remember to close channels properly in `Execute()` cleanup goroutine +4. Mode validation happens in `Options.Validate()` with backwards-compatible defaults ## Testing Strategy @@ -182,11 +244,14 @@ Generates all combinations of payloads across variables: ## Important Notes - **Dedupe enabled by default:** `DedupeResults = true` in `mutator.go` -- **Prefix optimization:** ClusterBomb skips words already in leftmost subdomain +- **Prefix optimization:** ClusterBomb skips words already in leftmost subdomain (lines 378-387 in `mutator.go`) - **Pattern quality critical:** Low thresholds generate millions of subdomains -- **Distance memoization:** Pattern mining caches Levenshtein distances for performance +- **Distance memoization:** Pattern mining caches Levenshtein distances for performance in `Miner.memo` map - **DFA minimization:** Three-pass Brzozowski ensures minimal automaton - **No breaking changes:** All pattern mining is additive; default behavior unchanged +- **SaveRules timing:** Must be called AFTER `Execute()` to ensure mining completes (line 68-72 in `cmd/alterx/main.go`) +- **Homogeneous domains required:** Discover/both modes validate all domains share same root via `getNValidateRootDomain()` +- **Goroutine-safe:** Pattern mining and default mode run in separate goroutines with WaitGroup coordination ## Credits @@ -202,3 +267,27 @@ Generates all combinations of payloads across variables: - Use `gologger` for all logging (not fmt.Println) - Follow Go naming conventions and project structure - Add tests for new features +- Use `fasttemplate` for variable replacement (already integrated) +- Respect memory limits via `MaxSize` option in output writing + +## Common Gotchas & Troubleshooting + +### Pattern Mining Issues +- **"domains do not have the same root"**: All input domains must share a common root (e.g., all under `.example.com`). Use `getNValidateRootDomain()` to validate. +- **Too many patterns generated**: Decrease `-max-distance` or increase `-pattern-threshold` and `-quality-ratio` +- **No patterns discovered**: Increase `-max-distance` or decrease `-min-distance` to allow more clustering + +### ClusterBomb Performance +- **Memory exhaustion**: Reduce payload sizes or use `-limit` to cap output +- **Slow execution**: Check that prefix optimization is working (should skip redundant words) +- **Expected combinations not appearing**: Verify variables exist in pattern template and payload map + +### Mode Selection +- **Default mode** works without any special validation (backwards compatible) +- **Discover/Both modes** require homogeneous domains (same root) +- **SaveRules only works** with discover/both modes after execution completes + +### Testing Tips +- Use `DryRun()` or `EstimateCount()` to validate logic without generating output +- Test pattern mining with small domain sets first (5-10 domains) +- For ClusterBomb testing, use simple 2-variable patterns to verify cartesian product logic diff --git a/cmd/alterx/main.go b/cmd/alterx/main.go index 964f516c..4fc9b29e 100644 --- a/cmd/alterx/main.go +++ b/cmd/alterx/main.go @@ -3,101 +3,34 @@ package main import ( "io" "os" - "strings" "github.com/projectdiscovery/alterx" - "github.com/projectdiscovery/alterx/internal/patternmining" "github.com/projectdiscovery/alterx/internal/runner" "github.com/projectdiscovery/gologger" - "golang.org/x/net/publicsuffix" ) func main() { cliOpts := runner.ParseFlags() - // Validate mode - if cliOpts.Mode != "default" && cliOpts.Mode != "discover" && cliOpts.Mode != "both" { - gologger.Fatal().Msgf("invalid mode: %s (must be 'default', 'discover', or 'both')", cliOpts.Mode) - } - // Write output with deduplication output := getOutputWriter(cliOpts.Output) - defer closeOutput(output, cliOpts.Output) - // we intentionally remove all known subdomains from the output - // that way only the discovered subdomains are included in the output - dedupWriter := alterx.NewDedupingWriter(output, cliOpts.Domains...) - defer func() { - if err := dedupWriter.Close(); err != nil { - gologger.Error().Msgf("failed to close dedup writer: %v", err) - } - }() - - var estimatedDiscoverOutputs = 0 - - // Handle pattern mining modes (discover or both) - var minedPatterns []string - if cliOpts.Mode == "discover" || cliOpts.Mode == "both" { - target := getNValidateRootDomain(cliOpts.Domains) - if target == "" { - gologger.Fatal().Msgf("pattern mining requires domains with a common target (e.g., sub.example.com)") - } - gologger.Info().Msgf("Target domain: %s", target) - - miner := patternmining.NewMiner(&patternmining.Options{ - Domains: cliOpts.Domains, - Target: target, - MinDistance: cliOpts.MinDistance, - MaxDistance: cliOpts.MaxDistance, - PatternThreshold: cliOpts.PatternThreshold, - QualityRatio: float64(cliOpts.QualityRatio), - MaxLength: 1000, - NgramsLimit: cliOpts.NgramsLimit, - }) - - result, err := miner.Mine() - if err != nil { - gologger.Fatal().Msgf("pattern mining failed: %v", err) - } - - // Save rules if requested - if cliOpts.SaveRules != "" { - if err := miner.SaveRules(result, cliOpts.SaveRules); err != nil { - gologger.Error().Msgf("failed to save rules: %v", err) - } else { - gologger.Info().Msgf("Saved %d patterns to %s", len(result.Patterns), cliOpts.SaveRules) - } - } - - estimatedDiscoverOutputs = int(miner.EstimateCount(result.Patterns)) - - // Generate subdomains from discovered patterns - // and exit early - if cliOpts.Mode == "discover" { - // In discover mode, only use mined patterns - generated := miner.GenerateFromPatterns(result.Patterns) - for _, subdomain := range generated { - if _, err := dedupWriter.Write([]byte(subdomain + "\n")); err != nil { - gologger.Error().Msgf("failed to write subdomain: %v", err) - } - } - gologger.Info().Msgf("Generated %d unique subdomains from discovered patterns", dedupWriter.Count()) - return - } - - // In 'both' mode, collect mined patterns for combination - minedPatterns = result.Patterns - gologger.Info().Msgf("Discovered %d patterns, combining with user-defined patterns", len(minedPatterns)) - } - // Handle default mode or 'both' mode + // Build alterx options with all modes supported alterOpts := alterx.Options{ - Domains: cliOpts.Domains, - Patterns: cliOpts.Patterns, - Payloads: cliOpts.Payloads, - Limit: cliOpts.Limit, - Enrich: cliOpts.Enrich, - MaxSize: cliOpts.MaxSize, + Domains: cliOpts.Domains, + Patterns: cliOpts.Patterns, + Payloads: cliOpts.Payloads, + Limit: cliOpts.Limit, + Enrich: cliOpts.Enrich, + MaxSize: cliOpts.MaxSize, + Mode: cliOpts.Mode, + MinDistance: cliOpts.MinDistance, + MaxDistance: cliOpts.MaxDistance, + PatternThreshold: cliOpts.PatternThreshold, + QualityRatio: float64(cliOpts.QualityRatio), + NgramsLimit: cliOpts.NgramsLimit, + MaxLength: 1000, } if cliOpts.PermutationConfig != "" { @@ -115,20 +48,26 @@ func main() { m, err := alterx.New(&alterOpts) if err != nil { - gologger.Fatal().Msgf("failed to parse alterx config got %v", err) + gologger.Fatal().Msgf("failed to initialize alterx: %v", err) } if cliOpts.Estimate { - estimated := m.EstimateCount() + estimatedDiscoverOutputs + estimated := m.EstimateCount() gologger.Info().Msgf("Estimated Payloads (including duplicates): %v", estimated) return } - // Write alterx results to same dedupWriter (automatic deduplication) - if err = m.ExecuteWithWriter(dedupWriter); err != nil { - gologger.Error().Msgf("failed to write output to file got %v", err) + + // Execute mutator (handles all modes internally) + if err = m.ExecuteWithWriter(output); err != nil { + gologger.Error().Msgf("failed to execute alterx: %v", err) } - gologger.Info().Msgf("Generated %d total unique subdomains (both modes)", dedupWriter.Count()) + // Save rules if requested (must be after Execute to ensure mining is complete) + if cliOpts.SaveRules != "" { + if err := m.SaveRules(cliOpts.SaveRules); err != nil { + gologger.Error().Msgf("failed to save rules: %v", err) + } + } } // getOutputWriter returns the appropriate output writer @@ -142,40 +81,3 @@ func getOutputWriter(outputPath string) io.Writer { } return os.Stdout } - -// closeOutput closes the output writer if it's a file -func closeOutput(output io.Writer, outputPath string) { - if outputPath != "" { - if closer, ok := output.(io.Closer); ok { - if err := closer.Close(); err != nil { - gologger.Error().Msgf("failed to close output file: %v", err) - } - } - } -} - -func getNValidateRootDomain(domains []string) string { - if len(domains) == 0 { - return "" - } - - var rootDomain string - // parse root domain from publicsuffix for first entry - for _, domain := range domains { - if strings.TrimSpace(domain) == "" { - continue - } - if rootDomain == "" { - root, err := publicsuffix.EffectiveTLDPlusOne(domain) - if err != nil || root == "" { - gologger.Fatal().Msgf("failed to derive root domain from %v: %v", domain, err) - } - rootDomain = root - } else { - if domain != rootDomain && !strings.HasSuffix(domain, "."+rootDomain) { - gologger.Fatal().Msgf("domain %v does not have the same root domain as %v, only homogeneous domains are supported in discover mode", domain, rootDomain) - } - } - } - return rootDomain -} diff --git a/internal/patternmining/patternmining.go b/internal/patternmining/patternmining.go index cc76da09..48c48944 100644 --- a/internal/patternmining/patternmining.go +++ b/internal/patternmining/patternmining.go @@ -277,6 +277,26 @@ func (m *Miner) validateDomains() []string { gologger.Verbose().Msgf("Rejecting malformed input: %s", host) continue } + // see: https://github.com/projectdiscovery/alterx/issues/285 + // due to known blocking issues we add some safety check to skip certain domains + // this isn't silver bullet but avoids known blocking issues + if len(tokens[0]) > 5 { + // if subdomain has more than 5 levels then skip it + // ex: service.api.dev.home.us1.americas.example.com + gologger.Verbose().Msgf("Rejecting input: %s since it has more than 5 levels", host) + continue + } + // to avoid expensive computation skip any subdomain that can be tokenized into more than 10 tokens + tokenCount := 0 + for _, tokenGroup := range tokens[0] { + tokenCount += len(tokenGroup) + } + if tokenCount > 10 { + // ex: api1dev-home-us1..... basically even if subdomain levels are less than 5 but have too many + // separators the vector length would become too long + gologger.Verbose().Msgf("Rejecting input: %s since it has more than 10 tokens (found %d)", host, tokenCount) + continue + } knownHosts = append(knownHosts, host) } return m.removeDuplicatesAndSort(knownHosts) diff --git a/mutator.go b/mutator.go index 16ddd94f..64d2870a 100644 --- a/mutator.go +++ b/mutator.go @@ -7,8 +7,11 @@ import ( "io" "regexp" "strings" + "sync" + "sync/atomic" "time" + "github.com/projectdiscovery/alterx/internal/patternmining" "github.com/projectdiscovery/fasttemplate" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/utils/dedupe" @@ -40,6 +43,51 @@ type Options struct { Enrich bool // MaxSize limits output data size MaxSize int + // Mode specifies the operation mode: "default" (default), "discover" (pattern mining only), "both" (combined) + // Empty string defaults to "default" mode for backwards compatibility + Mode string + // MinDistance is the minimum levenshtein distance for clustering + MinDistance int + // MaxDistance is the maximum levenshtein distance for clustering + MaxDistance int + // PatternThreshold is the threshold for pattern quality filtering + PatternThreshold int + // QualityRatio is the maximum ratio of synthetic/observed for pattern validation + QualityRatio float64 + // NgramsLimit limits the number of n-grams to process (0 = no limit) + NgramsLimit int + // MaxLength is the maximum pattern length + MaxLength int +} + +func (v *Options) Validate() error { + // Default to "default" mode if not specified (backwards compatibility) + if v.Mode == "" { + v.Mode = "default" + } + if v.Mode != "default" && v.Mode != "discover" && v.Mode != "both" { + return fmt.Errorf("invalid mode: %s (must be 'default', 'discover', or 'both')", v.Mode) + } + // auto fill default values + if v.MinDistance == 0 { + v.MinDistance = 2 + } + if v.MaxDistance == 0 { + v.MaxDistance = 10 + } + if v.QualityRatio == 0 { + v.QualityRatio = 25 + } + if v.PatternThreshold == 0 { + v.PatternThreshold = 500 + } + if v.NgramsLimit == 0 { + v.NgramsLimit = 0 + } + if v.MaxLength == 0 { + v.MaxLength = 1000 + } + return nil } // Mutator @@ -47,50 +95,98 @@ type Mutator struct { Options *Options payloadCount int Inputs []*Input // all processed inputs - timeTaken time.Duration + timeTaken int64 // atomic access only (stores nanoseconds as int64) // internal or unexported variables maxkeyLenInBytes int + rootDomain string + miner *patternmining.Miner + miningResult *patternmining.Result } // New creates and returns new mutator instance from options func New(opts *Options) (*Mutator, error) { - if len(opts.Domains) == 0 { - return nil, fmt.Errorf("no input provided to calculate permutations") + if err := opts.Validate(); err != nil { + return nil, err } - if len(opts.Payloads) == 0 { - opts.Payloads = map[string][]string{} - if len(DefaultConfig.Payloads) == 0 { - return nil, fmt.Errorf("something went wrong, `DefaultWordList` and input wordlist are empty") + + m := &Mutator{ + Options: opts, + } + + if opts.Mode == "discover" || opts.Mode == "both" { + + // run validation and save root domain in case of discover mode + rootDomain, err := getNValidateRootDomain(m.Options.Domains) + if err != nil { + return nil, err } - opts.Payloads = DefaultConfig.Payloads + m.rootDomain = rootDomain + + miner := patternmining.NewMiner(&patternmining.Options{ + Domains: opts.Domains, + Target: m.rootDomain, + MinDistance: m.Options.MinDistance, + MaxDistance: m.Options.MaxDistance, + PatternThreshold: m.Options.PatternThreshold, + QualityRatio: m.Options.QualityRatio, + MaxLength: m.Options.MaxLength, + NgramsLimit: m.Options.NgramsLimit, + }) + m.miner = miner + } - if len(opts.Patterns) == 0 { - if len(DefaultConfig.Patterns) == 0 { - return nil, fmt.Errorf("something went wrong,`DefaultPatters` and input patterns are empty") + + if opts.Mode == "default" || opts.Mode == "both" { + // validate payloads and patterns for default and both modes + if len(opts.Payloads) == 0 { + opts.Payloads = map[string][]string{} + if len(DefaultConfig.Payloads) == 0 { + return nil, fmt.Errorf("something went wrong, `DefaultWordList` and input wordlist are empty") + } + opts.Payloads = DefaultConfig.Payloads } - opts.Patterns = DefaultConfig.Patterns - } - // purge duplicates if any - for k, v := range opts.Payloads { - dedupe := sliceutil.Dedupe(v) - if len(v) != len(dedupe) { - gologger.Warning().Msgf("%v duplicate payloads found in %v. purging them..", len(v)-len(dedupe), k) - opts.Payloads[k] = dedupe + if len(opts.Patterns) == 0 { + if len(DefaultConfig.Patterns) == 0 { + return nil, fmt.Errorf("something went wrong,`DefaultPatters` and input patterns are empty") + } + opts.Patterns = DefaultConfig.Patterns + } + // purge duplicates if any + for k, v := range opts.Payloads { + dedupe := sliceutil.Dedupe(v) + if len(v) != len(dedupe) { + gologger.Warning().Msgf("%v duplicate payloads found in %v. purging them..", len(v)-len(dedupe), k) + opts.Payloads[k] = dedupe + } + } + + if err := m.validatePatterns(); err != nil { + return nil, err + } + if err := m.prepareInputs(); err != nil { + return nil, err + } + if opts.Enrich { + m.enrichPayloads() } } - m := &Mutator{ - Options: opts, - } - if err := m.validatePatterns(); err != nil { - return nil, err + + return m, nil +} + +// SaveRules saves pattern mining result to a file +func (m *Mutator) SaveRules(filename string) error { + if m.miner == nil { + return fmt.Errorf("pattern mining is not enabled") } - if err := m.prepareInputs(); err != nil { - return nil, err + if m.miningResult == nil { + return fmt.Errorf("pattern mining result is not available") } - if opts.Enrich { - m.enrichPayloads() + if err := m.miner.SaveRules(m.miningResult, filename); err != nil { + return err } - return m, nil + gologger.Info().Msgf("Saved %d patterns to %s", len(m.miningResult.Patterns), filename) + return nil } // Execute calculates all permutations using input wordlist and patterns @@ -101,28 +197,71 @@ func (m *Mutator) Execute(ctx context.Context) <-chan string { count := m.EstimateCount() maxBytes = count * m.maxkeyLenInBytes } + results := make(chan string, 100) + wg := &sync.WaitGroup{} - results := make(chan string, len(m.Options.Patterns)) - go func() { - now := time.Now() - for _, v := range m.Inputs { - varMap := getSampleMap(v.GetMap(), m.Options.Payloads) - for _, pattern := range m.Options.Patterns { - if err := checkMissing(pattern, varMap); err == nil { - statement := Replace(pattern, v.GetMap()) - select { - case <-ctx.Done(): - return - default: - m.clusterBomb(statement, results) + now := time.Now() + + if m.miner != nil { + wg.Add(1) + go func() { + defer wg.Done() + + // Run pattern mining + gologger.Info().Msgf("Running pattern mining on %d domains...", len(m.Options.Domains)) + result, err := m.miner.Mine() + if err != nil { + gologger.Error().Msgf("pattern mining failed: %v", err) + return + } + m.miningResult = result + gologger.Info().Msgf("Discovered %d patterns from input domains", len(result.Patterns)) + + var seen = make(map[string]bool) + for _, sub := range m.Options.Domains { + seen[sub] = true + } + // In discover mode, only use mined patterns + generated := m.miner.GenerateFromPatterns(m.miningResult.Patterns) + for _, subdomain := range generated { + if seen[subdomain] { + // skip the input subdomains + // regulator algo has tendency to generate input subdomains as patterns + continue + } + seen[subdomain] = true + results <- subdomain + } + }() + } + + if len(m.Inputs) > 0 { + wg.Add(1) + go func() { + defer wg.Done() + for _, v := range m.Inputs { + varMap := getSampleMap(v.GetMap(), m.Options.Payloads) + for _, pattern := range m.Options.Patterns { + if err := checkMissing(pattern, varMap); err == nil { + statement := Replace(pattern, v.GetMap()) + select { + case <-ctx.Done(): + return + default: + m.clusterBomb(statement, results) + } + } else { + gologger.Warning().Msgf("%v : failed to evaluate pattern %v. skipping", err.Error(), pattern) } - } else { - gologger.Warning().Msgf("%v : failed to evaluate pattern %v. skipping", err.Error(), pattern) } } - } - m.timeTaken = time.Since(now) + }() + } + + go func() { + wg.Wait() close(results) + atomic.StoreInt64(&m.timeTaken, int64(time.Since(now))) }() if DedupeResults { @@ -145,7 +284,7 @@ func (m *Mutator) ExecuteWithWriter(Writer io.Writer) error { for { value, ok := <-resChan if !ok { - gologger.Info().Msgf("Generated %v permutations in %v", m.payloadCount, m.Time()) + gologger.Info().Msgf("Generated %v unique subdomains in %v", m.payloadCount, m.Time()) return nil } if m.Options.Limit > 0 && m.payloadCount == m.Options.Limit { @@ -207,6 +346,9 @@ func (m *Mutator) EstimateCount() int { } } } + if m.miner != nil && m.miningResult != nil { + counter += int(m.miner.EstimateCount(m.miningResult.Patterns)) + } return counter } @@ -323,5 +465,6 @@ func (m *Mutator) PayloadCount() int { // Time returns time taken to create permutations in seconds func (m *Mutator) Time() string { - return fmt.Sprintf("%.4fs", m.timeTaken.Seconds()) + duration := time.Duration(atomic.LoadInt64(&m.timeTaken)) + return fmt.Sprintf("%.4fs", duration.Seconds()) } diff --git a/util.go b/util.go index ece16517..97c6a40c 100644 --- a/util.go +++ b/util.go @@ -5,6 +5,8 @@ import ( "regexp" "strings" "unsafe" + + "golang.org/x/net/publicsuffix" ) var varRegex = regexp.MustCompile(`\{\{([a-zA-Z0-9]+)\}\}`) @@ -57,3 +59,32 @@ func checkMissing(template string, data map[string]interface{}) error { func unsafeToBytes(data string) []byte { return unsafe.Slice(unsafe.StringData(data), len(data)) } + +func getNValidateRootDomain(domains []string) (string, error) { + if len(domains) == 0 { + return "", fmt.Errorf("no domains provided") + } + + var rootDomain string + // parse root domain from publicsuffix for first entry + for _, domain := range domains { + if strings.TrimSpace(domain) == "" { + continue + } + if rootDomain == "" { + root, err := publicsuffix.EffectiveTLDPlusOne(domain) + if err != nil || root == "" { + return "", fmt.Errorf("failed to derive root domain from %v: %v", domain, err) + } + rootDomain = root + } else { + if domain != rootDomain && !strings.HasSuffix(domain, "."+rootDomain) { + return "", fmt.Errorf("domain %v does not have the same root domain as %v, only homogeneous domains are supported in discover mode", domain, rootDomain) + } + } + } + if rootDomain == "" { + return "", fmt.Errorf("no valid domains found after filtering empty entries") + } + return rootDomain, nil +}