From 3a856a05a994ea775c6a249f4f8212beb7630d65 Mon Sep 17 00:00:00 2001 From: R3dTr4p Date: Wed, 4 Mar 2026 23:27:03 +0100 Subject: [PATCH 1/2] add bulk download of HackerOne reports as Markdown New `bbscope reports h1` command fetches all reports via the Hacker API and saves them as structured Markdown files organized by program handle. Supports --dry-run, --program/--state/--severity filters, and --overwrite. Co-Authored-By: Claude Opus 4.6 --- cmd/reports.go | 21 ++++ cmd/reports_h1.go | 117 ++++++++++++++++++++++ pkg/reports/hackerone.go | 210 +++++++++++++++++++++++++++++++++++++++ pkg/reports/types.go | 44 ++++++++ pkg/reports/writer.go | 123 +++++++++++++++++++++++ 5 files changed, 515 insertions(+) create mode 100644 cmd/reports.go create mode 100644 cmd/reports_h1.go create mode 100644 pkg/reports/hackerone.go create mode 100644 pkg/reports/types.go create mode 100644 pkg/reports/writer.go diff --git a/cmd/reports.go b/cmd/reports.go new file mode 100644 index 0000000..461a71d --- /dev/null +++ b/cmd/reports.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +var reportsCmd = &cobra.Command{ + Use: "reports", + Short: "Download bug bounty reports as Markdown files", +} + +func init() { + rootCmd.AddCommand(reportsCmd) + + reportsCmd.PersistentFlags().String("output-dir", "reports", "Output directory for downloaded reports") + reportsCmd.PersistentFlags().StringSlice("program", nil, "Filter by program handle(s)") + reportsCmd.PersistentFlags().StringSlice("state", nil, "Filter by report state(s) (e.g. resolved,triaged)") + reportsCmd.PersistentFlags().StringSlice("severity", nil, "Filter by severity (e.g. high,critical)") + reportsCmd.PersistentFlags().Bool("dry-run", false, "List reports without downloading") + reportsCmd.PersistentFlags().Bool("overwrite", false, "Overwrite existing report files") +} diff --git a/cmd/reports_h1.go b/cmd/reports_h1.go new file mode 100644 index 0000000..152d956 --- /dev/null +++ b/cmd/reports_h1.go @@ -0,0 +1,117 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "text/tabwriter" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/sw33tLie/bbscope/v2/internal/utils" + "github.com/sw33tLie/bbscope/v2/pkg/reports" + "github.com/sw33tLie/bbscope/v2/pkg/whttp" +) + +var reportsH1Cmd = &cobra.Command{ + Use: "h1", + Short: "Download reports from HackerOne", + RunE: func(cmd *cobra.Command, _ []string) error { + user := viper.GetString("hackerone.username") + token := viper.GetString("hackerone.token") + if user == "" || token == "" { + utils.Log.Error("hackerone requires a username and token") + return nil + } + + proxy, _ := rootCmd.Flags().GetString("proxy") + if proxy != "" { + whttp.SetupProxy(proxy) + } + + outputDir, _ := cmd.Flags().GetString("output-dir") + programs, _ := cmd.Flags().GetStringSlice("program") + states, _ := cmd.Flags().GetStringSlice("state") + severities, _ := cmd.Flags().GetStringSlice("severity") + dryRun, _ := cmd.Flags().GetBool("dry-run") + overwrite, _ := cmd.Flags().GetBool("overwrite") + + fetcher := reports.NewH1Fetcher(user, token) + opts := reports.FetchOptions{ + Programs: programs, + States: states, + Severities: severities, + DryRun: dryRun, + Overwrite: overwrite, + OutputDir: outputDir, + } + + return runReportsH1(cmd.Context(), fetcher, opts) + }, +} + +func init() { + reportsCmd.AddCommand(reportsH1Cmd) + reportsH1Cmd.Flags().StringP("user", "u", "", "HackerOne username") + reportsH1Cmd.Flags().StringP("token", "t", "", "HackerOne API token") + viper.BindPFlag("hackerone.username", reportsH1Cmd.Flags().Lookup("user")) + viper.BindPFlag("hackerone.token", reportsH1Cmd.Flags().Lookup("token")) +} + +func runReportsH1(ctx context.Context, fetcher *reports.H1Fetcher, opts reports.FetchOptions) error { + utils.Log.Info("Fetching report list from HackerOne...") + + summaries, err := fetcher.ListReports(ctx, opts) + if err != nil { + return fmt.Errorf("listing reports: %w", err) + } + + utils.Log.Infof("Found %d reports", len(summaries)) + + if len(summaries) == 0 { + return nil + } + + // Dry-run: print table and exit + if opts.DryRun { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tPROGRAM\tSTATE\tSEVERITY\tCREATED\tTITLE") + for _, s := range summaries { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", + s.ID, s.ProgramHandle, s.Substate, s.SeverityRating, s.CreatedAt, s.Title) + } + w.Flush() + return nil + } + + // Download mode + var written, skipped, errored int + total := len(summaries) + + for i, s := range summaries { + utils.Log.Infof("[%d/%d] Fetching report %s: %s", i+1, total, s.ID, s.Title) + + report, err := fetcher.FetchReport(ctx, s.ID) + if err != nil { + utils.Log.Warnf("Error fetching report %s: %v", s.ID, err) + errored++ + continue + } + + ok, err := reports.WriteReport(report, opts.OutputDir, opts.Overwrite) + if err != nil { + utils.Log.Warnf("Error writing report %s: %v", s.ID, err) + errored++ + continue + } + + if ok { + written++ + } else { + skipped++ + } + } + + utils.Log.Infof("Done: %d written, %d skipped, %d errors", written, skipped, errored) + return nil +} diff --git a/pkg/reports/hackerone.go b/pkg/reports/hackerone.go new file mode 100644 index 0000000..5dbc5e6 --- /dev/null +++ b/pkg/reports/hackerone.go @@ -0,0 +1,210 @@ +package reports + +import ( + "context" + "encoding/base64" + "fmt" + "strconv" + "strings" + "time" + + "github.com/sw33tLie/bbscope/v2/internal/utils" + "github.com/sw33tLie/bbscope/v2/pkg/whttp" + "github.com/tidwall/gjson" +) + +// H1Fetcher fetches reports from the HackerOne Hacker API. +type H1Fetcher struct { + authB64 string +} + +// NewH1Fetcher creates a fetcher using the same base64 auth pattern as the poller. +func NewH1Fetcher(username, token string) *H1Fetcher { + raw := username + ":" + token + return &H1Fetcher{authB64: base64.StdEncoding.EncodeToString([]byte(raw))} +} + +// ListReports fetches all report summaries matching the given options. +func (f *H1Fetcher) ListReports(ctx context.Context, opts FetchOptions) ([]ReportSummary, error) { + var summaries []ReportSummary + + queryFilter := buildQueryFilter(opts) + currentURL := "https://api.hackerone.com/v1/hackers/me/reports?page%5Bsize%5D=100" + if queryFilter != "" { + currentURL += "&filter%5Bkeyword%5D=" + queryFilter + } + + for { + body, err := f.doRequest(currentURL) + if err != nil { + return summaries, err + } + if body == "" { + break // non-retryable status, stop + } + + count := int(gjson.Get(body, "data.#").Int()) + for i := 0; i < count; i++ { + prefix := "data." + strconv.Itoa(i) + summary := ReportSummary{ + ID: gjson.Get(body, prefix+".id").String(), + Title: gjson.Get(body, prefix+".attributes.title").String(), + State: gjson.Get(body, prefix+".attributes.state").String(), + Substate: gjson.Get(body, prefix+".attributes.substate").String(), + CreatedAt: gjson.Get(body, prefix+".attributes.created_at").String(), + SeverityRating: gjson.Get(body, prefix+".relationships.severity.data.attributes.rating").String(), + } + + // Program handle from relationships + summary.ProgramHandle = gjson.Get(body, prefix+".relationships.program.data.attributes.handle").String() + + summaries = append(summaries, summary) + } + + nextURL := gjson.Get(body, "links.next").String() + if nextURL == "" { + break + } + currentURL = nextURL + } + + return summaries, nil +} + +// FetchReport fetches the full detail of a single report by ID. +func (f *H1Fetcher) FetchReport(ctx context.Context, reportID string) (*Report, error) { + url := "https://api.hackerone.com/v1/hackers/reports/" + reportID + + body, err := f.doRequest(url) + if err != nil { + return nil, err + } + if body == "" { + return nil, fmt.Errorf("report %s: not found or not accessible", reportID) + } + + r := &Report{ + ID: gjson.Get(body, "data.id").String(), + Title: gjson.Get(body, "data.attributes.title").String(), + State: gjson.Get(body, "data.attributes.state").String(), + Substate: gjson.Get(body, "data.attributes.substate").String(), + CreatedAt: formatTimestamp(gjson.Get(body, "data.attributes.created_at").String()), + TriagedAt: formatTimestamp(gjson.Get(body, "data.attributes.triaged_at").String()), + ClosedAt: formatTimestamp(gjson.Get(body, "data.attributes.closed_at").String()), + DisclosedAt: formatTimestamp(gjson.Get(body, "data.attributes.disclosed_at").String()), + VulnerabilityInformation: gjson.Get(body, "data.attributes.vulnerability_information").String(), + Impact: gjson.Get(body, "data.attributes.impact").String(), + ProgramHandle: gjson.Get(body, "data.relationships.program.data.attributes.handle").String(), + SeverityRating: gjson.Get(body, "data.relationships.severity.data.attributes.rating").String(), + CVSSScore: gjson.Get(body, "data.relationships.severity.data.attributes.score").String(), + WeaknessName: gjson.Get(body, "data.relationships.weakness.data.attributes.name").String(), + WeaknessCWE: gjson.Get(body, "data.relationships.weakness.data.attributes.external_id").String(), + AssetIdentifier: gjson.Get(body, "data.relationships.structured_scope.data.attributes.asset_identifier").String(), + } + + // Bounty amounts — sum all bounty relationships + var totalBounty float64 + bounties := gjson.Get(body, "data.relationships.bounties.data") + if bounties.Exists() { + bounties.ForEach(func(_, v gjson.Result) bool { + totalBounty += v.Get("attributes.amount").Float() + return true + }) + } + if totalBounty > 0 { + r.BountyAmount = fmt.Sprintf("%.2f", totalBounty) + } + + // CVE IDs + cves := gjson.Get(body, "data.attributes.cve_ids") + if cves.Exists() { + cves.ForEach(func(_, v gjson.Result) bool { + if id := v.String(); id != "" { + r.CVEIDs = append(r.CVEIDs, id) + } + return true + }) + } + + return r, nil +} + +// doRequest performs an authenticated GET with retries and rate-limit handling. +func (f *H1Fetcher) doRequest(url string) (string, error) { + retries := 3 + for retries > 0 { + res, err := whttp.SendHTTPRequest(&whttp.WHTTPReq{ + Method: "GET", + URL: url, + Headers: []whttp.WHTTPHeader{{Name: "Authorization", Value: "Basic " + f.authB64}}, + }, nil) + + if err != nil { + retries-- + utils.Log.Warnf("HTTP request failed (%s), retrying: %v", url, err) + time.Sleep(2 * time.Second) + continue + } + + // Rate limited + if res.StatusCode == 429 { + utils.Log.Warn("Rate limited by HackerOne, waiting 60s...") + time.Sleep(60 * time.Second) + continue // don't decrement retries for rate limits + } + + // Non-retryable errors + if res.StatusCode == 400 || res.StatusCode == 403 || res.StatusCode == 404 { + utils.Log.Warnf("Got status %d for %s, skipping", res.StatusCode, url) + return "", nil + } + + if res.StatusCode != 200 { + retries-- + utils.Log.Warnf("Got status %d for %s, retrying", res.StatusCode, url) + time.Sleep(2 * time.Second) + continue + } + + return res.BodyString, nil + } + + return "", fmt.Errorf("failed to fetch %s after retries", url) +} + +// buildQueryFilter builds a Lucene-syntax filter string for the H1 API. +func buildQueryFilter(opts FetchOptions) string { + var parts []string + + if len(opts.Programs) > 0 { + for _, p := range opts.Programs { + parts = append(parts, "team:"+p) + } + } + + if len(opts.States) > 0 { + for _, s := range opts.States { + parts = append(parts, "substate:"+s) + } + } + + if len(opts.Severities) > 0 { + for _, s := range opts.Severities { + parts = append(parts, "severity_rating:"+s) + } + } + + return strings.Join(parts, " ") +} + +// formatTimestamp converts an ISO 8601 timestamp to a human-readable format. +func formatTimestamp(ts string) string { + if ts == "" { + return "" + } + t, err := time.Parse(time.RFC3339, ts) + if err != nil { + return ts // return as-is if unparseable + } + return t.UTC().Format("2006-01-02 15:04 UTC") +} diff --git a/pkg/reports/types.go b/pkg/reports/types.go new file mode 100644 index 0000000..1ee0f59 --- /dev/null +++ b/pkg/reports/types.go @@ -0,0 +1,44 @@ +package reports + +// Report holds the full detail of a single HackerOne report. +type Report struct { + ID string + Title string + State string + Substate string + CreatedAt string + TriagedAt string + ClosedAt string + DisclosedAt string + VulnerabilityInformation string + Impact string + CVEIDs []string + ProgramHandle string + SeverityRating string + CVSSScore string + WeaknessName string + WeaknessCWE string + AssetIdentifier string + BountyAmount string +} + +// ReportSummary is the lightweight version returned by the list endpoint. +type ReportSummary struct { + ID string + Title string + State string + Substate string + ProgramHandle string + SeverityRating string + CreatedAt string +} + +// FetchOptions controls which reports to fetch and how to save them. +type FetchOptions struct { + Programs []string + States []string + Severities []string + DryRun bool + Overwrite bool + OutputDir string +} diff --git a/pkg/reports/writer.go b/pkg/reports/writer.go new file mode 100644 index 0000000..49e0f20 --- /dev/null +++ b/pkg/reports/writer.go @@ -0,0 +1,123 @@ +package reports + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "text/template" +) + +var unsafeChars = regexp.MustCompile(`[^a-zA-Z0-9_\-]`) +var multiUnderscore = regexp.MustCompile(`_+`) + +const reportTemplate = `# {{.Title}} + +| Field | Value | +|-------|-------| +| **Report ID** | {{.ID}} | +| **Program** | {{.ProgramHandle}} | +| **State** | {{.Substate}} | +{{- if .SeverityRating}} +| **Severity** | {{.SeverityRating}}{{if .CVSSScore}} (CVSS: {{.CVSSScore}}){{end}} | +{{- end}} +{{- if .WeaknessName}} +| **Weakness** | {{.WeaknessName}}{{if .WeaknessCWE}} ({{.WeaknessCWE}}){{end}} | +{{- end}} +{{- if .AssetIdentifier}} +| **Asset** | {{.AssetIdentifier}} | +{{- end}} +{{- if .BountyAmount}} +| **Bounty** | ${{.BountyAmount}} | +{{- end}} +{{- if .CVEList}} +| **CVE(s)** | {{.CVEList}} | +{{- end}} +| **Created** | {{.CreatedAt}} | +{{- if .TriagedAt}} +| **Triaged** | {{.TriagedAt}} | +{{- end}} +{{- if .ClosedAt}} +| **Closed** | {{.ClosedAt}} | +{{- end}} +{{- if .DisclosedAt}} +| **Disclosed** | {{.DisclosedAt}} | +{{- end}} + +--- +{{if .VulnerabilityInformation}} + +## Vulnerability Information + +{{.VulnerabilityInformation}} + +--- +{{end}} +{{- if .Impact}} + +## Impact + +{{.Impact}} +{{end}}` + +var tmpl = template.Must(template.New("report").Parse(reportTemplate)) + +// templateData wraps Report with computed fields for the template. +type templateData struct { + Report + CVEList string +} + +func sanitizeFilename(s string) string { + s = unsafeChars.ReplaceAllString(s, "_") + s = multiUnderscore.ReplaceAllString(s, "_") + s = strings.Trim(s, "_") + if len(s) > 100 { + s = s[:100] + } + return s +} + +// ReportFilePath returns the filesystem path for a report markdown file. +func ReportFilePath(outputDir string, r *Report) string { + handle := sanitizeFilename(r.ProgramHandle) + if handle == "" { + handle = "unknown" + } + filename := fmt.Sprintf("%s_%s.md", r.ID, sanitizeFilename(r.Title)) + return filepath.Join(outputDir, "h1", handle, filename) +} + +// WriteReport renders a report as Markdown and writes it to disk. +// Returns true if the file was written, false if skipped. +func WriteReport(r *Report, outputDir string, overwrite bool) (bool, error) { + path := ReportFilePath(outputDir, r) + + if !overwrite { + if _, err := os.Stat(path); err == nil { + return false, nil // file exists, skip + } + } + + data := templateData{ + Report: *r, + CVEList: strings.Join(r.CVEIDs, ", "), + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return false, fmt.Errorf("rendering template for report %s: %w", r.ID, err) + } + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return false, fmt.Errorf("creating directory for report %s: %w", r.ID, err) + } + + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + return false, fmt.Errorf("writing report %s: %w", r.ID, err) + } + + return true, nil +} From a83a24199bc1223ef810e2a20a54efebff1fa525 Mon Sep 17 00:00:00 2001 From: R3dTr4p Date: Wed, 4 Mar 2026 23:54:57 +0100 Subject: [PATCH 2/2] parallelize H1 report fetching with 10 workers --- cmd/reports_h1.go | 70 +++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/cmd/reports_h1.go b/cmd/reports_h1.go index 152d956..920f3b8 100644 --- a/cmd/reports_h1.go +++ b/cmd/reports_h1.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "os" + "sync" + "sync/atomic" "text/tabwriter" "github.com/spf13/cobra" @@ -84,34 +86,54 @@ func runReportsH1(ctx context.Context, fetcher *reports.H1Fetcher, opts reports. return nil } - // Download mode - var written, skipped, errored int + // Download mode with worker pool + var written, skipped, errored atomic.Int32 total := len(summaries) - for i, s := range summaries { - utils.Log.Infof("[%d/%d] Fetching report %s: %s", i+1, total, s.ID, s.Title) - - report, err := fetcher.FetchReport(ctx, s.ID) - if err != nil { - utils.Log.Warnf("Error fetching report %s: %v", s.ID, err) - errored++ - continue - } - - ok, err := reports.WriteReport(report, opts.OutputDir, opts.Overwrite) - if err != nil { - utils.Log.Warnf("Error writing report %s: %v", s.ID, err) - errored++ - continue - } + workers := 10 + if total < workers { + workers = total + } - if ok { - written++ - } else { - skipped++ - } + jobs := make(chan int, total) + for i := range summaries { + jobs <- i + } + close(jobs) + + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := range jobs { + s := summaries[i] + utils.Log.Infof("[%d/%d] Fetching report %s: %s", i+1, total, s.ID, s.Title) + + report, err := fetcher.FetchReport(ctx, s.ID) + if err != nil { + utils.Log.Warnf("Error fetching report %s: %v", s.ID, err) + errored.Add(1) + continue + } + + ok, err := reports.WriteReport(report, opts.OutputDir, opts.Overwrite) + if err != nil { + utils.Log.Warnf("Error writing report %s: %v", s.ID, err) + errored.Add(1) + continue + } + + if ok { + written.Add(1) + } else { + skipped.Add(1) + } + } + }() } + wg.Wait() - utils.Log.Infof("Done: %d written, %d skipped, %d errors", written, skipped, errored) + utils.Log.Infof("Done: %d written, %d skipped, %d errors", written.Load(), skipped.Load(), errored.Load()) return nil }