From c403c9f1f25c426d4ca4dc333d243501c95bb66b Mon Sep 17 00:00:00 2001 From: R3dTr4p Date: Thu, 5 Mar 2026 00:44:57 +0100 Subject: [PATCH] add docs for bulk download reports and fix pagination bug --- README.md | 20 +++++++++ docs/src/SUMMARY.md | 1 + docs/src/cli/reports.md | 90 ++++++++++++++++++++++++++++++++++++++++ pkg/reports/hackerone.go | 12 ++---- 4 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 docs/src/cli/reports.md diff --git a/README.md b/README.md index 31eed4e..eee3aa3 100755 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ Visit [bbscope.com](https://bbscope.com/) to explore an hourly-updated list of p - **Track Changes**: Monitor scope additions and removals over time. - **LLM Cleanup (opt-in)**: Let GPT-style models fix messy scope strings in bulk when polling. - **Flexible Output**: Get your data in plain text, JSON, or CSV. +- **Report Downloads**: Bulk download your HackerOne reports as Markdown files, with parallel fetching and filtering by program, state, or severity. --- @@ -288,6 +289,25 @@ Add a custom target to the database manually. --- +### `reports` - Downloading Reports + +The `reports` command bulk downloads your vulnerability reports as Markdown files, organized by program. + +```bash +# Download all your HackerOne reports +bbscope reports h1 --output-dir ./reports + +# Preview what would be downloaded +bbscope reports h1 --output-dir ./reports --dry-run + +# Filter by program, state, or severity +bbscope reports h1 --output-dir ./reports --program google --state resolved --severity critical +``` + +Reports are saved as `{output-dir}/h1/{program}/{id}_{title}.md` with metadata tables and full vulnerability details. 10 parallel workers handle the downloads with automatic rate-limit handling. + +--- + ## 📖 Examples **1. First-Time Setup: Poll all private, bounty-only programs and save to DB** diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index b12f71f..4a2fdfe 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -13,6 +13,7 @@ - [Polling Scopes](./cli/polling.md) - [Database Commands](./cli/database.md) - [Extracting Targets](./cli/targets.md) +- [Downloading Reports](./cli/reports.md) - [Output Formatting](./cli/output.md) # Database diff --git a/docs/src/cli/reports.md b/docs/src/cli/reports.md new file mode 100644 index 0000000..0ed41c9 --- /dev/null +++ b/docs/src/cli/reports.md @@ -0,0 +1,90 @@ +# Downloading Reports + +The `bbscope reports` command downloads your vulnerability reports from bug bounty platforms as Markdown files. + +## HackerOne + +```bash +# Download all your reports +bbscope reports h1 --output-dir ./reports + +# Preview what would be downloaded (dry-run) +bbscope reports h1 --output-dir ./reports --dry-run + +# Filter by program +bbscope reports h1 --output-dir ./reports --program google --program microsoft + +# Filter by state (e.g., resolved, triaged, new, duplicate, informative, not-applicable, spam) +bbscope reports h1 --output-dir ./reports --state resolved --state triaged + +# Filter by severity +bbscope reports h1 --output-dir ./reports --severity critical --severity high + +# Combine filters +bbscope reports h1 --output-dir ./reports --program google --state resolved --severity critical + +# Overwrite existing files +bbscope reports h1 --output-dir ./reports --overwrite +``` + +### Authentication + +Credentials can be provided via CLI flags or config file: + +```bash +# CLI flags +bbscope reports h1 --user your_username --token your_api_token --output-dir ./reports +``` + +```yaml +# ~/.bbscope.yaml +hackerone: + username: "your_username" + token: "your_api_token" +``` + +### Output structure + +Reports are saved as Markdown files organized by program: + +``` +reports/ +└── h1/ + ├── google/ + │ ├── 123456_XSS_in_login_page.md + │ └── 123457_IDOR_in_user_profile.md + └── microsoft/ + └── 234567_SSRF_in_webhook_handler.md +``` + +Each file contains a metadata table (ID, program, state, severity, weakness, asset, bounty, CVE IDs, timestamps) followed by the vulnerability information and impact sections. + +### Dry-run output + +The `--dry-run` flag prints a table of matching reports without downloading: + +``` +ID PROGRAM STATE SEVERITY CREATED TITLE +123456 google resolved high 2024-01-15T10:30:00.000Z XSS in login page +123457 google triaged critical 2024-02-20T14:00:00.000Z IDOR in user profile +``` + +## Flags + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | | Output directory for downloaded reports (required) | +| `--program` | | Filter by program handle(s) | +| `--state` | | Filter by report state(s) | +| `--severity` | | Filter by severity level(s) | +| `--dry-run` | | List reports without downloading | +| `--overwrite` | | Overwrite existing report files | + +## How it works + +1. **List phase**: fetches all report summaries from the HackerOne API (`/v1/hackers/me/reports`), paginated at 100 per page. Filters are applied server-side using Lucene query syntax. +2. **Download phase**: 10 parallel workers fetch full report details (`/v1/hackers/reports/{id}`) and write them as Markdown files. +3. **Skip logic**: existing files are skipped unless `--overwrite` is set. +4. **Rate limiting**: HTTP 429 responses trigger a 60-second backoff. Other transient errors are retried up to 3 times with a 2-second delay. + +> **Note**: The HackerOne Hacker API may not return draft reports or reports where you are a collaborator but not the primary reporter. If your downloaded count is lower than your dashboard total, this is likely the cause. diff --git a/pkg/reports/hackerone.go b/pkg/reports/hackerone.go index 5dbc5e6..0c2f064 100644 --- a/pkg/reports/hackerone.go +++ b/pkg/reports/hackerone.go @@ -37,11 +37,9 @@ func (f *H1Fetcher) ListReports(ctx context.Context, opts FetchOptions) ([]Repor for { body, err := f.doRequest(currentURL) if err != nil { + utils.Log.Warnf("Error fetching report list page: %v", err) return summaries, err } - if body == "" { - break // non-retryable status, stop - } count := int(gjson.Get(body, "data.#").Int()) for i := 0; i < count; i++ { @@ -77,10 +75,7 @@ func (f *H1Fetcher) FetchReport(ctx context.Context, reportID string) (*Report, body, err := f.doRequest(url) if err != nil { - return nil, err - } - if body == "" { - return nil, fmt.Errorf("report %s: not found or not accessible", reportID) + return nil, fmt.Errorf("report %s: %w", reportID, err) } r := &Report{ @@ -155,8 +150,7 @@ func (f *H1Fetcher) doRequest(url string) (string, error) { // Non-retryable errors if res.StatusCode == 400 || res.StatusCode == 403 || res.StatusCode == 404 { - utils.Log.Warnf("Got status %d for %s, skipping", res.StatusCode, url) - return "", nil + return "", fmt.Errorf("got status %d for %s", res.StatusCode, url) } if res.StatusCode != 200 {