diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..090687df --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,57 @@ +# AGENTS.md + +## Testing + +- **Every feature must have a test.** +- **Every bug must have a regression test.** +- Bug fix workflow: FIRST write a test that expects proper behavior, verify it fails, THEN make the fix, and verify the test passes. + +### Running Tests + +Tests require MongoDB and Redis. Start them if they aren't already running: + +```bash +# Start MongoDB (if not already running) +docker ps | grep -q mongo || docker run -d --name bbot-mongo --ulimit nofile=64000:64000 --rm -p 127.0.0.1:27017:27017 mongo + +# Start Redis (if not already running) +docker ps | grep -q redis || docker run -d --name bbot-redis --rm -p 127.0.0.1:6379:6379 redis +``` + +Then run the tests: + +```bash +# run all tests +uv run pytest -v + +# run specific tests +uv run pytest -v -k test_applet_targets + +# stop on first failure +uv run pytest -x +``` + +### Test Framework + +- Tests use **pytest** with **pytest-asyncio** (`asyncio_mode = "auto"`). +- Tests live in `tests/`. +- Test database config is in `tests/test_config.yml` (MongoDB: `mongodb://localhost:27017/test_bbot`, Redis: `redis://localhost:6379/15`). +- Database cleanup fixtures (`mongo_cleanup`, `redis_cleanup`) run before/after each test. + +### Test Patterns + +Features typically need **both** an API test and a CLI test: +- **API test** in `tests/test_applets/` — verifies the feature works through the Python and HTTP interfaces. +- **CLI test** in `tests/test_cli/` — verifies the feature is accessible and correct via `bbctl` command-line flags. + +#### API Tests + +- **Unit/integration tests** are async functions (`async def test_*`) that use the `bbot_server` fixture. + - The `bbot_server` fixture is parametrized across `python` and `http` interfaces. Call it as `bbot_server = await bbot_server()`. +- **Applet lifecycle tests** inherit from `tests.test_applets.base.BaseAppletTest` and override `setup()`, `after_scan_1()`, `after_scan_2()`, `after_archive()`. Set `needs_worker = True` if the test requires a worker. +- Target models can be tested directly: `from bbot_server.modules.targets.targets_models import CreateTarget, Target`. + +#### CLI Tests + +- Use subprocess calls via `BBCTL_COMMAND` and the `bbot_server_http` fixture (see existing tests in `tests/test_cli/` for the pattern). +- Parse JSON output with `orjson.loads(process.stdout)`, check return codes and stderr messages. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index 7e6c3b63..126c8797 100644 --- a/README.md +++ b/README.md @@ -545,22 +545,7 @@ if __name__ == "__main__": ## Running Tests -When running tests, first start MongoDB and Redis via Docker: - -```bash -docker run --ulimit nofile=64000:64000 --rm -p 127.0.0.1:27017:27017 mongo -docker run --rm -p 6379:6379 redis -``` - -Then execute `pytest`: - -```bash -# run all tests -uv run pytest -v - -# run specific tests -uv run pytest -v -k test_applet_scans -``` +See [AGENTS.md](AGENTS.md) for test setup and instructions. ## Screenshots diff --git a/bbot_server/modules/targets/targets_cli.py b/bbot_server/modules/targets/targets_cli.py index f2959634..9e4adf11 100644 --- a/bbot_server/modules/targets/targets_cli.py +++ b/bbot_server/modules/targets/targets_cli.py @@ -29,6 +29,14 @@ def create( blacklist: Annotated[Path, Option("--blacklist", "-b", help="File containing blacklist")] = None, name: Annotated[str, Option("--name", "-n", help="Target name")] = "", description: Annotated[str, Option("--description", "-d", help="Target description")] = "", + append_seeds: Annotated[ + bool, + Option( + "--append-seeds", + "-as", + help="Append seeds to the default target-based seeds instead of replacing them", + ), + ] = False, strict_scope: Annotated[ bool, Option( @@ -54,6 +62,7 @@ def create( description=description, target=target, seeds=seeds, + append_seeds=append_seeds, blacklist=blacklist, strict_scope=strict_scope, allow_duplicate_hash=allow_duplicates, diff --git a/bbot_server/modules/targets/targets_models.py b/bbot_server/modules/targets/targets_models.py index 1084029e..0205666f 100644 --- a/bbot_server/modules/targets/targets_models.py +++ b/bbot_server/modules/targets/targets_models.py @@ -64,6 +64,10 @@ class BaseTarget(BaseBBOTServerModel): None, description="Domains, IPs, CIDRs, URLs, etc. to seed the scan. If not provided, the target list will be used as seeds.", ) + append_seeds: bool = Field( + False, + description="If True, seeds are appended to the default target-based seeds instead of replacing them. This is useful when you want to add a few extra seeds without having to duplicate all the targets.", + ) blacklist: Optional[list[str]] = Field( default_factory=list, description="Domains, IPs, CIDRs, URLs, etc. to blacklist from the scan. If a host is blacklisted, it will not be scanned.", @@ -100,8 +104,11 @@ class Target(BaseTarget): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + seeds = self.seeds + if self.append_seeds and seeds is not None: + seeds = list(self.target or []) + list(seeds) self._bbot_target = BBOTTarget( - target=self.target, seeds=self.seeds, blacklist=self.blacklist, strict_scope=self.strict_scope + target=self.target, seeds=seeds, blacklist=self.blacklist, strict_scope=self.strict_scope ) # self.target = sorted(self.target.inputs) diff --git a/pyproject.toml b/pyproject.toml index f95ee225..574b6d23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bbot-server" -version = "0.1.6" +version = "0.2.0" description = "" authors = [{name = "TheTechromancer"}] license = "AGPL-3.0" diff --git a/tests/test_applets/test_applet_targets.py b/tests/test_applets/test_applet_targets.py index 278c9af6..de7877a0 100644 --- a/tests/test_applets/test_applet_targets.py +++ b/tests/test_applets/test_applet_targets.py @@ -245,6 +245,59 @@ async def test_target_default_names(bbot_server): assert target3.name == "Target 3" +async def test_append_seeds(bbot_server): + bbot_server = await bbot_server() + + # default behavior: seeds=None means targets are used as seeds + target_default = await bbot_server.create_target( + name="default_seeds", + target=["evilcorp.com", "evilcorp.org"], + ) + assert target_default.seeds is None + assert target_default.seed_size == 0 + # internally, bbot_target uses targets as seeds + assert len(target_default.bbot_target.seeds) == 2 + + # default behavior: explicit seeds replace targets-as-seeds + target_override = await bbot_server.create_target( + name="override_seeds", + target=["evilcorp.com", "evilcorp.org"], + seeds=["only-this.com"], + ) + assert target_override.seeds == ["only-this.com"] + assert target_override.seed_size == 1 + + # append_seeds=True with seeds: targets + extra seeds + target_append = await bbot_server.create_target( + name="append_seeds", + target=["evilcorp.com", "evilcorp.org"], + seeds=["extra-seed.com"], + append_seeds=True, + ) + # the model still stores the original seeds field as-is + assert target_append.seeds == ["extra-seed.com"] + # but the bbot_target has all three as seeds + seed_inputs = {str(s) for s in target_append.bbot_target.seeds.inputs} + assert "evilcorp.com" in seed_inputs + assert "evilcorp.org" in seed_inputs + assert "extra-seed.com" in seed_inputs + assert target_append.seed_size == 3 + + # append_seeds=True with seeds=None: no-op, same as default + target_append_none = await bbot_server.create_target( + name="append_seeds_none", + target=["evilcorp.com"], + append_seeds=True, + ) + assert target_append_none.seeds is None + assert target_append_none.seed_size == 0 + # internally still uses targets as seeds + assert len(target_append_none.bbot_target.seeds) == 1 + + # append_seeds hashes differently from override + assert target_append.seed_hash != target_override.seed_hash + + async def test_target_size(bbot_server): bbot_server = await bbot_server() diff --git a/tests/test_cli/test_cli_targetctl.py b/tests/test_cli/test_cli_targetctl.py index 412e2a8c..72a9b872 100644 --- a/tests/test_cli/test_cli_targetctl.py +++ b/tests/test_cli/test_cli_targetctl.py @@ -125,3 +125,67 @@ def test_cli_targetctl(bbot_server_http): ) assert process.returncode == 1 assert "Target not found" in process.stderr + + +def test_cli_targetctl_append_seeds(bbot_server_http): + target_file = BBOT_SERVER_TEST_DIR / "target.txt" + seeds_file = BBOT_SERVER_TEST_DIR / "seeds.txt" + + target_file.write_text("evilcorp.com\nevilcorp.org") + seeds_file.write_text("extra-seed.com") + + # create target with --append-seeds + process = subprocess.run( + BBCTL_COMMAND + + [ + "--no-color", + "scan", + "target", + "create", + "--target", + str(target_file), + "--seeds", + str(seeds_file), + "--append-seeds", + "--name", + "append_test", + ], + capture_output=True, + text=True, + ) + assert process.returncode == 0 + assert "Target created successfully" in process.stderr + target = orjson.loads(process.stdout) + assert set(target["target"]) == {"evilcorp.com", "evilcorp.org"} + assert target["seeds"] == ["extra-seed.com"] + assert target["append_seeds"] is True + # seed_size should reflect targets + extra seeds (3 total) + assert target["seed_size"] == 3 + + # create target WITHOUT --append-seeds for comparison + process = subprocess.run( + BBCTL_COMMAND + + [ + "--no-color", + "scan", + "target", + "create", + "--target", + str(target_file), + "--seeds", + str(seeds_file), + "--name", + "no_append_test", + ], + capture_output=True, + text=True, + ) + assert process.returncode == 0 + target_no_append = orjson.loads(process.stdout) + assert target_no_append["seeds"] == ["extra-seed.com"] + assert target_no_append["append_seeds"] is False + # seed_size should be just the explicit seeds (1) + assert target_no_append["seed_size"] == 1 + + target_file.unlink() + seeds_file.unlink() diff --git a/uv.lock b/uv.lock index 3f305de0..6b4ffabc 100644 --- a/uv.lock +++ b/uv.lock @@ -194,7 +194,7 @@ dependencies = [ [[package]] name = "bbot-server" -version = "0.1.6" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "bbot" },