Multi-agent scanning support and other improvements

.gitignore

@@ -1,3 +1,11 @@
+# container environment variables
+.env
+
+# cached python stuff
 __pycache__/
+
+# ??
 textual/
-mongodb
+
+# mongodb content
+mongodb

Dockerfile

@@ -1,13 +1,26 @@
 FROM ghcr.io/astral-sh/uv:python3.11-trixie
-RUN apt-get -y update && apt-get -y install curl jq
+
 WORKDIR /app
-COPY pyproject.toml uv.lock ./
-RUN uv sync --frozen --no-install-project
+
 COPY . .
-RUN cp bbot_server/defaults_docker.yml bbot_server/defaults.yml
-RUN uv sync --frozen
-RUN useradd -u 1000 -m bbot && chown -R bbot:bbot /app
+
+# run as much as possible in one layer to minimise the number of layers in, and susequent size of, the final image
+RUN apt-get -y update && \
+  apt-get -y install --no-install-recommends \
+  # ensure sudo is available to install dependencies for containers operating as non-root agents
+  sudo curl jq \
+  && uv sync --frozen --no-install-project \
+  && cp bbot_server/defaults_docker.yml bbot_server/defaults.yml \
+  && uv sync --frozen \
+  && useradd -u 1000 -m bbot \
+  # ensure bbot user can run sudo without a password in order to install dependencies for containers operating as non-root agents
+  && echo "bbot ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/bbot \
+  && chown -R bbot:bbot /app
+
 USER bbot
+
 ENV PATH="/app/.venv/bin:$PATH"
+
 EXPOSE 8807
+
 CMD ["uv", "run", "bbctl", "server", "start", "--api-only"]

README.md

@@ -194,9 +194,12 @@ export BBOT_SERVER_EVENT_STORE__URI="mongodb://localhost:27017/bbot_server"
 export BBOT_SERVER_ASSET_STORE__URI="mongodb://localhost:27017/bbot_server"
 export BBOT_SERVER_USER_STORE__URI="mongodb://localhost:27017/bbot_server"
 
-# Message Queue URI
+# Message Queue URI (standalone Redis)
 export BBOT_SERVER_MESSAGE_QUEUE__URI="redis://localhost:6379/0"
 
+# Message Queue URI (Redis Cluster - use redis+cluster:// scheme)
+export BBOT_SERVER_MESSAGE_QUEUE__URI="redis+cluster://your-redis-cluster-entrypoint:6379/0"
+
 # Agent configuration
 export BBOT_SERVER_AGENT__BASE_PRESET='{"modules": ["nmap"]}'
 

bbot_server/message_queue/redis.py

@@ -4,7 +4,7 @@
 from contextlib import suppress
 
 import redis.asyncio as redis
-from taskiq_redis import RedisStreamBroker
+from taskiq_redis import RedisStreamBroker, RedisStreamClusterBroker
 
 from .base import BaseMessageQueue
 from bbot_server.utils.misc import smart_encode
@@ -29,6 +29,9 @@ class RedisMessageQueue(BaseMessageQueue):
     - bbot:stream:{subject}: for persistent, tailable streams - e.g. events, activities
     - bbot:work:{subject}: for one-time messages, e.g. tasks
 
+    Redis Cluster mode is enabled by using the redis+cluster:// URI scheme, e.g.:
+        redis+cluster://172.16.255.33:6379/0
+
     docker run --rm -p 127.0.0.1:6379:6379 redis
 
     To monitor Redis:
@@ -41,20 +44,30 @@ class RedisMessageQueue(BaseMessageQueue):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._active_subscriptions = []
+        self._cluster_mode = self.uri.startswith("redis+cluster://")
+
+    def _broker_uri(self):
+        """Normalise the URI to redis:// for libraries that don't understand redis+cluster://."""
+        return self.uri.replace("redis+cluster://", "redis://", 1)
 
     async def setup(self):
-        self.log.debug(f"Setting up Redis message queue at {self.uri}")
+        self.log.debug(f"Setting up Redis message queue at {self.uri} (cluster={self._cluster_mode})")
 
         while True:
             try:
-                self.redis = redis.from_url(self.uri)
+                if self._cluster_mode:
+                    self.redis = redis.RedisCluster.from_url(self._broker_uri())
+                else:
+                    self.redis = redis.from_url(self.uri)
                 await self.redis.ping()
                 break
             except Exception as e:
                 self.log.error(f"Failed to connect to Redis at {self.uri}: {e}, retrying...")
                 await asyncio.sleep(1)
 
     async def make_taskiq_broker(self):
+        if self._cluster_mode:
+            return RedisStreamClusterBroker(self._broker_uri())
         return RedisStreamBroker(self.uri)
 
     async def get(self, subject: str, timeout=None):

bbot_server/modules/scans/scans_api.py

@@ -191,44 +191,45 @@ async def cancel_scan(self, id: str, force: bool = False):
     async def start_scans_loop(self):
         try:
             while True:
-                # get all queued scans
                 queued_scans = await self.get_queued_scans()
                 if not queued_scans:
                     await self.sleep(1)
                     continue
                 self.log.info(f"Found {len(queued_scans):,} queued scans")
-                # get all alive agents
                 ready_agents = {str(agent.id): agent for agent in await self.get_online_agents(status="READY")}
                 if not ready_agents:
                     self.log.warning("No agents are currently ready")
                     await self.sleep(1)
                     continue
                 self.log.info(f"Found {len(ready_agents):,} ready agents")
                 for scan in queued_scans:
-                    # find a suitable agent for the scan
+                    if not ready_agents:
+                        break
+
                     if scan.agent_id is None:
                         selected_agent = random.choice(list(ready_agents.values()))
                     else:
                         try:
                             selected_agent = ready_agents[str(scan.agent_id)]
                         except KeyError:
                             self.log.warning(f"Agent {scan.agent_id} was selected for a scan, but it is not online")
-                            # check if agent doesn't exist anymore. if so, we'll clear it from the scan.
                             try:
-                                selected_agent = await self.root.get_agent(str(scan.agent_id))
+                                await self.root.get_agent(str(scan.agent_id))
+                                self.log.info(
+                                    f"Agent {scan.agent_id} exists but is not ready. "
+                                    "Clearing agent from scan so it can be re-assigned"
+                                )
                             except self.BBOTServerNotFoundError:
-                                self.log.warning(f"Scan's agent no longer exists. Clearing agent from scan")
-                                await self.collection.update_one({"id": str(scan.id)}, {"$set": {"agent_id": None}})
-                                continue
+                                self.log.warning("Scan's agent no longer exists. Clearing agent from scan")
+                            await self.collection.update_one({"id": str(scan.id)}, {"$set": {"agent_id": None}})
+                            continue
 
                     self.log.info(f"Selected agent: {selected_agent.name}")
 
-                    # assign the agent to the scan
                     await self.collection.update_one(
                         {"id": str(scan.id)}, {"$set": {"agent_id": str(selected_agent.id)}}
                     )
 
-                    # merge target and preset
                     scan_preset = dict(scan.preset.preset)
                     scan_preset["scan_name"] = scan.name
                     scan_preset["target"] = scan.target.target
@@ -240,7 +241,6 @@ async def start_scans_loop(self):
                     config["scope"] = scope_config
                     scan_preset["config"] = config
 
-                    # send the scan to the agent
                     scan_start_response = await self.execute_agent_command(
                         str(selected_agent.id), "start_scan", scan_id=scan.id, preset=scan_preset
                     )
@@ -258,6 +258,10 @@ async def start_scans_loop(self):
                                 "error": scan_start_response.error,
                             },
                         )
+                        await self.collection.update_one({"id": str(scan.id)}, {"$set": {"agent_id": None}})
+                        ready_agents.pop(str(selected_agent.id), None)
+                        if not ready_agents:
+                            break
                         await self.sleep(1)
                         continue
 
@@ -266,8 +270,10 @@ async def start_scans_loop(self):
                         description=f"Scan [[COLOR]{scan.name}[/COLOR]] sent to agent [[bold]{selected_agent.name}[/bold]]",
                         detail={"scan_id": scan.id, "agent_id": str(selected_agent.id)},
                     )
-                    # make the scan as sent
                     await self.collection.update_one({"id": str(scan.id)}, {"$set": {"status": "SENT_TO_AGENT"}})
+                    ready_agents.pop(str(selected_agent.id), None)
+                    if not ready_agents:
+                        break
 
         except Exception as e:
             self.log.error(f"Error in scans loop: {e}")

build.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# defaults
+IMAGE_NAME=${IMAGE_NAME:-blacklanternsecurity/bbot-server}
+TAG=${TAG:-latest}
+PLATFORMS=${PLATFORMS:-linux/amd64,linux/arm64}
+REGISTRY_TAG=${REGISTRY_TAG:-latest}
+
+# store any custom build environment variables in the .env file...
+# this allows you to build and store you own images, for whatever platforms you need, in whatever registry you want/need to...
+test -f .env || {
+    echo "Error: .env file not found"
+    exit 1
+}
+
+source .env
+
+docker buildx create --use --name multi-builder 
+docker buildx build --platform "${PLATFORMS}" -t "${IMAGE_NAME}:${TAG}" --load .
+
+# only try to push the image to the registry if $REGISTRY_IMAGE_NAME has been set...
+if [ "${REGISTRY_IMAGE_NAME}x" != "x" ]; then
+  docker tag "${IMAGE_NAME}:${TAG}" "${REGISTRY_IMAGE_NAME}:${REGISTRY_TAG}"
+  docker push "${REGISTRY_IMAGE_NAME}:${REGISTRY_TAG}"
+fi
+
+# EOF

compose-agent.yml

@@ -0,0 +1,22 @@
+---
+services:
+  agent:
+    image: ${IMAGE_NAME:-blacklanternsecurity/bbot-server}:${TAG:-latest}
+    user: root
+    # use the following to utilise an explicit agent ID
+    # NOTE: --name has no effect, but bbctl presently requires that it is provided.
+    command: ["bbctl", "agent", "start", "--id", "${BBOT_AGENT_ID:-CHANGE_ME}", "--name", "${BBOT_AGENT_NAME:-Example Agent}"]
+    # use the following to auto-create the agent as "Docker Default Agent"
+    #entrypoint: ["bash", "/app/bbot_server/default_agent.sh"]
+    restart: always
+    environment:
+      - BBOT_SERVER_URL=${BBOT_SERVER_URL:-http://server:8807/v1/}
+      - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true}
+      - BBOT_SERVER_AUTH_HEADER=${BBOT_SERVER_AUTH_HEADER:-X-API-Key}
+      - BBOT_SERVER_API_KEY=${BBOT_SERVER_API_KEY:-CHANGE_ME}
+      # redirect python/certifi to use custom CA bundle if necessary
+      #- SSL_CERT_FILE=${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}
+    volumes:
+      # inject custom CA bundle if necessary
+      #- /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro
+      - ~/.bbot:/root/.bbot

compose-dev.yml

@@ -0,0 +1,68 @@
+x-bbot-server-base: &bbot-server-base
+  build:
+    context: .
+  image: ${IMAGE_NAME:-blacklanternsecurity/bbot-server}:${TAG:-latest}
+  restart: unless-stopped
+  volumes:
+    - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/home/bbot/.config/bbot_server/config.yml"
+    - ./bbot_server:/app/bbot_server
+    - ./bbot_server/defaults_docker.yml:/app/bbot_server/defaults.yml
+
+x-bbot-agent-base: &bbot-agent-base
+  build:
+    context: .
+  image: ${IMAGE_NAME:-blacklanternsecurity/bbot-server}:${TAG:-latest}
+  user: root
+  command: ["bbctl", "agent", "start"]
+  entrypoint: ["bash", "/app/bbot_server/default_agent.sh"]
+  restart: unless-stopped
+  environment:
+    - BBOT_SERVER_URL=${BBOT_SERVER_URL:-http://server:8807/v1/}
+    - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true}
+    - BBOT_SERVER_AUTH_HEADER=${BBOT_SERVER_AUTH_HEADER:-X-API-Key}
+    - BBOT_SERVER_API_KEY=${BBOT_SERVER_API_KEY:-}
+  volumes:
+    - ~/.bbot:/root/.bbot
+
+services:
+  server:
+    <<: *bbot-server-base
+    ports:
+      - "${BBOT_LISTEN_ADDRESS:-127.0.0.1}:${BBOT_PORT:-8807}:8807"
+    command: ["bbctl", "server", "start", "--api-only", "--listen", "0.0.0.0", "--reload"]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8807/v1/docs"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    environment:
+      - BBOT_SERVER_API_KEY=${BBOT_SERVER_API_KEY:-CHANGE_ME}
+      - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true}
+      - BBOT_SERVER_AUTH_HEADER=${BBOT_SERVER_AUTH_HEADER:-X-API-Key}
+    depends_on:
+      - mongodb
+      - redis
+
+  worker:
+    <<: *bbot-server-base
+    command: ["bbctl", "server", "start", "--worker-only"]
+    depends_on:
+      server:
+        condition: service_healthy
+
+  agent:
+    <<: *bbot-agent-base
+    depends_on:
+      server:
+        condition: service_healthy
+
+  mongodb:
+    image: mongo:latest
+    restart: unless-stopped
+    volumes:
+      - ./mongodb:/data/db
+
+  redis:
+    image: redis:latest
+    restart: unless-stopped

compose-server.yml

@@ -0,0 +1,56 @@
+---
+networks:
+  network:
+    name: ${NETWORK:-systemd-traefik}
+    external: true
+
+x-bbot-server-base: &bbot-server-base
+  build:
+    context: .
+  restart: always
+  image: ${IMAGE_NAME:-blacklanternsecurity/bbot-server}:${TAG:-latest}
+  networks:
+    network:
+  volumes:
+    - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/home/bbot/.config/bbot_server/config.yml"
+    # inject custom CA bundle if necessary
+    #- /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:/etc/ssl/certs/ca-certificates.crt:ro
+  environment:
+    - BBOT_SERVER_URL=${BBOT_SERVER_URL:-http://server:8807/v1/}
+    - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true}
+    - BBOT_SERVER_AUTH_HEADER=${BBOT_SERVER_AUTH_HEADER:-X-API-Key}
+    - BBOT_SERVER_API_KEY=${BBOT_SERVER_API_KEY:-CHANGE_ME}
+    - BBOT_SERVER_EVENT_STORE__URI=${BBOT_SERVER_EVENT_STORE__URI:-mongodb://CHANGE_ME:CHANGE_ME@mongodb:27017/bbot}
+    - BBOT_SERVER_ASSET_STORE__URI=${BBOT_SERVER_ASSET_STORE__URI:-mongodb://CHANGE_ME:CHANGE_ME@mongodb:27017/bbot}
+    - BBOT_SERVER_USER_STORE__URI=${BBOT_SERVER_USER_STORE__URI:-mongodb://CHANGE_ME:CHANGE_ME@mongodb:27017/bbot}
+    - BBOT_SERVER_MESSAGE_QUEUE__URI=${BBOT_SERVER_MESSAGE_QUEUE__URI:-redis://redis:6379/0}
+    # redirect python/certifi to use custom CA bundle if necessary
+    #- SSL_CERT_FILE=${SSL_CERT_FILE:-/etc/ssl/certs/ca-certificates.crt}
+
+services:
+  server:
+    <<: *bbot-server-base
+    ports:
+      - "${BBOT_LISTEN_ADDRESS:-0.0.0.0}:${BBOT_PORT:-8807}:8807"
+    command: ["bbctl", "server", "start", "--api-only", "--listen", "0.0.0.0"]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8807/v1/docs"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.bbot_server.rule=Host(`bbot-server.example`)"
+      - "traefik.http.services.bbot_server.loadbalancer.server.port=8807"
+      - "io.containers.autoupdate=registry"
+
+  worker:
+    <<: *bbot-server-base
+    command: ["bbctl", "server", "start", "--worker-only"]
+    labels:
+      - "traefik.enable=false"
+      - "io.containers.autoupdate=registry"
+    depends_on:
+      server:
+        condition: service_healthy

compose.yml

@@ -1,6 +1,7 @@
 x-bbot-server-base: &bbot-server-base
   build:
     context: .
+  image: ${IMAGE_NAME:-blacklanternsecurity/bbot-server}:${TAG:-latest}
   restart: unless-stopped
   volumes:
     - "${BBOT_SERVER_CONFIG:-~/.config/bbot_server/config.yml}:/home/bbot/.config/bbot_server/config.yml"
@@ -21,6 +22,7 @@ services:
       start_period: 30s
     environment:
       - BBOT_SERVER_AUTH_ENABLED=${BBOT_SERVER_AUTH_ENABLED:-true}
+      - BBOT_SERVER_AUTH_HEADER=${BBOT_SERVER_AUTH_HEADER:-X-API-Key}
     depends_on:
       - mongodb
       - redis