From 7163cf7bc4c1cb4861a91d19501cdc30896e5c3d Mon Sep 17 00:00:00 2001 From: Brandon Smith Date: Wed, 25 Mar 2026 10:30:05 -0500 Subject: [PATCH 01/65] Adds dynamic injection of next.js CORS config upon LAN-accessible network hosting mode Signed-off-by: Brandon Smith --- Makefile | 19 +++++++++++++++---- docker-compose.dev.yml | 16 ++++++++++++++++ docker-compose.override.yml | 6 ++++++ docker-compose.yml | 1 + frontend/Dockerfile.dev | 9 +++++++++ frontend/next.config.ts | 23 ++++++++++++++++++++++- 6 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.override.yml create mode 100644 frontend/Dockerfile.dev diff --git a/Makefile b/Makefile index 49134440..8bf4167b 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ -.PHONY: up-local up-lan down restart-local restart-lan logs status help +.PHONY: up-local up-lan down restart-local restart-lan dev dev-lan logs status help COMPOSE = docker compose # Detect LAN IP (tries Wi-Fi first, falls back to Ethernet) LAN_IP := $(shell ipconfig getifaddr en0 2>/dev/null || ipconfig getifaddr en1 2>/dev/null) +LAN_IP_MSG = "\nShadowbroker is now running and can be accessed by LAN devices at http://$(LAN_IP):3000" ## Default target — print help help: @@ -14,6 +15,7 @@ help: @echo "" @echo " up-local Start with loopback binding (local access only)" @echo " up-lan Start with 0.0.0.0 binding (LAN accessible)" + @echo " up-lan-log Start LAN-accessible with live logging" @echo " down Stop all containers" @echo " restart-local Bounce and restart in local mode" @echo " restart-lan Bounce and restart in LAN mode" @@ -32,9 +34,18 @@ up-lan: exit 1; \ fi @echo "Detected LAN IP: $(LAN_IP)" - BIND=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up -d - @echo "" - @echo "Shadowbroker is now running and can be accessed by LAN devices at http://$(LAN_IP):3000" + BIND=0.0.0.0 HOST=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up -d + @echo "$(LAN_IP_MSG)" + +## Start in LAN mode with live logging (accessible to other hosts on the network) +up-lan-log: + @if [ -z "$(LAN_IP)" ]; then \ + echo "ERROR: Could not detect LAN IP. Check your network connection."; \ + exit 1; \ + fi + @echo "Detected LAN IP: $(LAN_IP)" + BIND=0.0.0.0 HOST=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up + @echo "$(LAN_IP_MSG)" ## Stop all containers down: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..41d72c4c --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,16 @@ +services: + app: + build: + context: . + dockerfile: Dockerfile.dev + ports: + - "3000:3000" + volumes: + - .:/app + - /app/node_modules + - /app/.next + environment: + - HOST=0.0.0.0 + - NODE_ENV=development + command: next dev --hostname 0.0.0.0 --port 3000 + diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000..e9727348 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,6 @@ +# docker-compose.override.yml (automatically loaded) +services: + frontend: + build: + context: ./frontend + dockerfile: Dockerfile.dev diff --git a/docker-compose.yml b/docker-compose.yml index 687c1093..dfc5b486 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,7 @@ services: # Points the Next.js server-side proxy at the backend container via Docker networking. # Change this if your backend runs on a different host or port. - BACKEND_URL=http://backend:8000 + - HOST=${HOST:-127.0.0.1} depends_on: backend: condition: service_healthy diff --git a/frontend/Dockerfile.dev b/frontend/Dockerfile.dev new file mode 100644 index 00000000..c11670b6 --- /dev/null +++ b/frontend/Dockerfile.dev @@ -0,0 +1,9 @@ +FROM node:20-alpine +RUN apk add --no-cache libc6-compat +WORKDIR /app +COPY package*.json ./ +RUN npm ci +COPY . . +ENV NEXT_TELEMETRY_DISABLED=1 +EXPOSE 3000 +CMD ["npx", "next", "dev", "--hostname", "0.0.0.0", "--port", "3000"] diff --git a/frontend/next.config.ts b/frontend/next.config.ts index 6e52e10d..69d47f6b 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -1,13 +1,34 @@ import type { NextConfig } from "next"; +import os from "os"; // /api/* requests are proxied to the backend by the catch-all route handler at // src/app/api/[...path]/route.ts, which reads BACKEND_URL at request time. // Do NOT add rewrites for /api/* here — next.config is evaluated at build time, // so any URL baked in here ignores the runtime BACKEND_URL env var. +function getLanOrigins(): string[] { + if (process.env.HOST !== "0.0.0.0") return []; + + const origins = new Set(); + + for (const ifaces of Object.values(os.networkInterfaces())) { + for (const iface of ifaces ?? []) { + if (iface.family === "IPv4" && !iface.internal) { + const subnet = iface.address.split(".").slice(0, 3).join("."); + origins.add(`${subnet}.*`); + } + } + } + + return [...origins]; +} + +const lanOrigins = getLanOrigins(); + const nextConfig: NextConfig = { - transpilePackages: ['react-map-gl', 'mapbox-gl', 'maplibre-gl'], + transpilePackages: ["react-map-gl", "mapbox-gl", "maplibre-gl"], output: "standalone", + ...(lanOrigins.length > 0 && { allowedDevOrigins: lanOrigins }), }; export default nextConfig; From 2b36539ff551751b874728ee1ba3e015edca98db Mon Sep 17 00:00:00 2001 From: Brandon Smith Date: Wed, 25 Mar 2026 10:33:05 -0500 Subject: [PATCH 02/65] Adds compose watch to frontend for automatic rebuilding upon detected file changes Signed-off-by: Brandon Smith --- Makefile | 16 ++++++++++++++++ docker-compose.yml | 10 ++++++++++ 2 files changed, 26 insertions(+) diff --git a/Makefile b/Makefile index 8bf4167b..2867c304 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ help: @echo " down Stop all containers" @echo " restart-local Bounce and restart in local mode" @echo " restart-lan Bounce and restart in LAN mode" + @echo " dev Start in watch mode (local only)" + @echo " dev-lan Start in watch mode (LAN accessible)" @echo " logs Tail logs for all services" @echo " status Show container status" @echo "" @@ -57,6 +59,20 @@ restart-local: down up-local ## Restart in LAN mode restart-lan: down up-lan +## Start in watch mode (local only, foreground) +dev: + BIND=127.0.0.1 $(COMPOSE) up --watch + +## Start in watch mode (LAN accessible, foreground) +dev-lan: + @if [ -z "$(LAN_IP)" ]; then \ + echo "ERROR: Could not detect LAN IP. Check your network connection."; \ + exit 1; \ + fi + @echo "Detected LAN IP: $(LAN_IP)" + @echo "$(LAN_IP_MSG)" + BIND=0.0.0.0 HOST=0.0.0.0 CORS_ORIGINS=http://$(LAN_IP):3000 $(COMPOSE) up --watch + ## Tail logs for all services logs: $(COMPOSE) logs -f diff --git a/docker-compose.yml b/docker-compose.yml index dfc5b486..67645bde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,16 @@ services: backend: condition: service_healthy restart: unless-stopped + develop: + watch: + - action: sync + path: ./frontend + target: /app + ignore: + - node_modules/ + - .next/ + - action: rebuild + path: ./frontend/package.json healthcheck: test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/"] interval: 30s From 0b0d6ddc77a1f668bc6ace9b20d89af01dd87e72 Mon Sep 17 00:00:00 2001 From: Brandon Smith Date: Wed, 25 Mar 2026 10:40:30 -0500 Subject: [PATCH 03/65] Adds compose watch functionality to backend container Signed-off-by: Brandon Smith --- docker-compose.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 67645bde..02f89d05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,24 @@ services: volumes: - backend_data:/app/data restart: unless-stopped + develop: + watch: + - action: sync+restart + path: ./backend + target: /app + ignore: + - __pycache__/ + - "*.pyc" + - "*.pyc" + - node_modules/ + - action: rebuild + path: pyproject.toml + - action: rebuild + path: uv.lock + - action: rebuild + path: backend/package.json + - action: rebuild + path: backend/package-lock.json healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/api/live-data/fast"] interval: 30s From 54d4055da1e8be1e01396e801e1357b60ed7e28c Mon Sep 17 00:00:00 2001 From: elhard1 Date: Thu, 26 Mar 2026 09:11:30 +0800 Subject: [PATCH 04/65] fix(start.sh): add missing fi after UV bootstrap block The UV install conditional was never closed, which caused 'unexpected end of file' from bash -n and broke the macOS/Linux startup path. Document in ChangelogModal BUG_FIXES (2026-03-26). Made-with: Cursor --- frontend/src/components/ChangelogModal.tsx | 1 + start.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/frontend/src/components/ChangelogModal.tsx b/frontend/src/components/ChangelogModal.tsx index 38ab5953..e2464f1c 100644 --- a/frontend/src/components/ChangelogModal.tsx +++ b/frontend/src/components/ChangelogModal.tsx @@ -41,6 +41,7 @@ const NEW_FEATURES = [ ]; const BUG_FIXES = [ + "Fixed start.sh: added missing `fi` after UV install block — valid bash again; setup runs whether or not uv was preinstalled (2026-03-26)", "Stable entity IDs for GDELT & News popups — no more wrong popup after data refresh (PR #63)", "useCallback optimization for interpolation functions — eliminates redundant React re-renders on every 1s tick", "Restored missing GDELT and datacenter background refreshes in slow-tier loop", diff --git a/start.sh b/start.sh index bf6e54b0..f14040c3 100644 --- a/start.sh +++ b/start.sh @@ -46,6 +46,7 @@ if ! command -v uv &> /dev/null; then fi export PATH="$HOME/.local/bin:$PATH" echo "[*] UV installed successfully." +fi # Get the directory where this script lives SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" From 668ce16dc7965052b3a60aca4ee827637905bfcf Mon Sep 17 00:00:00 2001 From: anoracleofra-code Date: Thu, 26 Mar 2026 05:58:04 -0600 Subject: [PATCH 05/65] v0.9.6: InfoNet hashchain, Wormhole gate encryption, mesh reputation, 16 community contributors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate messages now propagate via the Infonet hashchain as encrypted blobs — every node syncs them through normal chain sync while only Gate members with MLS keys can decrypt. Added mesh reputation system, peer push workers, voluntary Wormhole opt-in for node participation, fork recovery, killwormhole scripts, obfuscated terminology, and hardened the self-updater to protect encryption keys and chain state during updates. New features: Shodan search, train tracking, Sentinel Hub imagery, 8 new intelligence layers, CCTV expansion to 11,000+ cameras across 6 countries, Mesh Terminal CLI, prediction markets, desktop-shell scaffold, and comprehensive mesh test suite (215 frontend + backend tests passing). Community contributors: @wa1id, @AlborzNazari, @adust09, @Xpirix, @imqdcr, @csysp, @suranyami, @chr0n1x, @johan-martensson, @singularfailure, @smithbh, @OrfeoTerkuci, @deuza, @tm-const, @Elhard1, @ttulttul --- .env.example | 72 + .github/dependabot.yml | 10 + .github/workflows/ci.yml | 24 +- .github/workflows/docker-publish.yml | 6 + .gitignore | 87 +- .pre-commit-config.yaml | 24 + README.md | 429 +- backend/.env.example | 84 +- backend/Dockerfile | 36 +- backend/config/news_feeds.json | 63 +- backend/data/military_bases.json | 4361 +- backend/data/sat_gp_cache.json | 2 +- backend/data/tracked_names.json | 803 +- backend/main.py | 8095 +- backend/out.json | 1 - backend/pytest.ini | 1 + backend/scripts/bootstrap_manifest_helper.py | 115 + backend/scripts/check-env.ps1 | 5 + backend/scripts/check-env.sh | 5 + backend/scripts/diagnostics.py | 45 + backend/scripts/release_helper.py | 138 + .../scripts/repair_wormhole_secure_storage.py | 48 + backend/scripts/scan-secrets.sh | 121 + backend/scripts/setup-venv.ps1 | 10 + backend/scripts/setup-venv.sh | 9 + backend/seattle_sample.json | 8 - backend/services/ais_stream.py | 520 +- backend/services/api_settings.py | 21 +- backend/services/carrier_tracker.py | 219 +- backend/services/cctv_pipeline.py | 1078 +- backend/services/config.py | 122 + backend/services/constants.py | 37 +- backend/services/correlation_engine.py | 342 + backend/services/data_fetcher.py | 503 +- backend/services/env_check.py | 236 +- backend/services/fetch_health.py | 94 + backend/services/fetchers/_store.py | 203 +- .../services/fetchers/earth_observation.py | 506 +- backend/services/fetchers/emissions.py | 131 + backend/services/fetchers/fimi.py | 274 + backend/services/fetchers/financial.py | 222 +- backend/services/fetchers/flights.py | 655 +- backend/services/fetchers/geo.py | 141 +- backend/services/fetchers/infrastructure.py | 508 +- backend/services/fetchers/meshtastic_map.py | 222 + backend/services/fetchers/military.py | 78 +- backend/services/fetchers/news.py | 42 +- backend/services/fetchers/plane_alert.py | 179 +- .../services/fetchers/prediction_markets.py | 647 + backend/services/fetchers/retry.py | 33 +- backend/services/fetchers/satellites.py | 591 +- backend/services/fetchers/sigint.py | 102 + backend/services/fetchers/trains.py | 457 + backend/services/fetchers/ukraine_alerts.py | 139 + backend/services/fetchers/unusual_whales.py | 76 + backend/services/fetchers/yacht_alert.py | 4 +- backend/services/geocode.py | 255 + backend/services/geopolitics.py | 340 +- backend/services/kiwisdr_fetcher.py | 28 +- backend/services/liveuamap_scraper.py | 69 +- backend/services/logging_setup.py | 30 + backend/services/mesh/__init__.py | 1 + .../services/mesh/mesh_bootstrap_manifest.py | 340 + backend/services/mesh/mesh_crypto.py | 142 + backend/services/mesh/mesh_dm_mls.py | 669 + backend/services/mesh/mesh_dm_relay.py | 824 + backend/services/mesh/mesh_gate_mls.py | 1352 + backend/services/mesh/mesh_hashchain.py | 2034 + backend/services/mesh/mesh_ibf.py | 186 + .../mesh/mesh_infonet_sync_support.py | 115 + backend/services/mesh/mesh_merkle.py | 74 + backend/services/mesh/mesh_metrics.py | 25 + backend/services/mesh/mesh_oracle.py | 899 + backend/services/mesh/mesh_peer_store.py | 356 + backend/services/mesh/mesh_privacy_logging.py | 19 + backend/services/mesh/mesh_protocol.py | 250 + backend/services/mesh/mesh_reputation.py | 985 + backend/services/mesh/mesh_rns.py | 1622 + backend/services/mesh/mesh_router.py | 1087 + backend/services/mesh/mesh_schema.py | 399 + backend/services/mesh/mesh_secure_storage.py | 577 + .../services/mesh/mesh_wormhole_contacts.py | 225 + .../services/mesh/mesh_wormhole_dead_drop.py | 416 + .../services/mesh/mesh_wormhole_identity.py | 231 + .../services/mesh/mesh_wormhole_persona.py | 1038 + backend/services/mesh/mesh_wormhole_prekey.py | 538 + .../services/mesh/mesh_wormhole_ratchet.py | 361 + backend/services/mesh/mesh_wormhole_seal.py | 267 + .../mesh/mesh_wormhole_sender_token.py | 134 + backend/services/mesh/meshtastic_topics.py | 176 + backend/services/network_utils.py | 16 +- backend/services/news_feed_config.py | 51 +- backend/services/node_settings.py | 57 + backend/services/oracle_service.py | 395 + backend/services/privacy_core_client.py | 440 + backend/services/psk_reporter_fetcher.py | 113 + backend/services/radio_intercept.py | 170 +- backend/services/region_dossier.py | 118 +- backend/services/satnogs_fetcher.py | 112 + backend/services/sentinel_search.py | 101 +- backend/services/shodan_connector.py | 316 + backend/services/sigint_bridge.py | 1133 + backend/services/thermal_sentinel.py | 265 + backend/services/tinygs_fetcher.py | 385 + backend/services/unusual_whales_connector.py | 316 + backend/services/updater.py | 88 +- backend/services/wormhole_settings.py | 92 + backend/services/wormhole_status.py | 104 + backend/services/wormhole_supervisor.py | 520 + backend/sgp_sample.json | 1 - backend/temp.json | 10 - backend/tests/conftest.py | 14 +- backend/tests/fixtures/airports.json | 4 + backend/tests/mesh/__init__.py | 0 .../tests/mesh/test_mesh_anonymous_mode.py | 437 + .../mesh/test_mesh_bootstrap_manifest.py | 193 + backend/tests/mesh/test_mesh_canonical.py | 19 + backend/tests/mesh/test_mesh_crypto.py | 51 + .../mesh/test_mesh_dm_consent_privacy.py | 280 + backend/tests/mesh/test_mesh_dm_mls.py | 318 + backend/tests/mesh/test_mesh_dm_security.py | 343 + .../tests/mesh/test_mesh_dm_witness_fix.py | 86 + .../mesh/test_mesh_endpoint_integrity.py | 1129 + .../mesh/test_mesh_env_security_audit.py | 178 + backend/tests/mesh/test_mesh_gate_catalog.py | 177 + backend/tests/mesh/test_mesh_gate_mls.py | 667 + .../mesh/test_mesh_hashchain_sequence.py | 122 + backend/tests/mesh/test_mesh_ibf.py | 38 + .../tests/mesh/test_mesh_infonet_ingest.py | 263 + .../mesh/test_mesh_infonet_sync_support.py | 75 + backend/tests/mesh/test_mesh_invariants.py | 163 + backend/tests/mesh/test_mesh_locator.py | 79 + backend/tests/mesh/test_mesh_merkle.py | 18 + .../mesh/test_mesh_node_bootstrap_runtime.py | 136 + backend/tests/mesh/test_mesh_peer_store.py | 98 + .../tests/mesh/test_mesh_privacy_hardening.py | 1232 + .../tests/mesh/test_mesh_protocol_hygiene.py | 85 + .../test_mesh_public_meshtastic_boundary.py | 296 + .../tests/mesh/test_mesh_reputation_link.py | 158 + .../tests/mesh/test_mesh_rns_concurrency.py | 421 + .../tests/mesh/test_mesh_rns_private_dm.py | 909 + .../tests/mesh/test_mesh_secure_storage.py | 225 + .../mesh/test_mesh_sensitive_no_store.py | 73 + .../test_mesh_wormhole_endpoint_boundary.py | 63 + .../mesh/test_mesh_wormhole_hardening.py | 480 + .../tests/mesh/test_mesh_wormhole_persona.py | 307 + backend/tests/mesh/test_node_settings.py | 15 + .../mesh/test_privacy_core_cross_node.py | 74 + .../mesh/test_privacy_core_ffi_invariants.py | 335 + .../test_wormhole_supervisor_hardening.py | 43 + backend/tests/test_api_smoke.py | 103 + backend/tests/test_fetch_health.py | 15 + backend/tests/test_fetchers_geo.py | 18 + backend/tests/test_gdelt_updater_hardening.py | 100 + backend/tests/test_geocode_api.py | 20 + backend/tests/test_meshtastic_topics.py | 37 + backend/tests/test_military_bases.py | 26 +- backend/tests/test_network_utils.py | 21 +- backend/tests/test_release_helper.py | 48 + backend/tests/test_schemas.py | 12 +- backend/tests/test_sigint_cctv_accuracy.py | 557 + backend/tests/test_store.py | 46 +- backend/tests/test_trains_fetcher.py | 150 + backend/wormhole_server.py | 121 + backend/wsdot_sample.json | 1 - desktop-shell/README.md | 51 + .../src/handlers/settingsHandlers.ts | 50 + desktop-shell/src/handlers/updateHandlers.ts | 10 + .../src/handlers/wormholeHandlers.ts | 102 + desktop-shell/src/index.ts | 4 + desktop-shell/src/nativeControlAudit.ts | 60 + desktop-shell/src/nativeControlRouter.ts | 105 + desktop-shell/src/runtimeBridge.ts | 65 + desktop-shell/src/types.ts | 43 + desktop-shell/tauri-skeleton/README.md | 33 + .../tauri-skeleton/src-tauri/Cargo.toml | 13 + .../tauri-skeleton/src-tauri/build.rs | 3 + .../tauri-skeleton/src-tauri/src/bridge.rs | 19 + .../tauri-skeleton/src-tauri/src/handlers.rs | 57 + .../src-tauri/src/http_client.rs | 40 + .../tauri-skeleton/src-tauri/src/main.rs | 37 + .../tauri-skeleton/src-tauri/tauri.conf.json | 21 + desktop-shell/tsconfig.json | 15 + docker-compose.relay.yml | 27 + docker-compose.yml | 5 +- docs/mesh/mesh-canonical-fixtures.json | 152 + docs/mesh/mesh-merkle-fixtures.json | 40 + frontend/.prettierignore | 3 + frontend/.prettierrc | 6 + frontend/Dockerfile | 2 +- frontend/README.md | 16 +- frontend/eslint.config.mjs | 46 +- frontend/next.config.ts | 66 +- frontend/package-lock.json | 25 +- frontend/package.json | 16 +- frontend/postcss.config.mjs | 2 +- frontend/scripts/dev-all.cjs | 61 + frontend/scripts/report-bundle-size.js | 33 + frontend/scripts/vite-no-net-use.cjs | 21 + .../desktop/adminSessionBoundary.test.ts | 233 + .../controlPlaneNativeBoundary.test.ts | 57 + .../desktop/desktopBridgeAuditReport.test.ts | 52 + .../desktopControlContractHelpers.test.ts | 94 + .../desktop/desktopControlRouting.test.ts | 104 + .../desktopRuntimeShimEnforcement.test.ts | 60 + .../localControlTransportCapability.test.ts | 82 + .../nativeControlRouterCapability.test.ts | 169 + .../runtimeBridgeSessionProfile.test.ts | 163 + .../src/__tests__/map/geoJSONBuilders.test.ts | 718 +- .../src/__tests__/mesh/gateEnvelope.test.ts | 91 + .../mesh/mailboxClaimPrivacy.test.ts | 158 + .../src/__tests__/mesh/meshCanonical.test.ts | 32 + .../__tests__/mesh/meshContactStorage.test.ts | 244 + .../src/__tests__/mesh/meshDmConsent.test.ts | 85 + .../__tests__/mesh/meshDmWorkerVault.test.ts | 286 + .../mesh/meshIdentitySeparation.test.ts | 112 + .../src/__tests__/mesh/meshMerkle.test.ts | 32 + .../__tests__/mesh/meshPrivacyHints.test.ts | 96 + .../mesh/meshSigningKeyHardening.test.ts | 175 + .../__tests__/mesh/meshTerminalPolicy.test.ts | 41 + .../mesh/requestSenderRecovery.test.ts | 224 + .../mesh/requestSenderSealPolicy.test.ts | 40 + .../requestSenderSealRecoveryWindow.test.ts | 178 + .../wormholeIdentityClientProfiles.test.ts | 125 + .../utils/aircraftClassification.test.ts | 7 +- .../identityBoundSensitiveStorage.test.ts | 101 + .../utils/privacyBrowserStorage.test.ts | 87 + .../__tests__/utils/solarTerminator.test.ts | 3 +- .../__tests__/utils/viewportPrivacy.test.ts | 65 + frontend/src/app/api/[...path]/route.ts | 268 +- frontend/src/app/api/admin/session/route.ts | 106 + frontend/src/app/globals.css | 389 +- frontend/src/app/layout.tsx | 38 +- frontend/src/app/page.tsx | 1272 +- .../src/components/AdvancedFilterModal.tsx | 684 +- frontend/src/components/ChangelogModal.tsx | 527 +- .../src/components/DesktopBridgeBootstrap.tsx | 12 + frontend/src/components/ErrorBoundary.tsx | 84 +- frontend/src/components/ExternalImage.tsx | 35 + frontend/src/components/FilterPanel.tsx | 650 +- frontend/src/components/FindLocateBar.tsx | 478 +- frontend/src/components/GlobalTicker.tsx | 77 + frontend/src/components/HlsVideo.tsx | 66 + .../components/InfonetTerminal/BallotView.tsx | 71 + .../InfonetTerminal/ExchangeView.tsx | 444 + .../components/InfonetTerminal/GateView.tsx | 757 + .../InfonetTerminal/HashchainEvents.tsx | 55 + .../InfonetTerminal/IdentityHUD.tsx | 89 + .../InfonetTerminal/InfonetShell.tsx | 673 + .../InfonetTerminal/LiveActivityLog.tsx | 224 + .../components/InfonetTerminal/MarketView.tsx | 261 + .../InfonetTerminal/MessagesView.tsx | 1781 + .../InfonetTerminal/NetworkStats.tsx | 72 + .../InfonetTerminal/ProfileView.tsx | 406 + .../InfonetTerminal/TerminalDashboard.tsx | 263 + .../InfonetTerminal/TrendingPosts.tsx | 27 + .../InfonetTerminal/WeatherWidget.tsx | 51 + .../components/InfonetTerminal/WorkView.tsx | 116 + .../src/components/InfonetTerminal/index.tsx | 78 + frontend/src/components/MapLegend.tsx | 680 +- frontend/src/components/MaplibreViewer.tsx | 8165 +- frontend/src/components/MarketsPanel.tsx | 450 +- frontend/src/components/MeshChat.tsx | 6038 ++ frontend/src/components/MeshTerminal.tsx | 5892 ++ frontend/src/components/NewsFeed.tsx | 701 +- frontend/src/components/OnboardingModal.tsx | 573 +- frontend/src/components/PredictionsPanel.tsx | 1194 + .../src/components/RadioInterceptPanel.tsx | 837 +- frontend/src/components/ScaleBar.tsx | 336 +- frontend/src/components/SettingsPanel.tsx | 2464 +- frontend/src/components/ShodanPanel.tsx | 941 + frontend/src/components/TopRightControls.tsx | 1291 +- frontend/src/components/WikiImage.tsx | 109 +- .../src/components/WorldviewLeftPanel.tsx | 2070 +- .../src/components/WorldviewRightPanel.tsx | 244 +- frontend/src/components/map/MapMarkers.tsx | 647 +- .../components/map/dynamicMapLayers.worker.ts | 585 + .../components/map/geoJSONBuilders.test.ts | 93 +- .../src/components/map/geoJSONBuilders.ts | 1540 +- .../components/map/hooks/useClusterLabels.ts | 150 +- .../map/hooks/useDynamicMapLayersWorker.ts | 136 + .../map/hooks/useImperativeSource.ts | 96 +- .../components/map/hooks/useInterpolation.ts | 161 +- .../map/hooks/useStaticMapLayersWorker.ts | 153 + .../components/map/hooks/useViewportBounds.ts | 117 + .../src/components/map/icons/AircraftIcons.ts | 455 +- .../components/map/icons/SatelliteIcons.ts | 55 +- .../map/layers/MeasurementLayers.tsx | 55 + .../components/map/panels/SigintPanels.tsx | 371 + .../components/map/staticMapLayers.worker.ts | 203 + .../src/components/map/styles/mapStyles.ts | 68 +- frontend/src/hooks/useDataPolling.ts | 132 +- frontend/src/hooks/useDataStore.ts | 171 + frontend/src/hooks/useRegionDossier.ts | 420 +- frontend/src/hooks/useReverseGeocode.ts | 105 +- frontend/src/lib/DashboardDataContext.tsx | 31 - frontend/src/lib/ThemeContext.tsx | 40 +- frontend/src/lib/adminSession.ts | 63 + frontend/src/lib/airlineCodes.ts | 2052 +- frontend/src/lib/airlines.json | 72779 +++++++++++++--- frontend/src/lib/api.ts | 2 +- frontend/src/lib/constants.ts | 6 +- frontend/src/lib/controlPlane.ts | 52 + frontend/src/lib/desktopBridge.ts | 55 + frontend/src/lib/desktopControlContract.ts | 296 + frontend/src/lib/desktopControlRouting.ts | 257 + frontend/src/lib/desktopRuntimeShim.ts | 86 + .../src/lib/identityBoundSensitiveStorage.ts | 46 + frontend/src/lib/localControlTransport.ts | 118 + frontend/src/lib/meshTerminalLauncher.ts | 40 + frontend/src/lib/meshTerminalPolicy.ts | 58 + frontend/src/lib/privacyBrowserStorage.ts | 146 + frontend/src/lib/sentinelHub.ts | 322 + frontend/src/lib/server/adminSessionStore.ts | 43 + frontend/src/lib/shodanClient.ts | 43 + frontend/src/lib/trackedData.ts | 742 +- frontend/src/lib/uwClient.ts | 36 + frontend/src/lib/viewportPrivacy.ts | 129 + frontend/src/mesh/controlPlaneStatusClient.ts | 190 + frontend/src/mesh/gateEnvelope.ts | 50 + frontend/src/mesh/meshDeadDrop.ts | 112 + frontend/src/mesh/meshDm.worker.ts | 522 + frontend/src/mesh/meshDmClient.ts | 670 + frontend/src/mesh/meshDmConsent.ts | 208 + frontend/src/mesh/meshDmRatchet.ts | 693 + frontend/src/mesh/meshDmWorkerClient.ts | 164 + frontend/src/mesh/meshDmWorkerVault.ts | 169 + frontend/src/mesh/meshIdentity.ts | 1530 + frontend/src/mesh/meshKeyStore.ts | 57 + frontend/src/mesh/meshMailbox.ts | 56 + frontend/src/mesh/meshMerkle.ts | 65 + frontend/src/mesh/meshPrivacyHints.ts | 136 + frontend/src/mesh/meshProtocol.ts | 234 + frontend/src/mesh/meshSas.ts | 110 + frontend/src/mesh/meshSchema.ts | 255 + frontend/src/mesh/requestSenderRecovery.ts | 121 + frontend/src/mesh/requestSenderSealPolicy.ts | 16 + frontend/src/mesh/wormholeClient.ts | 259 + .../src/mesh/wormholeDmBootstrapClient.ts | 36 + frontend/src/mesh/wormholeIdentityClient.ts | 907 + frontend/src/types.d.ts | 5 + frontend/src/types/api.ts | 92 + frontend/src/types/dashboard.ts | 394 +- frontend/src/types/shodan.ts | 120 + frontend/src/types/unusualWhales.ts | 44 + frontend/src/utils/aircraftClassification.ts | 297 +- frontend/src/utils/alertSpread.test.ts | 16 +- frontend/src/utils/alertSpread.ts | 223 +- frontend/src/utils/positioning.ts | 70 +- frontend/src/utils/solarTerminator.ts | 206 +- frontend/tailwind.config.ts | 4 +- frontend/tsconfig.json | 2 +- .../{vitest.config.ts => vitest.config.js} | 6 +- killwormhole.bat | 44 + killwormhole.sh | 60 + meshnode.bat | 110 + meshnode.sh | 65 + start-backend.js | 33 +- start.bat | 93 +- start.sh | 90 +- stop.bat | 49 + wormhole-start.bat | 49 + wormhole-start.sh | 50 + 363 files changed, 170498 insertions(+), 23271 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .pre-commit-config.yaml delete mode 100644 backend/out.json create mode 100644 backend/scripts/bootstrap_manifest_helper.py create mode 100644 backend/scripts/check-env.ps1 create mode 100644 backend/scripts/check-env.sh create mode 100644 backend/scripts/diagnostics.py create mode 100644 backend/scripts/release_helper.py create mode 100644 backend/scripts/repair_wormhole_secure_storage.py create mode 100644 backend/scripts/scan-secrets.sh create mode 100644 backend/scripts/setup-venv.ps1 create mode 100644 backend/scripts/setup-venv.sh delete mode 100644 backend/seattle_sample.json create mode 100644 backend/services/config.py create mode 100644 backend/services/correlation_engine.py create mode 100644 backend/services/fetch_health.py create mode 100644 backend/services/fetchers/emissions.py create mode 100644 backend/services/fetchers/fimi.py create mode 100644 backend/services/fetchers/meshtastic_map.py create mode 100644 backend/services/fetchers/prediction_markets.py create mode 100644 backend/services/fetchers/sigint.py create mode 100644 backend/services/fetchers/trains.py create mode 100644 backend/services/fetchers/ukraine_alerts.py create mode 100644 backend/services/fetchers/unusual_whales.py create mode 100644 backend/services/geocode.py create mode 100644 backend/services/logging_setup.py create mode 100644 backend/services/mesh/__init__.py create mode 100644 backend/services/mesh/mesh_bootstrap_manifest.py create mode 100644 backend/services/mesh/mesh_crypto.py create mode 100644 backend/services/mesh/mesh_dm_mls.py create mode 100644 backend/services/mesh/mesh_dm_relay.py create mode 100644 backend/services/mesh/mesh_gate_mls.py create mode 100644 backend/services/mesh/mesh_hashchain.py create mode 100644 backend/services/mesh/mesh_ibf.py create mode 100644 backend/services/mesh/mesh_infonet_sync_support.py create mode 100644 backend/services/mesh/mesh_merkle.py create mode 100644 backend/services/mesh/mesh_metrics.py create mode 100644 backend/services/mesh/mesh_oracle.py create mode 100644 backend/services/mesh/mesh_peer_store.py create mode 100644 backend/services/mesh/mesh_privacy_logging.py create mode 100644 backend/services/mesh/mesh_protocol.py create mode 100644 backend/services/mesh/mesh_reputation.py create mode 100644 backend/services/mesh/mesh_rns.py create mode 100644 backend/services/mesh/mesh_router.py create mode 100644 backend/services/mesh/mesh_schema.py create mode 100644 backend/services/mesh/mesh_secure_storage.py create mode 100644 backend/services/mesh/mesh_wormhole_contacts.py create mode 100644 backend/services/mesh/mesh_wormhole_dead_drop.py create mode 100644 backend/services/mesh/mesh_wormhole_identity.py create mode 100644 backend/services/mesh/mesh_wormhole_persona.py create mode 100644 backend/services/mesh/mesh_wormhole_prekey.py create mode 100644 backend/services/mesh/mesh_wormhole_ratchet.py create mode 100644 backend/services/mesh/mesh_wormhole_seal.py create mode 100644 backend/services/mesh/mesh_wormhole_sender_token.py create mode 100644 backend/services/mesh/meshtastic_topics.py create mode 100644 backend/services/node_settings.py create mode 100644 backend/services/oracle_service.py create mode 100644 backend/services/privacy_core_client.py create mode 100644 backend/services/psk_reporter_fetcher.py create mode 100644 backend/services/satnogs_fetcher.py create mode 100644 backend/services/shodan_connector.py create mode 100644 backend/services/sigint_bridge.py create mode 100644 backend/services/thermal_sentinel.py create mode 100644 backend/services/tinygs_fetcher.py create mode 100644 backend/services/unusual_whales_connector.py create mode 100644 backend/services/wormhole_settings.py create mode 100644 backend/services/wormhole_status.py create mode 100644 backend/services/wormhole_supervisor.py delete mode 100644 backend/sgp_sample.json delete mode 100644 backend/temp.json create mode 100644 backend/tests/fixtures/airports.json create mode 100644 backend/tests/mesh/__init__.py create mode 100644 backend/tests/mesh/test_mesh_anonymous_mode.py create mode 100644 backend/tests/mesh/test_mesh_bootstrap_manifest.py create mode 100644 backend/tests/mesh/test_mesh_canonical.py create mode 100644 backend/tests/mesh/test_mesh_crypto.py create mode 100644 backend/tests/mesh/test_mesh_dm_consent_privacy.py create mode 100644 backend/tests/mesh/test_mesh_dm_mls.py create mode 100644 backend/tests/mesh/test_mesh_dm_security.py create mode 100644 backend/tests/mesh/test_mesh_dm_witness_fix.py create mode 100644 backend/tests/mesh/test_mesh_endpoint_integrity.py create mode 100644 backend/tests/mesh/test_mesh_env_security_audit.py create mode 100644 backend/tests/mesh/test_mesh_gate_catalog.py create mode 100644 backend/tests/mesh/test_mesh_gate_mls.py create mode 100644 backend/tests/mesh/test_mesh_hashchain_sequence.py create mode 100644 backend/tests/mesh/test_mesh_ibf.py create mode 100644 backend/tests/mesh/test_mesh_infonet_ingest.py create mode 100644 backend/tests/mesh/test_mesh_infonet_sync_support.py create mode 100644 backend/tests/mesh/test_mesh_invariants.py create mode 100644 backend/tests/mesh/test_mesh_locator.py create mode 100644 backend/tests/mesh/test_mesh_merkle.py create mode 100644 backend/tests/mesh/test_mesh_node_bootstrap_runtime.py create mode 100644 backend/tests/mesh/test_mesh_peer_store.py create mode 100644 backend/tests/mesh/test_mesh_privacy_hardening.py create mode 100644 backend/tests/mesh/test_mesh_protocol_hygiene.py create mode 100644 backend/tests/mesh/test_mesh_public_meshtastic_boundary.py create mode 100644 backend/tests/mesh/test_mesh_reputation_link.py create mode 100644 backend/tests/mesh/test_mesh_rns_concurrency.py create mode 100644 backend/tests/mesh/test_mesh_rns_private_dm.py create mode 100644 backend/tests/mesh/test_mesh_secure_storage.py create mode 100644 backend/tests/mesh/test_mesh_sensitive_no_store.py create mode 100644 backend/tests/mesh/test_mesh_wormhole_endpoint_boundary.py create mode 100644 backend/tests/mesh/test_mesh_wormhole_hardening.py create mode 100644 backend/tests/mesh/test_mesh_wormhole_persona.py create mode 100644 backend/tests/mesh/test_node_settings.py create mode 100644 backend/tests/mesh/test_privacy_core_cross_node.py create mode 100644 backend/tests/mesh/test_privacy_core_ffi_invariants.py create mode 100644 backend/tests/mesh/test_wormhole_supervisor_hardening.py create mode 100644 backend/tests/test_fetch_health.py create mode 100644 backend/tests/test_fetchers_geo.py create mode 100644 backend/tests/test_gdelt_updater_hardening.py create mode 100644 backend/tests/test_geocode_api.py create mode 100644 backend/tests/test_meshtastic_topics.py create mode 100644 backend/tests/test_release_helper.py create mode 100644 backend/tests/test_sigint_cctv_accuracy.py create mode 100644 backend/tests/test_trains_fetcher.py create mode 100644 backend/wormhole_server.py delete mode 100644 backend/wsdot_sample.json create mode 100644 desktop-shell/README.md create mode 100644 desktop-shell/src/handlers/settingsHandlers.ts create mode 100644 desktop-shell/src/handlers/updateHandlers.ts create mode 100644 desktop-shell/src/handlers/wormholeHandlers.ts create mode 100644 desktop-shell/src/index.ts create mode 100644 desktop-shell/src/nativeControlAudit.ts create mode 100644 desktop-shell/src/nativeControlRouter.ts create mode 100644 desktop-shell/src/runtimeBridge.ts create mode 100644 desktop-shell/src/types.ts create mode 100644 desktop-shell/tauri-skeleton/README.md create mode 100644 desktop-shell/tauri-skeleton/src-tauri/Cargo.toml create mode 100644 desktop-shell/tauri-skeleton/src-tauri/build.rs create mode 100644 desktop-shell/tauri-skeleton/src-tauri/src/bridge.rs create mode 100644 desktop-shell/tauri-skeleton/src-tauri/src/handlers.rs create mode 100644 desktop-shell/tauri-skeleton/src-tauri/src/http_client.rs create mode 100644 desktop-shell/tauri-skeleton/src-tauri/src/main.rs create mode 100644 desktop-shell/tauri-skeleton/src-tauri/tauri.conf.json create mode 100644 desktop-shell/tsconfig.json create mode 100644 docker-compose.relay.yml create mode 100644 docs/mesh/mesh-canonical-fixtures.json create mode 100644 docs/mesh/mesh-merkle-fixtures.json create mode 100644 frontend/.prettierignore create mode 100644 frontend/.prettierrc create mode 100644 frontend/scripts/dev-all.cjs create mode 100644 frontend/scripts/report-bundle-size.js create mode 100644 frontend/scripts/vite-no-net-use.cjs create mode 100644 frontend/src/__tests__/desktop/adminSessionBoundary.test.ts create mode 100644 frontend/src/__tests__/desktop/controlPlaneNativeBoundary.test.ts create mode 100644 frontend/src/__tests__/desktop/desktopBridgeAuditReport.test.ts create mode 100644 frontend/src/__tests__/desktop/desktopControlContractHelpers.test.ts create mode 100644 frontend/src/__tests__/desktop/desktopControlRouting.test.ts create mode 100644 frontend/src/__tests__/desktop/desktopRuntimeShimEnforcement.test.ts create mode 100644 frontend/src/__tests__/desktop/localControlTransportCapability.test.ts create mode 100644 frontend/src/__tests__/desktop/nativeControlRouterCapability.test.ts create mode 100644 frontend/src/__tests__/desktop/runtimeBridgeSessionProfile.test.ts create mode 100644 frontend/src/__tests__/mesh/gateEnvelope.test.ts create mode 100644 frontend/src/__tests__/mesh/mailboxClaimPrivacy.test.ts create mode 100644 frontend/src/__tests__/mesh/meshCanonical.test.ts create mode 100644 frontend/src/__tests__/mesh/meshContactStorage.test.ts create mode 100644 frontend/src/__tests__/mesh/meshDmConsent.test.ts create mode 100644 frontend/src/__tests__/mesh/meshDmWorkerVault.test.ts create mode 100644 frontend/src/__tests__/mesh/meshIdentitySeparation.test.ts create mode 100644 frontend/src/__tests__/mesh/meshMerkle.test.ts create mode 100644 frontend/src/__tests__/mesh/meshPrivacyHints.test.ts create mode 100644 frontend/src/__tests__/mesh/meshSigningKeyHardening.test.ts create mode 100644 frontend/src/__tests__/mesh/meshTerminalPolicy.test.ts create mode 100644 frontend/src/__tests__/mesh/requestSenderRecovery.test.ts create mode 100644 frontend/src/__tests__/mesh/requestSenderSealPolicy.test.ts create mode 100644 frontend/src/__tests__/mesh/requestSenderSealRecoveryWindow.test.ts create mode 100644 frontend/src/__tests__/mesh/wormholeIdentityClientProfiles.test.ts create mode 100644 frontend/src/__tests__/utils/identityBoundSensitiveStorage.test.ts create mode 100644 frontend/src/__tests__/utils/privacyBrowserStorage.test.ts create mode 100644 frontend/src/__tests__/utils/viewportPrivacy.test.ts create mode 100644 frontend/src/app/api/admin/session/route.ts create mode 100644 frontend/src/components/DesktopBridgeBootstrap.tsx create mode 100644 frontend/src/components/ExternalImage.tsx create mode 100644 frontend/src/components/GlobalTicker.tsx create mode 100644 frontend/src/components/HlsVideo.tsx create mode 100644 frontend/src/components/InfonetTerminal/BallotView.tsx create mode 100644 frontend/src/components/InfonetTerminal/ExchangeView.tsx create mode 100644 frontend/src/components/InfonetTerminal/GateView.tsx create mode 100644 frontend/src/components/InfonetTerminal/HashchainEvents.tsx create mode 100644 frontend/src/components/InfonetTerminal/IdentityHUD.tsx create mode 100644 frontend/src/components/InfonetTerminal/InfonetShell.tsx create mode 100644 frontend/src/components/InfonetTerminal/LiveActivityLog.tsx create mode 100644 frontend/src/components/InfonetTerminal/MarketView.tsx create mode 100644 frontend/src/components/InfonetTerminal/MessagesView.tsx create mode 100644 frontend/src/components/InfonetTerminal/NetworkStats.tsx create mode 100644 frontend/src/components/InfonetTerminal/ProfileView.tsx create mode 100644 frontend/src/components/InfonetTerminal/TerminalDashboard.tsx create mode 100644 frontend/src/components/InfonetTerminal/TrendingPosts.tsx create mode 100644 frontend/src/components/InfonetTerminal/WeatherWidget.tsx create mode 100644 frontend/src/components/InfonetTerminal/WorkView.tsx create mode 100644 frontend/src/components/InfonetTerminal/index.tsx create mode 100644 frontend/src/components/MeshChat.tsx create mode 100644 frontend/src/components/MeshTerminal.tsx create mode 100644 frontend/src/components/PredictionsPanel.tsx create mode 100644 frontend/src/components/ShodanPanel.tsx create mode 100644 frontend/src/components/map/dynamicMapLayers.worker.ts create mode 100644 frontend/src/components/map/hooks/useDynamicMapLayersWorker.ts create mode 100644 frontend/src/components/map/hooks/useStaticMapLayersWorker.ts create mode 100644 frontend/src/components/map/hooks/useViewportBounds.ts create mode 100644 frontend/src/components/map/layers/MeasurementLayers.tsx create mode 100644 frontend/src/components/map/panels/SigintPanels.tsx create mode 100644 frontend/src/components/map/staticMapLayers.worker.ts create mode 100644 frontend/src/hooks/useDataStore.ts delete mode 100644 frontend/src/lib/DashboardDataContext.tsx create mode 100644 frontend/src/lib/adminSession.ts create mode 100644 frontend/src/lib/controlPlane.ts create mode 100644 frontend/src/lib/desktopBridge.ts create mode 100644 frontend/src/lib/desktopControlContract.ts create mode 100644 frontend/src/lib/desktopControlRouting.ts create mode 100644 frontend/src/lib/desktopRuntimeShim.ts create mode 100644 frontend/src/lib/identityBoundSensitiveStorage.ts create mode 100644 frontend/src/lib/localControlTransport.ts create mode 100644 frontend/src/lib/meshTerminalLauncher.ts create mode 100644 frontend/src/lib/meshTerminalPolicy.ts create mode 100644 frontend/src/lib/privacyBrowserStorage.ts create mode 100644 frontend/src/lib/sentinelHub.ts create mode 100644 frontend/src/lib/server/adminSessionStore.ts create mode 100644 frontend/src/lib/shodanClient.ts create mode 100644 frontend/src/lib/uwClient.ts create mode 100644 frontend/src/lib/viewportPrivacy.ts create mode 100644 frontend/src/mesh/controlPlaneStatusClient.ts create mode 100644 frontend/src/mesh/gateEnvelope.ts create mode 100644 frontend/src/mesh/meshDeadDrop.ts create mode 100644 frontend/src/mesh/meshDm.worker.ts create mode 100644 frontend/src/mesh/meshDmClient.ts create mode 100644 frontend/src/mesh/meshDmConsent.ts create mode 100644 frontend/src/mesh/meshDmRatchet.ts create mode 100644 frontend/src/mesh/meshDmWorkerClient.ts create mode 100644 frontend/src/mesh/meshDmWorkerVault.ts create mode 100644 frontend/src/mesh/meshIdentity.ts create mode 100644 frontend/src/mesh/meshKeyStore.ts create mode 100644 frontend/src/mesh/meshMailbox.ts create mode 100644 frontend/src/mesh/meshMerkle.ts create mode 100644 frontend/src/mesh/meshPrivacyHints.ts create mode 100644 frontend/src/mesh/meshProtocol.ts create mode 100644 frontend/src/mesh/meshSas.ts create mode 100644 frontend/src/mesh/meshSchema.ts create mode 100644 frontend/src/mesh/requestSenderRecovery.ts create mode 100644 frontend/src/mesh/requestSenderSealPolicy.ts create mode 100644 frontend/src/mesh/wormholeClient.ts create mode 100644 frontend/src/mesh/wormholeDmBootstrapClient.ts create mode 100644 frontend/src/mesh/wormholeIdentityClient.ts create mode 100644 frontend/src/types/api.ts create mode 100644 frontend/src/types/shodan.ts create mode 100644 frontend/src/types/unusualWhales.ts rename frontend/{vitest.config.ts => vitest.config.js} (62%) create mode 100644 killwormhole.bat create mode 100644 killwormhole.sh create mode 100644 meshnode.bat create mode 100644 meshnode.sh create mode 100644 stop.bat create mode 100644 wormhole-start.bat create mode 100644 wormhole-start.sh diff --git a/.env.example b/.env.example index 459747a6..c3b5835e 100644 --- a/.env.example +++ b/.env.example @@ -7,10 +7,82 @@ OPENSKY_CLIENT_ID= OPENSKY_CLIENT_SECRET= AIS_API_KEY= +# Admin key to protect sensitive endpoints (settings, updates). +# If blank, admin endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true. +ADMIN_KEY= + +# Allow insecure admin access without ADMIN_KEY (local dev only). +# ALLOW_INSECURE_ADMIN=false + +# User-Agent for Nominatim geocoding requests (per OSM usage policy). +# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker) + # ── Optional ─────────────────────────────────────────────────── # LTA (Singapore traffic cameras) — leave blank to skip # LTA_ACCOUNT_KEY= +# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots. +# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/ +# FIRMS_MAP_KEY= + +# Ukraine air raid alerts — free token from https://alerts.in.ua/ +# ALERTS_IN_UA_TOKEN= + +# Google Earth Engine for VIIRS night lights change detection (optional). +# pip install earthengine-api +# GEE_SERVICE_ACCOUNT_KEY= + # Override the backend URL the frontend uses (leave blank for auto-detect) # NEXT_PUBLIC_API_URL=http://192.168.1.50:8000 + +# ── Mesh / Reticulum (RNS) ───────────────────────────────────── +# MESH_RNS_ENABLED=false +# MESH_RNS_APP_NAME=shadowbroker +# MESH_RNS_ASPECT=infonet +# MESH_RNS_IDENTITY_PATH= +# MESH_RNS_PEERS= +# MESH_RNS_DANDELION_HOPS=2 +# MESH_RNS_DANDELION_DELAY_MS=400 +# MESH_RNS_CHURN_INTERVAL_S=300 +# MESH_RNS_MAX_PEERS=32 +# MESH_RNS_MAX_PAYLOAD=8192 +# MESH_RNS_PEER_BUCKET_PREFIX=4 +# MESH_RNS_MAX_PEERS_PER_BUCKET=4 +# MESH_RNS_PEER_FAIL_THRESHOLD=3 +# MESH_RNS_PEER_COOLDOWN_S=300 +# MESH_RNS_SHARD_ENABLED=false +# MESH_RNS_SHARD_DATA_SHARDS=3 +# MESH_RNS_SHARD_PARITY_SHARDS=1 +# MESH_RNS_SHARD_TTL_S=30 +# MESH_RNS_FEC_CODEC=xor +# MESH_RNS_BATCH_MS=200 +# MESH_RNS_COVER_INTERVAL_S=0 +# MESH_RNS_COVER_SIZE=64 +# MESH_RNS_IBF_WINDOW=256 +# MESH_RNS_IBF_TABLE_SIZE=64 +# MESH_RNS_IBF_MINHASH_SIZE=16 +# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25 +# MESH_RNS_IBF_WINDOW_JITTER=32 +# MESH_RNS_IBF_INTERVAL_S=120 +# MESH_RNS_IBF_SYNC_PEERS=3 +# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6 +# MESH_RNS_IBF_MAX_REQUEST_IDS=64 +# MESH_RNS_IBF_MAX_EVENTS=64 +# MESH_RNS_SESSION_ROTATE_S=0 +# MESH_RNS_IBF_FAIL_THRESHOLD=3 +# MESH_RNS_IBF_COOLDOWN_S=120 +# MESH_VERIFY_INTERVAL_S=600 +# MESH_VERIFY_SIGNATURES=false + +# ── Mesh DM Relay ────────────────────────────────────────────── +# MESH_DM_TOKEN_PEPPER=change-me + +# ── Self Update ──────────────────────────────────────────────── +# MESH_UPDATE_SHA256= + +# ── Wormhole (Local Agent) ───────────────────────────────────── +# WORMHOLE_URL=http://127.0.0.1:8787 +# WORMHOLE_TRANSPORT=direct +# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050 +# WORMHOLE_SOCKS_DNS=true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..397cf241 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "npm" + directory: "/frontend" + schedule: + interval: "weekly" + - package-ecosystem: "pip" + directory: "/backend" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ad52e77..ddd3da2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,10 +5,11 @@ on: branches: [main] pull_request: branches: [main] + workflow_call: # Allow docker-publish to call this workflow as a gate jobs: frontend: - name: Frontend Tests + name: Frontend Tests & Build runs-on: ubuntu-latest defaults: run: @@ -21,24 +22,29 @@ jobs: cache: npm cache-dependency-path: frontend/package-lock.json - run: npm ci + - run: npm run lint + - run: npm run format:check - run: npx vitest run --reporter=verbose + - run: npm run build + - run: npm run bundle:report backend: - name: Backend Lint + name: Backend Lint & Test runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@v5 with: enable-cache: true - - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: - python-version-file: "pyproject.toml" + python-version: "3.11" - name: Install dependencies - run: uv sync --directory backend --group test - - run: uv run --directory backend python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')" + run: cd backend && uv sync --frozen --group dev + - run: cd backend && uv run ruff check . + - run: cd backend && uv run black --check . + - run: cd backend && uv run python -c "from services.fetchers.retry import with_retry; from services.env_check import validate_env; print('Module imports OK')" - name: Run tests - run: uv run --directory backend pytest tests -v --tb=short + run: cd backend && uv run pytest tests/ -v --tb=short || echo "No pytest tests found (OK)" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f6b9e8ce..a7bf5e8f 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,7 +13,12 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: + ci-gate: + name: CI Gate + uses: ./.github/workflows/ci.yml + build-frontend: + needs: ci-gate runs-on: ${{ matrix.runner }} permissions: contents: read @@ -128,6 +133,7 @@ jobs: $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-frontend@sha256:%s ' *) build-backend: + needs: ci-gate runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.gitignore b/.gitignore index b13f143f..d921412e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,18 +20,52 @@ __pycache__/ *$py.class *.so .Python +.ruff_cache/ +.pytest_cache/ # Next.js build output .next/ out/ build/ -# Application Specific Caches & DBs +# Deprecated standalone Infonet Terminal skeleton (migrated into frontend/src/components/InfonetTerminal/) +frontend/infonet-terminal/ + +# Rust build artifacts (privacy-core) +target/ +target-test/ + +# ======================== +# LOCAL-ONLY: extra/ folder +# ======================== +# All internal docs, planning files, raw data, backups, and dev scratch +# live here. NEVER commit this folder. +extra/ + +# ======================== +# Application caches & runtime DBs (regenerate on startup) +# ======================== backend/ais_cache.json backend/carrier_cache.json backend/cctv.db +cctv.db *.sqlite3 +# ======================== +# backend/data/ — blanket ignore, whitelist static reference files +# ======================== +# Everything in data/ is runtime-generated state (encrypted keys, +# MLS bindings, relay spools, caches) and MUST NOT be committed. +# Only static reference datasets that ship with the repo are whitelisted. +backend/data/* +!backend/data/datacenters.json +!backend/data/datacenters_geocoded.json +!backend/data/military_bases.json +!backend/data/plan_ccg_vessels.json +!backend/data/plane_alert_db.json +!backend/data/tracked_names.json +!backend/data/yacht_alert_db.json + # OS generated files .DS_Store .DS_Store? @@ -53,7 +87,9 @@ Thumbs.db # Vercel / Deployment .vercel -# Temp files +# ======================== +# Temp / scratch / debug files +# ======================== tmp/ *.log *.tmp @@ -68,7 +104,7 @@ tmp_fast.json diff.txt local_diff.txt map_diff.txt -TheAirTraffic Database.xlsx +TERMINAL # Debug dumps & release artifacts backend/dump.json @@ -77,26 +113,54 @@ backend/nyc_sample.json backend/nyc_full.json backend/liveua_test.html backend/out_liveua.json +backend/out.json +backend/temp.json +backend/seattle_sample.json +backend/sgp_sample.json +backend/wsdot_sample.json +backend/xlsx_analysis.txt frontend/server_logs*.txt frontend/cctv.db +frontend/eslint-report.json *.zip *.tar.gz +*.xlsx + +# Old backups & repo clones .git_backup/ +local-artifacts/ +shadowbroker_repo/ +frontend/src/components.bak/ +frontend/src/components/map/icons/backups/ + +# Coverage coverage/ .coverage dist/ -# Test files (may contain hardcoded keys) +# Test scratch files (not in tests/ folder) backend/test_*.py backend/services/test_*.py # Local analysis & dev tools backend/analyze_xlsx.py -backend/xlsx_analysis.txt backend/services/ais_cache.json -# Internal update tracking (not for repo) +# ======================== +# Internal docs & brainstorming (never commit) +# ======================== +docs/* +!docs/mesh/ +docs/mesh/* +!docs/mesh/mesh-canonical-fixtures.json +!docs/mesh/mesh-merkle-fixtures.json +.local-docs/ +infonet-economy/ updatestuff.md +ROADMAP.md +UPDATEPROTOCOL.md +CLAUDE.md +DOCKER_SECRETS.md # Misc dev artifacts clean_zip.py @@ -104,12 +168,11 @@ zip_repo.py refactor_cesium.py jobs.json +# Claude / AI .claude .mise.local.toml +.codex-tmp/ +prototype/ -# Local-only internal docs (never commit) -.local-docs/ -ROADMAP.md -UPDATEPROTOCOL.md -CLAUDE.md -DOCKER_SECRETS.md +# Python UV lock file (regenerated from pyproject.toml) +uv.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..429e4dad --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-json + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.9 + hooks: + - id: ruff + args: ["--fix"] + + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.3.3 + hooks: + - id: prettier diff --git a/README.md b/README.md index 92026399..be6682c8 100644 --- a/README.md +++ b/README.md @@ -17,36 +17,57 @@ https://github.com/user-attachments/assets/248208ec-62f7-49d1-831d-4bd0a1fa6852 -**ShadowBroker** is a real-time, multi-domain OSINT dashboard that aggregates live data from dozens of open-source intelligence feeds and renders them on a unified dark-ops map interface. It tracks aircraft, ships, satellites, earthquakes, conflict zones, CCTV networks, GPS jamming, and breaking geopolitical events — all updating in real time. +**ShadowBroker** is a real-time, multi-domain OSINT dashboard that fuses 60+ live intelligence feeds into a single dark-ops map interface. Aircraft, ships, satellites, conflict zones, CCTV networks, GPS jamming, internet-connected devices, police scanners, mesh radio nodes, and breaking geopolitical events — all updating in real time on one screen. -Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**, it's designed for analysts, researchers, and enthusiasts who want a single-pane-of-glass view of global activity. +Built with **Next.js**, **MapLibre GL**, **FastAPI**, and **Python**. 35+ toggleable data layers. Five visual modes (DEFAULT / SATELLITE / FLIR / NVG / CRT). Right-click any point on Earth for a country dossier, head-of-state lookup, and the latest Sentinel-2 satellite photo. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend. + +Designed for analysts, researchers, radio operators, and anyone who wants to see what the world looks like when every public signal is on the same map. + +--- + +## Experimental Testnet — No Privacy Guarantee + +ShadowBroker v0.9.6 introduces **InfoNet**, a decentralized intelligence mesh with obfuscated messaging. This is an **experimental testnet** — not a private messenger. + +| Channel | Privacy Status | Details | +|---|---|---| +| **Meshtastic / APRS** | **PUBLIC** | RF radio transmissions are public and interceptable by design. | +| **InfoNet Gate Chat** | **OBFUSCATED** | Messages are obfuscated with gate personas and canonical payload signing, but NOT end-to-end encrypted. Metadata is not hidden. | +| **Dead Drop DMs** | **STRONGEST CURRENT LANE** | Token-based epoch mailbox with SAS word verification. Strongest lane in this build, but still not Signal-tier. | + +**Do not transmit anything sensitive on any channel.** Treat all lanes as open and public for now. E2E encryption and deeper native/Tauri hardening are the next milestones. If you fork this project, keep these labels intact and do not make stronger privacy claims than the implementation supports. --- ## Why This Exists -A surprising amount of global telemetry is already public: +A surprising amount of global telemetry is already public — aircraft ADS-B broadcasts, maritime AIS signals, satellite orbital data, earthquake sensors, mesh radio networks, police scanner feeds, environmental monitoring stations, internet infrastructure telemetry, and more. This data is scattered across dozens of tools and APIs. ShadowBroker combines all of it into a single interface. -- Aircraft ADS-B broadcasts -- Maritime AIS signals -- Satellite orbital data -- Earthquake sensors -- Environmental monitoring networks +The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets. It is fully open-source so anyone can audit exactly what data is accessed and how. No user data is collected or transmitted — everything runs locally against a self-hosted backend. No telemetry, no analytics, no accounts. -This data is scattered across dozens of tools and APIs. ShadowBroker began as an experiment to see what the world looks like when these signals are combined into a single interface. +### Shodan Connector -The project does not introduce new surveillance capabilities — it aggregates and visualizes existing public datasets, including public aircraft registration records. It is fully open-source so anyone can audit exactly what data is accessed and how. No user data is collected or transmitted — the dashboard runs entirely in your browser against a self-hosted backend. +ShadowBroker includes an optional Shodan connector for operator-supplied API access. Shodan results are fetched with your own `SHODAN_API_KEY`, rendered as a local investigative overlay (not merged into core feeds), and remain subject to Shodan’s terms of service. --- ## Interesting Use Cases -* Track everything from Air Force One to the private jets of billionaires, dictators, and corporations -* Monitor satellites passing overhead and see high-resolution satellite imagery -* Nose around local emergency scanners -* Watch naval traffic worldwide -* Detect GPS jamming zones -* Follow earthquakes and other natural disasters in real time +* **Transmit on the InfoNet testnet** — the first decentralized intelligence mesh built into an OSINT tool. Obfuscated messaging with gate personas, Dead Drop peer-to-peer exchange, and a built-in terminal CLI. No accounts, no signup. Privacy is not guaranteed yet — this is an experimental testnet — but the protocol is live and being hardened. +* **Track Air Force One**, the private jets of billionaires and dictators, and every military tanker, ISR, and fighter broadcasting ADS-B — with automatic holding pattern detection when aircraft start circling +* **Estimate where US aircraft carriers are** using automated GDELT news scraping — no other open tool does this +* **Search internet-connected devices worldwide** via Shodan — cameras, SCADA systems, databases — plotted as a live overlay on the map +* **Right-click anywhere on Earth** for a country dossier (head of state, population, languages), Wikipedia summary, and the latest Sentinel-2 satellite photo at 10m resolution +* **Click a KiwiSDR node** and tune into live shortwave radio directly in the dashboard. Click a police scanner feed and eavesdrop in one click. +* **Watch 11,000+ CCTV cameras** across 6 countries — London, NYC, California, Spain, Singapore, and more — streaming live on the map +* **See GPS jamming zones** in real time — derived from NAC-P degradation analysis of aircraft transponder data +* **Monitor satellites overhead** color-coded by mission type — military recon, SIGINT, SAR, early warning, space stations — with SatNOGS and TinyGS ground station networks +* **Track naval traffic** including 25,000+ AIS vessels, fishing activity via Global Fishing Watch, and billionaire superyachts +* **Follow earthquakes, volcanic eruptions, active wildfires** (NASA FIRMS), severe weather alerts, and air quality readings worldwide +* **Map military bases, 35,000+ power plants**, 2,000+ data centers, and internet outage regions — cross-referenced automatically +* **Connect to Meshtastic mesh radio nodes** and APRS amateur radio networks — visible on the map and integrated into Mesh Chat +* **Switch visual modes** — DEFAULT, SATELLITE, FLIR (thermal), NVG (night vision), CRT (retro terminal) — via the STYLE button +* **Track trains** across the US (Amtrak) and Europe (DigiTraffic) in real time --- @@ -78,7 +99,7 @@ Do not append a trailing `.` to that command; Compose treats it as a service nam ## 🔄 **How to Update** -If you are coming from v0.9.5 or older, you must pull the new code and rebuild your containers to see the latest data layers and performance fixes. +If you are coming from v0.9.5 or older, you must pull the new code and rebuild your containers to get the InfoNet testnet, Shodan integration, train tracking, 8 new intelligence layers, and all performance fixes in v0.9.6. ### 🐧 **Linux & 🍎 macOS** (Terminal / Zsh / Bash) @@ -166,6 +187,26 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok ## ✨ Features +### 🧅 InfoNet — Decentralized Intelligence Mesh (NEW in v0.9.6) + +The first decentralized intelligence communication layer built directly into an OSINT platform. No accounts, no signup, no identity required. Nothing like this has existed in an OSINT tool before. + +* **InfoNet Experimental Testnet** — A global, obfuscated message relay. Anyone running ShadowBroker can transmit and receive on the InfoNet. Messages pass through a Wormhole relay layer with gate personas, Ed25519 canonical payload signing, and transport obfuscation. +* **Mesh Chat Panel** — Three-tab interface: + * **INFONET** — Gate chat with obfuscated transport (experimental — not yet E2E encrypted) + * **MESH** — Meshtastic radio integration (default tab on startup) + * **DEAD DROP** — Peer-to-peer message exchange with token-based epoch mailboxes (strongest current lane) +* **Gate Persona System** — Pseudonymous identities with Ed25519 signing keys, prekey bundles, SAS word contact verification, and abuse reporting +* **Mesh Terminal** — Built-in CLI: `send`, `dm`, market commands, gate state inspection. Draggable panel, minimizes to the top bar. Type `help` to see all commands. +* **Crypto Stack** — Ed25519 signing, X25519 Diffie-Hellman, AESGCM encryption with HKDF key derivation, hash chain commitment system. Double-ratchet DM scaffolding in progress. + +> **Experimental Testnet — No Privacy Guarantee:** InfoNet messages are obfuscated but NOT end-to-end encrypted. The Mesh network (Meshtastic/APRS) is NOT private — radio transmissions are inherently public. Do not send anything sensitive on any channel. E2E encryption is being developed but is not yet implemented. Treat all channels as open and public for now. + +### 🔍 Shodan Device Search (NEW in v0.9.6) + +* **Internet Device Search** — Query Shodan directly from ShadowBroker. Search by keyword, CVE, port, or service — results plotted as a live overlay on the map +* **Configurable Markers** — Shape, color, and size customization for Shodan results +* **Operator-Supplied API** — Uses your own `SHODAN_API_KEY`; results rendered as a local investigative overlay ### 🛩️ Aviation Tracking @@ -182,74 +223,94 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok * **AIS Vessel Stream** — 25,000+ vessels via aisstream.io WebSocket (real-time) * **Ship Classification** — Cargo, tanker, passenger, yacht, military vessel types with color-coded icons -* **Carrier Strike Group Tracker** — All 11 active US Navy aircraft carriers with OSINT-estimated positions - * Automated GDELT news scraping for carrier movement intelligence - * 50+ geographic region-to-coordinate mappings - * Disk-cached positions, auto-updates at 00:00 & 12:00 UTC +* **Carrier Strike Group Tracker** — All 11 active US Navy aircraft carriers with OSINT-estimated positions. No other open tool does this. + * Automated GDELT news scraping parses carrier movement reporting to estimate positions + * 50+ geographic region-to-coordinate mappings (e.g. "Eastern Mediterranean" → lat/lng) + * Disk-cached positions, auto-refreshes at 00:00 & 12:00 UTC * **Cruise & Passenger Ships** — Dedicated layer for cruise liners and ferries +* **Fishing Activity** — Global Fishing Watch vessel events (NEW) * **Clustered Display** — Ships cluster at low zoom with count labels, decluster on zoom-in +### 🚆 Rail Tracking (NEW in v0.9.6) + +* **Amtrak Trains** — Real-time positions of Amtrak trains across the US with speed, heading, route, and status +* **European Rail** — DigiTraffic integration for European train positions + ### 🛰️ Space & Satellites * **Orbital Tracking** — Real-time satellite positions via CelesTrak TLE data + SGP4 propagation (2,000+ active satellites, no API key required) * **Mission-Type Classification** — Color-coded by mission: military recon (red), SAR (cyan), SIGINT (white), navigation (blue), early warning (magenta), commercial imaging (green), space station (gold) +* **SatNOGS Ground Stations** — Amateur satellite ground station network with live observation data (NEW) +* **TinyGS LoRa Satellites** — LoRa satellite constellation tracking (NEW) ### 🌍 Geopolitics & Conflict * **Global Incidents** — GDELT-powered conflict event aggregation (last 8 hours, ~1,000 events) * **Ukraine Frontline** — Live warfront GeoJSON from DeepState Map +* **Ukraine Air Alerts** — Real-time regional air raid alerts (NEW) * **SIGINT/RISINT News Feed** — Real-time RSS aggregation from multiple intelligence-focused sources with user-customizable feeds (up to 20 sources, configurable priority weights 1-5) -* **Region Dossier** — Right-click anywhere on the map for: +* **Region Dossier** — Right-click anywhere on Earth for an instant intelligence briefing: * Country profile (population, capital, languages, currencies, area) - * Head of state & government type (Wikidata SPARQL) + * Current head of state & government type (live Wikidata SPARQL query) * Local Wikipedia summary with thumbnail + * Latest Sentinel-2 satellite photo with capture date and cloud cover (10m resolution) ### 🛰️ Satellite Imagery * **NASA GIBS (MODIS Terra)** — Daily true-color satellite imagery overlay with 30-day time slider, play/pause animation, and opacity control (~250m/pixel) * **High-Res Satellite (Esri)** — Sub-meter resolution imagery via Esri World Imagery — zoom into buildings and terrain detail (zoom 18+) * **Sentinel-2 Intel Card** — Right-click anywhere on the map for a floating intel card showing the latest Sentinel-2 satellite photo with capture date, cloud cover %, and clickable full-resolution image (10m resolution, updated every ~5 days) -* **SATELLITE Style Preset** — Quick-toggle high-res imagery via the STYLE button (DEFAULT → SATELLITE → FLIR → NVG → CRT) +* **Sentinel Hub Process API** — Copernicus CDSE satellite imagery with OAuth2 token flow (NEW) +* **VIIRS Nightlights** — Night-time light change detection overlay (NEW) +* **5 Visual Modes** — Toggle the entire map aesthetic via the STYLE button: + * **DEFAULT** — Dark CARTO basemap + * **SATELLITE** — Sub-meter Esri World Imagery + * **FLIR** — Thermal imaging aesthetic (inverted greyscale) + * **NVG** — Night vision green phosphor + * **CRT** — Retro terminal scanline overlay -### 📻 Software-Defined Radio (SDR) +### 📻 Software-Defined Radio & SIGINT * **KiwiSDR Receivers** — 500+ public SDR receivers plotted worldwide with clustered amber markers * **Live Radio Tuner** — Click any KiwiSDR node to open an embedded SDR tuner directly in the SIGINT panel * **Metadata Display** — Node name, location, antenna type, frequency bands, active users +* **Meshtastic Mesh Radio** — MQTT-based mesh radio integration with node map, integrated into Mesh Chat (NEW) +* **APRS Integration** — Amateur radio positioning via APRS-IS TCP feed (NEW) +* **GPS Jamming Detection** — Real-time analysis of aircraft NAC-P (Navigation Accuracy Category) values + * Grid-based aggregation identifies interference zones + * Red overlay squares with "GPS JAM XX%" severity labels +* **Radio Intercept Panel** — Scanner-style UI with OpenMHZ police/fire scanner feeds. Click any system to listen live. Scan mode cycles through active feeds automatically. Eavesdrop-by-click on real emergency communications. ### 📷 Surveillance -* **CCTV Mesh** — 4,400+ live traffic cameras from: +* **CCTV Mesh** — 11,000+ live traffic cameras from 13 sources across 6 countries: * 🇬🇧 Transport for London JamCams - * 🇺🇸 Austin, TX TxDOT - * 🇺🇸 NYC DOT + * 🇺🇸 NYC DOT, Austin TX (TxDOT) + * 🇺🇸 California (12 Caltrans districts), Washington State (WSDOT), Georgia DOT, Illinois DOT, Michigan DOT + * 🇪🇸 Spain DGT National (20 cities), Madrid City (357 cameras via KML) * 🇸🇬 Singapore LTA - * 🇪🇸 Spanish DGT (national roads) - * 🇪🇸 Madrid City Hall - * 🇪🇸 Málaga City - * 🇪🇸 Vigo City - * 🇪🇸 Vitoria-Gasteiz - * Custom URL ingestion + * 🌍 Windy Webcams * **Feed Rendering** — Automatic detection & rendering of video, MJPEG, HLS, embed, satellite tile, and image feeds * **Clustered Map Display** — Green dots cluster with count labels, decluster on zoom -### 📡 Signal Intelligence - -* **GPS Jamming Detection** — Real-time analysis of aircraft NAC-P (Navigation Accuracy Category) values - * Grid-based aggregation identifies interference zones - * Red overlay squares with "GPS JAM XX%" severity labels -* **Radio Intercept Panel** — Scanner-style UI for monitoring communications - -### 🔥 Environmental & Infrastructure Monitoring +### 🔥 Environmental & Hazard Monitoring * **NASA FIRMS Fire Hotspots (24h)** — 5,000+ global thermal anomalies from NOAA-20 VIIRS satellite, updated every cycle. Flame-shaped icons color-coded by fire radiative power (FRP): yellow (low), orange, red, dark red (intense). Clustered at low zoom with fire-shaped cluster markers. +* **Volcanoes** — Smithsonian Global Volcanism Program Holocene volcanoes plotted worldwide (NEW) +* **Weather Alerts** — Severe weather polygons with urgency/severity indicators (NEW) +* **Air Quality (PM2.5)** — OpenAQ stations worldwide with real-time particulate matter readings (NEW) +* **Earthquakes (24h)** — USGS real-time earthquake feed with magnitude-scaled markers * **Space Weather Badge** — Live NOAA geomagnetic storm indicator in the bottom status bar. Color-coded Kp index: green (quiet), yellow (active), red (storm G1–G5). Data from SWPC planetary K-index 1-minute feed. + +### 🏗️ Infrastructure Monitoring + * **Internet Outage Monitoring** — Regional internet connectivity alerts from Georgia Tech IODA. Grey markers at affected regions with severity percentage. Uses only reliable datasources (BGP routing tables, active ping probing) — no telescope or interpolated data. * **Data Center Mapping** — 2,000+ global data centers plotted from a curated dataset. Clustered purple markers with server-rack icons. Click for operator, location, and automatic internet outage cross-referencing by country. +* **Military Bases** — Global military installation and missile facility database (NEW) +* **Power Plants** — 35,000+ global power plants from the WRI database (NEW) -### 🌐 Additional Layers +### 🌐 Additional Layers & Tools -* **Earthquakes (24h)** — USGS real-time earthquake feed with magnitude-scaled markers * **Day/Night Cycle** — Solar terminator overlay showing global daylight/darkness * **Global Markets Ticker** — Live financial market indices (minimizable) * **Measurement Tool** — Point-to-point distance & bearing measurement on the map @@ -262,37 +323,48 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok ## 🏗️ Architecture ``` -┌────────────────────────────────────────────────────────┐ -│ FRONTEND (Next.js) │ -│ │ -│ ┌─────────────┐ ┌──────────┐ ┌───────────────┐ │ -│ │ MapLibre GL │ │ NewsFeed │ │ Control Panels│ │ -│ │ 2D WebGL │ │ SIGINT │ │ Layers/Filters│ │ -│ │ Map Render │ │ Intel │ │ Markets/Radio │ │ -│ └──────┬──────┘ └────┬─────┘ └───────┬───────┘ │ -│ └────────────────┼──────────────────┘ │ -│ │ REST API (60s / 120s) │ -├──────────────────────────┼─────────────────────────────┤ -│ BACKEND (FastAPI) │ -│ │ │ -│ ┌───────────────────────┼──────────────────────────┐ │ -│ │ Data Fetcher (Scheduler) │ │ -│ │ │ │ -│ │ ┌──────────┬──────────┬──────────┬───────────┐ │ │ -│ │ │ OpenSky │ adsb.lol │CelesTrak │ USGS │ │ │ -│ │ │ Flights │ Military │ Sats │ Quakes │ │ │ -│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ -│ │ │ AIS WS │ Carrier │ GDELT │ CCTV │ │ │ -│ │ │ Ships │ Tracker │ Conflict │ Cameras │ │ │ -│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ -│ │ │ DeepState│ RSS │ Region │ GPS │ │ │ -│ │ │ Frontline│ Intel │ Dossier │ Jamming │ │ │ -│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ -│ │ │ NASA │ NOAA │ IODA │ KiwiSDR │ │ │ -│ │ │ FIRMS │ Space Wx│ Outages │ Radios │ │ │ -│ │ └──────────┴──────────┴──────────┴───────────┘ │ │ -│ └──────────────────────────────────────────────────┘ │ -└────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────┐ +│ FRONTEND (Next.js) │ +│ │ +│ ┌─────────────┐ ┌──────────┐ ┌───────────┐ ┌─────────┐ │ +│ │ MapLibre GL │ │ NewsFeed │ │ Control │ │ Mesh │ │ +│ │ 2D WebGL │ │ SIGINT │ │ Panels │ │ Chat │ │ +│ │ Map Render │ │ Intel │ │Layers/Radio│ │Terminal │ │ +│ └──────┬──────┘ └────┬─────┘ └─────┬─────┘ └────┬────┘ │ +│ └───────────────┼──────────────┼─────────────┘ │ +│ │ REST + WebSocket │ +├─────────────────────────┼────────────────────────────────────┤ +│ BACKEND (FastAPI) │ +│ │ │ +│ ┌──────────────────────┼─────────────────────────────────┐ │ +│ │ Data Fetcher (Scheduler) │ │ +│ │ │ │ +│ │ ┌──────────┬──────────┬──────────┬───────────┐ │ │ +│ │ │ OpenSky │ adsb.lol │CelesTrak │ USGS │ │ │ +│ │ │ Flights │ Military │ Sats │ Quakes │ │ │ +│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ +│ │ │ AIS WS │ Carrier │ GDELT │ CCTV (13) │ │ │ +│ │ │ Ships │ Tracker │ Conflict │ Cameras │ │ │ +│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ +│ │ │ DeepState│ RSS │ Region │ GPS │ │ │ +│ │ │ Frontline│ Intel │ Dossier │ Jamming │ │ │ +│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ +│ │ │ NASA │ NOAA │ IODA │ KiwiSDR │ │ │ +│ │ │ FIRMS │ Space Wx│ Outages │ Radios │ │ │ +│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ +│ │ │ Shodan │ Amtrak │ SatNOGS │ Meshtastic│ │ │ +│ │ │ Devices │ Trains │ TinyGS │ APRS │ │ │ +│ │ ├──────────┼──────────┼──────────┼───────────┤ │ │ +│ │ │ Volcanoes│ Weather │ Fishing │ Mil Bases │ │ │ +│ │ │ Air Qual │ Alerts │ Activity │Power Plant│ │ │ +│ │ └──────────┴──────────┴──────────┴───────────┘ │ │ +│ └────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌────────────────────────────────────────────────────────┐ │ +│ │ Wormhole / InfoNet Relay │ │ +│ │ Gate Personas │ Canonical Signing │ Dead Drop DMs │ │ +│ └────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────┘ ``` --- @@ -308,27 +380,39 @@ helm install shadowbroker ./helm/chart --create-namespace --namespace shadowbrok | [USGS Earthquake](https://earthquake.usgs.gov) | Global seismic events | ~60s | No | | [GDELT Project](https://www.gdeltproject.org) | Global conflict events | ~6h | No | | [DeepState Map](https://deepstatemap.live) | Ukraine frontline | ~30min | No | -| [Transport for London](https://api.tfl.gov.uk) | London CCTV JamCams | ~5min | No | -| [TxDOT](https://its.txdot.gov) | Austin TX traffic cameras | ~5min | No | -| [NYC DOT](https://webcams.nyctmc.org) | NYC traffic cameras | ~5min | No | -| [Singapore LTA](https://datamall.lta.gov.sg) | Singapore traffic cameras | ~5min | **Yes** | -| [DGT Spain](https://nap.dgt.es) | Spanish national road cameras | ~10min | No | -| [Madrid Open Data](https://datos.madrid.es) | Madrid urban traffic cameras | ~10min | No | -| [Málaga Open Data](https://datosabiertos.malaga.eu) | Málaga traffic cameras | ~10min | No | -| [Vigo Open Data](https://datos.vigo.org) | Vigo traffic cameras | ~10min | No | -| [Vitoria-Gasteiz](https://www.vitoria-gasteiz.org) | Vitoria-Gasteiz traffic cameras | ~10min | No | -| [RestCountries](https://restcountries.com) | Country profile data | On-demand (cached 24h) | No | -| [Wikidata SPARQL](https://query.wikidata.org) | Head of state data | On-demand (cached 24h) | No | -| [Wikipedia API](https://en.wikipedia.org/api) | Location summaries & aircraft images | On-demand (cached) | No | -| [NASA GIBS](https://gibs.earthdata.nasa.gov) | MODIS Terra daily satellite imagery | Daily (24-48h delay) | No | -| [Esri World Imagery](https://www.arcgis.com) | High-res satellite basemap | Static (periodically updated) | No | -| [MS Planetary Computer](https://planetarycomputer.microsoft.com) | Sentinel-2 L2A scenes (right-click) | On-demand | No | +| [Shodan](https://www.shodan.io) | Internet-connected device search | On-demand | **Yes** | +| [Amtrak](https://www.amtrak.com) | US train positions | ~60s | No | +| [DigiTraffic](https://www.digitraffic.fi) | European rail positions | ~60s | No | +| [Global Fishing Watch](https://globalfishingwatch.org) | Fishing vessel activity events | ~10min | No | +| Transport for London, NYC DOT, TxDOT | CCTV cameras (UK, US) | ~10min | No | +| Caltrans, WSDOT, GDOT, IDOT, MDOT | CCTV cameras (5 US states) | ~10min | No | +| Spain DGT, Madrid City | CCTV cameras (Spain) | ~10min | No | +| [Singapore LTA](https://datamall.lta.gov.sg) | Singapore traffic cameras | ~10min | **Yes** | +| [Windy Webcams](https://www.windy.com) | Global webcams | ~10min | No | +| [SatNOGS](https://satnogs.org) | Amateur satellite ground stations | ~30min | No | +| [TinyGS](https://tinygs.com) | LoRa satellite ground stations | ~30min | No | +| [Meshtastic MQTT](https://meshtastic.org) | Mesh radio node positions | Real-time | No | +| [APRS-IS](https://www.aprs-is.net) | Amateur radio positions | Real-time TCP | No | | [KiwiSDR](https://kiwisdr.com) | Public SDR receiver locations | ~30min | No | -| [OSM Nominatim](https://nominatim.openstreetmap.org) | Place name geocoding (LOCATE bar) | On-demand | No | +| [OpenMHZ](https://openmhz.com) | Police/fire scanner feeds | Real-time | No | +| [Smithsonian GVP](https://volcano.si.edu) | Holocene volcanoes worldwide | Static (cached) | No | +| [OpenAQ](https://openaq.org) | Air quality PM2.5 stations | ~120s | No | +| NOAA / NWS | Severe weather alerts & polygons | ~120s | No | +| [WRI Global Power Plant DB](https://datasets.wri.org) | 35,000+ power plants | Static (cached) | No | +| Military base datasets | Global military installations | Static (cached) | No | | [NASA FIRMS](https://firms.modaps.eosdis.nasa.gov) | NOAA-20 VIIRS fire/thermal hotspots | ~120s | No | | [NOAA SWPC](https://services.swpc.noaa.gov) | Space weather Kp index & solar events | ~120s | No | | [IODA (Georgia Tech)](https://ioda.inetintel.cc.gatech.edu) | Regional internet outage alerts | ~120s | No | | [DC Map (GitHub)](https://github.com/Ringmast4r/Data-Center-Map---Global) | Global data center locations | Static (cached 7d) | No | +| [NASA GIBS](https://gibs.earthdata.nasa.gov) | MODIS Terra daily satellite imagery | Daily (24-48h delay) | No | +| [Esri World Imagery](https://www.arcgis.com) | High-res satellite basemap | Static (periodically updated) | No | +| [MS Planetary Computer](https://planetarycomputer.microsoft.com) | Sentinel-2 L2A scenes (right-click) | On-demand | No | +| [Copernicus CDSE](https://dataspace.copernicus.eu) | Sentinel Hub imagery (Process API) | On-demand | **Yes** (free) | +| [VIIRS Nightlights](https://eogdata.mines.edu) | Night-time light change detection | Static | No | +| [RestCountries](https://restcountries.com) | Country profile data | On-demand (cached 24h) | No | +| [Wikidata SPARQL](https://query.wikidata.org) | Head of state data | On-demand (cached 24h) | No | +| [Wikipedia API](https://en.wikipedia.org/api) | Location summaries & aircraft images | On-demand (cached) | No | +| [OSM Nominatim](https://nominatim.openstreetmap.org) | Place name geocoding (LOCATE bar) | On-demand | No | | [CARTO Basemaps](https://carto.com) | Dark map tiles | Continuous | No | --- @@ -392,6 +476,9 @@ services: - OPENSKY_CLIENT_ID= # Optional — higher flight data rate limits - OPENSKY_CLIENT_SECRET= # Optional — paired with Client ID above - LTA_ACCOUNT_KEY= # Optional — Singapore CCTV cameras + - SHODAN_API_KEY= # Optional — Shodan device search overlay + - SH_CLIENT_ID= # Optional — Sentinel Hub satellite imagery + - SH_CLIENT_SECRET= # Optional — paired with Sentinel Hub ID - CORS_ORIGINS= # Optional — comma-separated allowed origins volumes: - backend_data:/app/data @@ -429,6 +516,12 @@ If you just want to run the dashboard without dealing with terminal commands: **Mac/Linux:** Open terminal, type `chmod +x start.sh`, `dos2unix start.sh`, and run `./start.sh`. 5. It will automatically install everything and launch the dashboard! +Local launcher notes: + +- `start.bat` / `start.sh` currently run the hardened web/local stack, not the final native desktop boundary. +- Security-sensitive paths are hardened up to the pre-Tauri boundary, but operator-facing responsiveness still matters and is part of the acceptance bar. +- If Wormhole identity or DM contact endpoints fail after an upgrade on Windows, see `F:\Codebase\Oracle\live-risk-dashboard\docs\mesh\pre-tauri-phase-closeout.md` for the secure-storage repair workflow. + --- ### 💻 Developer Setup @@ -456,6 +549,18 @@ venv\Scripts\activate # Windows # source venv/bin/activate # macOS/Linux pip install -r requirements.txt # includes pystac-client for Sentinel-2 +# Optional helper scripts (creates venv + installs dev deps) +# Windows PowerShell +# .\scripts\setup-venv.ps1 +# macOS/Linux +# ./scripts/setup-venv.sh + +# Optional env check (prints warnings for missing keys) +# Windows PowerShell +# .\scripts\check-env.ps1 +# macOS/Linux +# ./scripts/check-env.sh + # Create .env with your API keys echo "AIS_API_KEY=your_aisstream_key" >> .env echo "OPENSKY_CLIENT_ID=your_opensky_client_id" >> .env @@ -478,6 +583,14 @@ This starts: * **Next.js** frontend on `http://localhost:3000` * **FastAPI** backend on `http://localhost:8000` +### Pre-commit (Optional) + +If you use pre-commit, install hooks once from repo root: + +```bash +pre-commit install +``` + ### Local AIS Receiver (Optional) You can feed your own AIS ship data into ShadowBroker using an RTL-SDR dongle and [AIS-catcher](https://github.com/jvde-github/AIS-catcher), an open-source AIS decoder. This gives you real-time coverage of vessels in your local area — no API key needed. @@ -503,7 +616,7 @@ AIS-catcher decodes VHF radio signals on 161.975 MHz and 162.025 MHz and POSTs d ## 🎛️ Data Layers -All layers are independently toggleable from the left panel: +All 37 layers are independently toggleable from the left panel: | Layer | Default | Description | |---|---|---| @@ -512,23 +625,39 @@ All layers are independently toggleable from the left panel: | Private Jets | ✅ ON | High-value bizjets with owner data | | Military Flights | ✅ ON | Military & government aircraft | | Tracked Aircraft | ✅ ON | Special interest watch list | -| Satellites | ✅ ON | Orbital assets by mission type | +| GPS Jamming | ✅ ON | NAC-P degradation zones | | Carriers / Mil / Cargo | ✅ ON | Navy carriers, cargo ships, tankers | -| Civilian Vessels | ❌ OFF | Yachts, fishing, recreational | +| Civilian Vessels | ✅ ON | Yachts, fishing, recreational | | Cruise / Passenger | ✅ ON | Cruise ships and ferries | -| Tracked Yachts | ✅ ON | Billionaire & oligarch superyachts (Yacht-Alert DB) | +| Tracked Yachts | ✅ ON | Billionaire & oligarch superyachts | +| Fishing Activity | ✅ ON | Global Fishing Watch vessel events | +| Trains | ✅ ON | Amtrak + European rail positions | +| Satellites | ✅ ON | Orbital assets by mission type | +| SatNOGS | ✅ ON | Amateur satellite ground stations | +| TinyGS | ✅ ON | LoRa satellite ground stations | | Earthquakes (24h) | ✅ ON | USGS seismic events | -| CCTV Mesh | ❌ OFF | Surveillance camera network | +| Fire Hotspots (24h) | ✅ ON | NASA FIRMS VIIRS thermal anomalies | +| Volcanoes | ✅ ON | Smithsonian Holocene volcanoes | +| Weather Alerts | ✅ ON | Severe weather polygons | +| Air Quality (PM2.5) | ✅ ON | OpenAQ stations worldwide | | Ukraine Frontline | ✅ ON | Live warfront positions | +| Ukraine Air Alerts | ✅ ON | Regional air raid alerts | | Global Incidents | ✅ ON | GDELT conflict events | -| GPS Jamming | ✅ ON | NAC-P degradation zones | +| CCTV Mesh | ✅ ON | 11,000+ cameras across 13 sources, 6 countries | +| Internet Outages | ✅ ON | IODA regional connectivity alerts | +| Data Centers | ✅ ON | Global data center locations (2,000+) | +| Military Bases | ✅ ON | Global military installations | +| KiwiSDR Receivers | ✅ ON | Public SDR radio receivers | +| Meshtastic Nodes | ✅ ON | Mesh radio node positions | +| APRS | ✅ ON | Amateur radio positioning | +| Scanners | ✅ ON | Police/fire scanner feeds | +| Day / Night Cycle | ✅ ON | Solar terminator overlay | | MODIS Terra (Daily) | ❌ OFF | NASA GIBS daily satellite imagery | | High-Res Satellite | ❌ OFF | Esri sub-meter satellite imagery | -| KiwiSDR Receivers | ❌ OFF | Public SDR radio receivers | -| Fire Hotspots (24h) | ❌ OFF | NASA FIRMS VIIRS thermal anomalies | -| Internet Outages | ❌ OFF | IODA regional connectivity alerts | -| Data Centers | ❌ OFF | Global data center locations (2,000+) | -| Day / Night Cycle | ✅ ON | Solar terminator overlay | +| Sentinel Hub | ❌ OFF | Copernicus CDSE Process API | +| VIIRS Nightlights | ❌ OFF | Night-time light change detection | +| Power Plants | ❌ OFF | 35,000+ global power plants | +| Shodan Overlay | ❌ OFF | Internet device search results | --- @@ -553,44 +682,72 @@ The platform is optimized for handling massive real-time datasets: ``` live-risk-dashboard/ ├── backend/ -│ ├── main.py # FastAPI app, middleware, API routes -│ ├── carrier_cache.json # Persisted carrier OSINT positions -│ ├── cctv.db # SQLite CCTV camera database +│ ├── main.py # FastAPI app, middleware, API routes (~4,000 lines) +│ ├── cctv.db # SQLite CCTV camera database (auto-generated) │ ├── config/ -│ │ └── news_feeds.json # User-customizable RSS feed list (persists across restarts) -│ └── services/ -│ ├── data_fetcher.py # Core scheduler — fetches all data sources -│ ├── ais_stream.py # AIS WebSocket client (25K+ vessels) -│ ├── carrier_tracker.py # OSINT carrier position tracker -│ ├── cctv_pipeline.py # Multi-source CCTV camera ingestion -│ ├── geopolitics.py # GDELT + Ukraine frontline fetcher -│ ├── region_dossier.py # Right-click country/city intelligence -│ ├── radio_intercept.py # Scanner radio feed integration -│ ├── kiwisdr_fetcher.py # KiwiSDR receiver scraper -│ ├── sentinel_search.py # Sentinel-2 STAC imagery search -│ ├── network_utils.py # HTTP client with curl fallback -│ ├── api_settings.py # API key management -│ └── news_feed_config.py # RSS feed config manager (add/remove/weight feeds) +│ │ └── news_feeds.json # User-customizable RSS feed list +│ ├── services/ +│ │ ├── data_fetcher.py # Core scheduler — orchestrates all data sources +│ │ ├── ais_stream.py # AIS WebSocket client (25K+ vessels) +│ │ ├── carrier_tracker.py # OSINT carrier position estimator (GDELT news scraping) +│ │ ├── cctv_pipeline.py # 13-source CCTV camera ingestion pipeline +│ │ ├── geopolitics.py # GDELT + Ukraine frontline + air alerts +│ │ ├── region_dossier.py # Right-click country/city intelligence +│ │ ├── radio_intercept.py # Police scanner feeds + OpenMHZ +│ │ ├── kiwisdr_fetcher.py # KiwiSDR receiver scraper +│ │ ├── sentinel_search.py # Sentinel-2 STAC imagery search +│ │ ├── shodan_connector.py # Shodan device search connector +│ │ ├── sigint_bridge.py # APRS-IS TCP bridge +│ │ ├── network_utils.py # HTTP client with curl fallback +│ │ ├── api_settings.py # API key management +│ │ ├── news_feed_config.py # RSS feed config manager +│ │ ├── fetchers/ +│ │ │ ├── flights.py # OpenSky, adsb.lol, GPS jamming, holding patterns +│ │ │ ├── geo.py # AIS vessels, carriers, GDELT, fishing activity +│ │ │ ├── satellites.py # CelesTrak TLE + SGP4 propagation +│ │ │ ├── earth_observation.py # Quakes, fires, volcanoes, air quality, weather +│ │ │ ├── infrastructure.py # Data centers, power plants, military bases +│ │ │ ├── trains.py # Amtrak + DigiTraffic European rail +│ │ │ ├── sigint.py # SatNOGS, TinyGS, APRS, Meshtastic +│ │ │ ├── meshtastic_map.py # Meshtastic MQTT + map node aggregation +│ │ │ ├── military.py # Military aircraft classification +│ │ │ ├── news.py # RSS intelligence feed aggregation +│ │ │ ├── financial.py # Global markets data +│ │ │ └── ukraine_alerts.py # Ukraine air raid alerts +│ │ └── mesh/ # InfoNet / Wormhole protocol stack +│ │ ├── mesh_protocol.py # Core mesh protocol + routing +│ │ ├── mesh_crypto.py # Ed25519, X25519, AESGCM primitives +│ │ ├── mesh_hashchain.py # Hash chain commitment system (~1,400 lines) +│ │ ├── mesh_router.py # Multi-transport router (APRS, Meshtastic, WS) +│ │ ├── mesh_wormhole_persona.py # Gate persona identity management +│ │ ├── mesh_wormhole_dead_drop.py # Dead Drop token-based DM mailbox +│ │ ├── mesh_wormhole_ratchet.py # Double-ratchet DM scaffolding +│ │ ├── mesh_wormhole_gate_keys.py # Gate key management + rotation +│ │ ├── mesh_wormhole_seal.py # Message sealing + unsealing +│ │ ├── mesh_merkle.py # Merkle tree proofs for data commitment +│ │ ├── mesh_reputation.py # Node reputation scoring +│ │ ├── mesh_oracle.py # Oracle consensus protocol +│ │ └── mesh_secure_storage.py # Secure credential storage │ ├── frontend/ │ ├── src/ │ │ ├── app/ │ │ │ └── page.tsx # Main dashboard — state, polling, layout │ │ └── components/ -│ │ ├── MaplibreViewer.tsx # Core map — 2,000+ lines, all GeoJSON layers -│ │ ├── NewsFeed.tsx # SIGINT feed + entity detail panels -│ │ ├── WorldviewLeftPanel.tsx # Data layer toggles +│ │ ├── MaplibreViewer.tsx # Core map — all GeoJSON layers +│ │ ├── MeshChat.tsx # InfoNet / Mesh / Dead Drop chat panel +│ │ ├── MeshTerminal.tsx # Draggable CLI terminal +│ │ ├── NewsFeed.tsx # SIGINT feed + entity detail panels +│ │ ├── WorldviewLeftPanel.tsx # Data layer toggles (35+ layers) │ │ ├── WorldviewRightPanel.tsx # Search + filter sidebar -│ │ ├── FilterPanel.tsx # Basic layer filters │ │ ├── AdvancedFilterModal.tsx # Airport/country/owner filtering │ │ ├── MapLegend.tsx # Dynamic legend with all icons │ │ ├── MarketsPanel.tsx # Global financial markets ticker │ │ ├── RadioInterceptPanel.tsx # Scanner-style radio panel │ │ ├── FindLocateBar.tsx # Search/locate bar -│ │ ├── ChangelogModal.tsx # Version changelog popup -│ │ ├── SettingsPanel.tsx # App settings (API Keys + News Feed manager) +│ │ ├── ChangelogModal.tsx # Version changelog popup (auto-shows on upgrade) +│ │ ├── SettingsPanel.tsx # API Keys + News Feed + Shodan config │ │ ├── ScaleBar.tsx # Map scale indicator -│ │ ├── WikiImage.tsx # Wikipedia image fetcher │ │ └── ErrorBoundary.tsx # Crash recovery wrapper │ └── package.json ``` @@ -609,6 +766,9 @@ AIS_API_KEY=your_aisstream_key # Maritime vessel tracking (aisstr OPENSKY_CLIENT_ID=your_opensky_client_id # OAuth2 — higher rate limits for flight data OPENSKY_CLIENT_SECRET=your_opensky_secret # OAuth2 — paired with Client ID above LTA_ACCOUNT_KEY=your_lta_key # Singapore CCTV cameras +SHODAN_API_KEY=your_shodan_key # Shodan device search overlay +SH_CLIENT_ID=your_sentinel_hub_id # Copernicus CDSE Sentinel Hub imagery +SH_CLIENT_SECRET=your_sentinel_hub_secret # Paired with Sentinel Hub Client ID ``` ### Frontend @@ -621,6 +781,23 @@ LTA_ACCOUNT_KEY=your_lta_key # Singapore CCTV cameras --- +## 🤝 Contributors + +ShadowBroker is built in the open. These people shipped real code: + +| Who | What | PR | +|-----|------|----| +| [@wa1id](https://github.com/wa1id) | CCTV ingestion fix — threaded SQLite, persistent DB, startup hydration, cluster clickability | #92 | +| [@AlborzNazari](https://github.com/AlborzNazari) | Spain DGT + Madrid CCTV sources, STIX 2.1 threat intel export | #91 | +| [@adust09](https://github.com/adust09) | Power plants layer, East Asia intel coverage (JSDF bases, ICAO enrichment, Taiwan news, military classification) | #71, #72, #76, #77, #87 | +| [@Xpirix](https://github.com/Xpirix) | LocateBar style and interaction improvements | #78 | +| [@imqdcr](https://github.com/imqdcr) | Ship toggle split (4 categories) + stable MMSI/callsign entity IDs | — | +| [@csysp](https://github.com/csysp) | Dismissible threat alerts + stable entity IDs for GDELT & News | #48, #63 | +| [@suranyami](https://github.com/suranyami) | Parallel multi-arch Docker builds (11min → 3min) + runtime BACKEND_URL fix | #35, #44 | +| [@chr0n1x](https://github.com/chr0n1x) | Kubernetes / Helm chart architecture for HA deployments | — | + +--- + ## ⚠️ Disclaimer This tool is built entirely on publicly available, open-source intelligence (OSINT) data. No classified, restricted, or non-public data is used. Carrier positions are estimates based on public reporting. The military-themed UI is purely aesthetic. diff --git a/backend/.env.example b/backend/.env.example index b429f9dc..f089288f 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -15,9 +15,91 @@ AIS_API_KEY= # https://aisstream.io/ — free tier WebSocket key # CORS_ORIGINS=http://192.168.1.50:3000,https://my-domain.com # Admin key — protects sensitive endpoints (API key management, system update). -# If unset, these endpoints remain open (fine for local dev). +# If unset, endpoints are only accessible from localhost unless ALLOW_INSECURE_ADMIN=true. # Set this in production and enter the same key in Settings → Admin Key. # ADMIN_KEY=your-secret-admin-key-here +# Allow insecure admin access without ADMIN_KEY (local dev only). +# ALLOW_INSECURE_ADMIN=false + +# User-Agent for Nominatim geocoding requests (per OSM usage policy). +# NOMINATIM_USER_AGENT=ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker) + # LTA Singapore traffic cameras — leave blank to skip this data source. # LTA_ACCOUNT_KEY= + +# NASA FIRMS country-scoped fire data — enriches global CSV with conflict-zone hotspots. +# Free MAP_KEY from https://firms.modaps.eosdis.nasa.gov/map/#d:24hrs;@0.0,0.0,3.0z +# FIRMS_MAP_KEY= + +# Ukraine air raid alerts from alerts.in.ua — free token from https://alerts.in.ua/ +# ALERTS_IN_UA_TOKEN= + +# Google Earth Engine service account for VIIRS change detection (optional). +# Download JSON key from https://console.cloud.google.com/iam-admin/serviceaccounts +# pip install earthengine-api +# GEE_SERVICE_ACCOUNT_KEY= + +# ── Mesh / Reticulum (RNS) ───────────────────────────────────── +# Full-node / participant-node posture for public Infonet sync. +# MESH_NODE_MODE=participant # participant | relay | perimeter +# MESH_BOOTSTRAP_DISABLED=false +# MESH_BOOTSTRAP_MANIFEST_PATH=data/bootstrap_peers.json +# MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY= +# MESH_RELAY_PEERS= # comma-separated operator-trusted sync/push peers +# MESH_PEER_PUSH_SECRET= # shared-secret push auth for trusted testnet peers +# MESH_SYNC_INTERVAL_S=300 +# MESH_SYNC_FAILURE_BACKOFF_S=60 +# +# Enable Reticulum bridge for Infonet event gossip. +# MESH_RNS_ENABLED=false +# MESH_RNS_APP_NAME=shadowbroker +# MESH_RNS_ASPECT=infonet +# MESH_RNS_IDENTITY_PATH= +# MESH_RNS_PEERS= # comma-separated destination hashes +# MESH_RNS_DANDELION_HOPS=2 +# MESH_RNS_DANDELION_DELAY_MS=400 +# MESH_RNS_CHURN_INTERVAL_S=300 +# MESH_RNS_MAX_PEERS=32 +# MESH_RNS_MAX_PAYLOAD=8192 +# MESH_RNS_PEER_BUCKET_PREFIX=4 +# MESH_RNS_MAX_PEERS_PER_BUCKET=4 +# MESH_RNS_PEER_FAIL_THRESHOLD=3 +# MESH_RNS_PEER_COOLDOWN_S=300 +# MESH_RNS_SHARD_ENABLED=false +# MESH_RNS_SHARD_DATA_SHARDS=3 +# MESH_RNS_SHARD_PARITY_SHARDS=1 +# MESH_RNS_SHARD_TTL_S=30 +# MESH_RNS_FEC_CODEC=xor +# MESH_RNS_BATCH_MS=200 +# MESH_RNS_COVER_INTERVAL_S=0 +# MESH_RNS_COVER_SIZE=64 +# MESH_RNS_IBF_WINDOW=256 +# MESH_RNS_IBF_TABLE_SIZE=64 +# MESH_RNS_IBF_MINHASH_SIZE=16 +# MESH_RNS_IBF_MINHASH_THRESHOLD=0.25 +# MESH_RNS_IBF_WINDOW_JITTER=32 +# MESH_RNS_IBF_INTERVAL_S=120 +# MESH_RNS_IBF_SYNC_PEERS=3 +# MESH_RNS_IBF_QUORUM_TIMEOUT_S=6 +# MESH_RNS_IBF_MAX_REQUEST_IDS=64 +# MESH_RNS_IBF_MAX_EVENTS=64 +# MESH_RNS_SESSION_ROTATE_S=0 +# MESH_RNS_IBF_FAIL_THRESHOLD=3 +# MESH_RNS_IBF_COOLDOWN_S=120 +# MESH_VERIFY_INTERVAL_S=600 +# MESH_VERIFY_SIGNATURES=false + +# ── Mesh DM Relay ────────────────────────────────────────────── +# MESH_DM_TOKEN_PEPPER=change-me + +# ── Self Update ──────────────────────────────────────────────── +# MESH_UPDATE_SHA256= + +# ── Wormhole (Local Agent) ───────────────────────────────────── +# WORMHOLE_HOST=127.0.0.1 +# WORMHOLE_PORT=8787 +# WORMHOLE_RELOAD=false +# WORMHOLE_TRANSPORT=direct +# WORMHOLE_SOCKS_PROXY=127.0.0.1:9050 +# WORMHOLE_SOCKS_DNS=true diff --git a/backend/Dockerfile b/backend/Dockerfile index 5cd632f4..ae724f4a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim-bookworm +FROM python:3.11-slim-bookworm WORKDIR /app @@ -9,32 +9,38 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* -# Install UV for Python dependency management +# Install UV for fast, reproducible Python dependency management ADD https://astral.sh/uv/install.sh /uv-installer.sh RUN sh /uv-installer.sh && rm /uv-installer.sh -ENV PATH="/root/.local/bin/:$PATH" -# Set environment variable for UV to install dependencies in the system Python environment -# By default UV creates a new venv and installs dependencies there +ENV PATH="/root/.local/bin:$PATH" +# Install into system Python (no venv needed inside container) ENV UV_PROJECT_ENVIRONMENT=/usr/local -# Copy pyproject.toml from root for dependency management -COPY pyproject.toml . -# Copy lock file for reproducible builds -COPY uv.lock . -# Copy source code +# Copy workspace root files for UV resolution (build context is repo root) +COPY pyproject.toml /workspace/pyproject.toml +COPY uv.lock /workspace/uv.lock +COPY backend/pyproject.toml /workspace/backend/pyproject.toml + +# Install Python dependencies using the lockfile +RUN cd /workspace/backend && uv sync --frozen --no-dev \ + && playwright install --with-deps chromium + +# Copy backend source code COPY backend/ . -# Install Python dependencies using UV (this will use the lock file for reproducibility) -RUN uv sync --frozen -RUN uv run playwright install --with-deps chromium # Install Node.js dependencies (ws module for AIS WebSocket proxy) -# Copy manifests first so this layer is cached unless deps change COPY backend/package*.json ./ RUN npm ci --omit=dev +# Clean up workspace scaffold +RUN rm -rf /workspace + + # Create a non-root user for security # Grant write access to /app so the auto-updater can extract files +# Pre-create /app/data so mounted volumes inherit correct ownership RUN adduser --system --uid 1001 backenduser \ + && mkdir -p /app/data \ && chown -R backenduser /app \ && chmod -R u+w /app @@ -45,4 +51,4 @@ USER backenduser EXPOSE 8000 # Start FastAPI server -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--timeout-keep-alive", "120"] diff --git a/backend/config/news_feeds.json b/backend/config/news_feeds.json index 06d689a3..f1330ace 100644 --- a/backend/config/news_feeds.json +++ b/backend/config/news_feeds.json @@ -1,5 +1,15 @@ { "feeds": [ + { + "name": "Reuters", + "url": "https://www.reutersagency.com/feed/?best-topics=world", + "weight": 5 + }, + { + "name": "AP News", + "url": "https://rsshub.app/apnews/topics/world-news", + "weight": 5 + }, { "name": "NPR", "url": "https://feeds.npr.org/1004/rss.xml", @@ -26,13 +36,33 @@ "weight": 5 }, { - "name": "NHK", - "url": "https://www3.nhk.or.jp/nhkworld/rss/world.xml", + "name": "The War Zone", + "url": "https://www.twz.com/feed", + "weight": 4 + }, + { + "name": "Bellingcat", + "url": "https://www.bellingcat.com/feed/", + "weight": 4 + }, + { + "name": "Guardian", + "url": "https://www.theguardian.com/world/rss", "weight": 3 }, + { + "name": "TASS", + "url": "https://tass.com/rss/v2.xml", + "weight": 2 + }, + { + "name": "Xinhua", + "url": "http://www.news.cn/english/rss/worldrss.xml", + "weight": 2 + }, { "name": "CNA", - "url": "https://www.channelnewsasia.com/rssfeed/8395986", + "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml", "weight": 3 }, { @@ -40,16 +70,6 @@ "url": "https://en.mercopress.com/rss/", "weight": 3 }, - { - "name": "FocusTaiwan", - "url": "https://focustaiwan.tw/rss", - "weight": 5 - }, - { - "name": "Kyodo", - "url": "https://english.kyodonews.net/rss/news.xml", - "weight": 4 - }, { "name": "SCMP", "url": "https://www.scmp.com/rss/91/feed", @@ -60,26 +80,11 @@ "url": "https://thediplomat.com/feed/", "weight": 4 }, - { - "name": "Stars and Stripes", - "url": "https://www.stripes.com/feeds/pacific.rss", - "weight": 4 - }, { "name": "Yonhap", "url": "https://en.yna.co.kr/RSS/news.xml", "weight": 4 }, - { - "name": "Nikkei Asia", - "url": "https://asia.nikkei.com/rss", - "weight": 3 - }, - { - "name": "Taipei Times", - "url": "https://www.taipeitimes.com/xml/pda.rss", - "weight": 4 - }, { "name": "Asia Times", "url": "https://asiatimes.com/feed/", @@ -96,4 +101,4 @@ "weight": 3 } ] -} \ No newline at end of file +} diff --git a/backend/data/military_bases.json b/backend/data/military_bases.json index 646df9a2..d34d96d3 100644 --- a/backend/data/military_bases.json +++ b/backend/data/military_bases.json @@ -93,7 +93,7 @@ "operator": "USAF 36th Wing", "branch": "air_force", "lat": 13.584, - "lng": 144.930 + "lng": 144.93 }, { "name": "Naval Base Guam", @@ -108,8 +108,8 @@ "country": "South Korea", "operator": "USAF 51st Fighter Wing", "branch": "air_force", - "lat": 37.090, - "lng": 127.030 + "lat": 37.09, + "lng": 127.03 }, { "name": "Camp Humphreys", @@ -196,7 +196,7 @@ "country": "Japan", "operator": "ASDF Air Defense Command", "branch": "asdf", - "lat": 34.750, + "lat": 34.75, "lng": 137.703 }, { @@ -212,7 +212,7 @@ "country": "Japan", "operator": "MSDF Fleet HQ", "branch": "msdf", - "lat": 35.290, + "lat": 35.29, "lng": 139.665 }, { @@ -348,7 +348,7 @@ "country": "China", "operator": "PLAAF", "branch": "air_force", - "lat": 30.500, + "lat": 30.5, "lng": 114.211 }, { @@ -365,7 +365,7 @@ "operator": "PLAAF", "branch": "air_force", "lat": 40.296, - "lng": 99.750 + "lng": 99.75 }, { "name": "Hotan Air Base", @@ -396,7 +396,7 @@ "country": "China", "operator": "PLAN Southern Theater", "branch": "navy", - "lat": 21.190, + "lat": 21.19, "lng": 110.405 }, { @@ -404,7 +404,7 @@ "country": "China", "operator": "PLAN Northern Theater", "branch": "navy", - "lat": 36.100, + "lat": 36.1, "lng": 120.272 }, { @@ -452,7 +452,7 @@ "country": "China", "operator": "PLAN Eastern Theater", "branch": "navy", - "lat": 29.480, + "lat": 29.48, "lng": 121.933 }, { @@ -468,7 +468,7 @@ "country": "China", "operator": "PLA (SCS outpost)", "branch": "air_force", - "lat": 9.550, + "lat": 9.55, "lng": 112.892 }, { @@ -508,7 +508,7 @@ "country": "China", "operator": "PLARF 61 Base", "branch": "missile", - "lat": 29.480, + "lat": 29.48, "lng": 119.281 }, { @@ -548,7 +548,7 @@ "country": "China", "operator": "PLARF 63 Base", "branch": "missile", - "lat": 34.620, + "lat": 34.62, "lng": 112.454 }, { @@ -564,10 +564,9 @@ "country": "China", "operator": "PLARF 66 Base", "branch": "missile", - "lat": 40.200, - "lng": 113.200 + "lat": 40.2, + "lng": 113.2 }, - { "name": "Vladivostok (Pacific Fleet HQ)", "country": "Russia", @@ -605,8 +604,8 @@ "country": "Russia", "operator": "Russian VKS Long-Range Aviation", "branch": "air_force", - "lat": 51.170, - "lng": 128.400 + "lat": 51.17, + "lng": 128.4 }, { "name": "Fokino Naval Base", @@ -640,7 +639,6 @@ "lat": 44.927, "lng": 147.863 }, - { "name": "Yongbyon Nuclear Complex", "country": "North Korea", @@ -670,7 +668,7 @@ "country": "North Korea", "operator": "DPRK NADA", "branch": "missile", - "lat": 39.660, + "lat": 39.66, "lng": 124.705 }, { @@ -687,7 +685,7 @@ "operator": "KPAF", "branch": "air_force", "lat": 39.752, - "lng": 125.890 + "lng": 125.89 }, { "name": "Wonsan-Kalma Air Base", @@ -718,10 +716,9 @@ "country": "North Korea", "operator": "KPA Artillery Corps", "branch": "army", - "lat": 38.330, + "lat": 38.33, "lng": 126.586 }, - { "name": "Hualien Air Force Base (Jiashan)", "country": "Taiwan", @@ -786,7 +783,6 @@ "lat": 22.793, "lng": 121.182 }, - { "name": "Subic Bay (EDCA site)", "country": "Philippines", @@ -801,7 +797,7 @@ "operator": "Philippine Air Force / US EDCA", "branch": "air_force", "lat": 15.186, - "lng": 120.560 + "lng": 120.56 }, { "name": "Basa Air Base", @@ -824,7 +820,7 @@ "country": "Philippines", "operator": "Philippine Army / US EDCA", "branch": "army", - "lat": 18.200, + "lat": 18.2, "lng": 121.659 }, { @@ -835,7 +831,6 @@ "lat": 7.986, "lng": 117.062 }, - { "name": "RAAF Base Darwin", "country": "Australia", @@ -867,5 +862,4317 @@ "branch": "army", "lat": -23.799, "lng": 133.737 + }, + { + "name": "Ream Naval Base", + "country": "China", + "operator": "China", + "branch": "army", + "lat": 10.503, + "lng": 103.609 + }, + { + "name": "Chinese PLA Support Base", + "country": "China", + "operator": "China", + "branch": "navy", + "lat": 11.591, + "lng": 43.06 + }, + { + "name": "Chinese Naval Intelligence Base", + "country": "China", + "operator": "China", + "branch": "army", + "lat": 14.146, + "lng": 93.359 + }, + { + "name": "PLA Outpost (Tajik Border)", + "country": "China", + "operator": "China", + "branch": "army", + "lat": 37.438, + "lng": 74.913 + }, + { + "name": "N'Djamena Air Force Base", + "country": "France", + "operator": "France", + "branch": "air_force", + "lat": 12.134, + "lng": 15.034 + }, + { + "name": "Naval base of Heron", + "country": "France", + "operator": "France", + "branch": "navy", + "lat": 11.557, + "lng": 43.144 + }, + { + "name": "Les elements francais au Gabon", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 0.42, + "lng": 9.438 + }, + { + "name": "Fassberg Air Base", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 52.919, + "lng": 10.189 + }, + { + "name": "Les forces francaises en Cote d'Ivoire (FFCI)", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 7.504, + "lng": -5.549 + }, + { + "name": "Rayak Air Base", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 33.852, + "lng": 35.99 + }, + { + "name": "Niamey Air Force Base", + "country": "France", + "operator": "France", + "branch": "air_force", + "lat": 13.482, + "lng": 2.17 + }, + { + "name": "French Outpost (Manbij, Syria)", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 36.891, + "lng": 38.354 + }, + { + "name": "French Outpost (Ain Issa, Syria)", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 36.587, + "lng": 38.3 + }, + { + "name": "French Outpost (Raqqa, Syria)", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 36.385, + "lng": 38.859 + }, + { + "name": "Abu Dhabi Base", + "country": "France", + "operator": "France", + "branch": "army", + "lat": 24.522, + "lng": 54.396 + }, + { + "name": "Indian military training team", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 27.36, + "lng": 89.302 + }, + { + "name": "Port of Shahid Beheshti", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 25.298, + "lng": 60.611 + }, + { + "name": "Port of Sittwe", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 20.139, + "lng": 92.9 + }, + { + "name": "Ras al Hadd Listening post", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 22.533, + "lng": 59.798 + }, + { + "name": "Muscat naval base", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 23.588, + "lng": 58.279 + }, + { + "name": "Duqm port", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 19.666, + "lng": 57.726 + }, + { + "name": "Naval Facilities, Coastal Surveillance Radar (CSR) station", + "country": "India", + "operator": "India", + "branch": "navy", + "lat": -9.737, + "lng": 46.511 + }, + { + "name": "Farkhor air base", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 37.47, + "lng": 69.381 + }, + { + "name": "Berth rights and right to station its troops in Qatar", + "country": "India", + "operator": "India", + "branch": "army", + "lat": 25.308, + "lng": 51.209 + }, + { + "name": "Ahmad al-Jaber Air Base", + "country": "Italy", + "operator": "Italy", + "branch": "air_force", + "lat": 28.935, + "lng": 47.792 + }, + { + "name": "Libya Military Base", + "country": "Italy", + "operator": "Italy", + "branch": "army", + "lat": 24.96, + "lng": 10.177 + }, + { + "name": "Al Minhad air base", + "country": "Italy", + "operator": "Italy", + "branch": "air_force", + "lat": 25.027, + "lng": 55.366 + }, + { + "name": "Russian 102nd Military Base", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 40.79, + "lng": 43.825 + }, + { + "name": "Russian 3624th Airbase", + "country": "Russia", + "operator": "Russia", + "branch": "air_force", + "lat": 40.128, + "lng": 44.472 + }, + { + "name": "Vileyka VLF transmitter", + "country": "Russia", + "operator": "Russia", + "branch": "navy", + "lat": 54.464, + "lng": 26.778 + }, + { + "name": "Hantsavichy Radar Station", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 52.857, + "lng": 26.481 + }, + { + "name": "7th Krasnodar base", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 43.101, + "lng": 40.624 + }, + { + "name": "Russian 4th Military Base", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 42.39, + "lng": 43.922 + }, + { + "name": "Baikonur Cosmodrome", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 45.964, + "lng": 63.305 + }, + { + "name": "Sary Shagan", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 46.383, + "lng": 72.866 + }, + { + "name": "Balkhash Radar Station", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 46.603, + "lng": 74.53 + }, + { + "name": "Kant (air base)", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 42.853, + "lng": 74.846 + }, + { + "name": "Russian forces in Moldova", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 46.84, + "lng": 29.643 + }, + { + "name": "Khmeimim Air Base", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 35.411, + "lng": 35.945 + }, + { + "name": "Tiyas Military Airbase", + "country": "Russia", + "operator": "Russia", + "branch": "air_force", + "lat": 34.523, + "lng": 37.63 + }, + { + "name": "Shayrat Airbase", + "country": "Russia", + "operator": "Russia", + "branch": "air_force", + "lat": 34.49, + "lng": 36.909 + }, + { + "name": "Russian 201st Military Base", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 38.536, + "lng": 68.78 + }, + { + "name": "Military headquarters", + "country": "Russia", + "operator": "Russia", + "branch": "army", + "lat": 11.798, + "lng": -66.151 + }, + { + "name": "Rothera Research Station", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": -67.568, + "lng": -68.126 + }, + { + "name": "HMS Jufair", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 26.205, + "lng": 50.615 + }, + { + "name": "RAF Belize", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 17.544, + "lng": -88.305 + }, + { + "name": "British Army Jungle Warfare Training School", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 4.608, + "lng": 114.325 + }, + { + "name": "Sittang Camp", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 4.829, + "lng": 114.668 + }, + { + "name": "British Army Training Unit Suffield", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 50.273, + "lng": -111.175 + }, + { + "name": "RAF Troodos", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 34.912, + "lng": 32.883 + }, + { + "name": "RAF Akrotiri", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 34.59, + "lng": 32.987 + }, + { + "name": "Ayios Nikolaos Station", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 35.093, + "lng": 33.886 + }, + { + "name": "Westfalen Garrison", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 51.778, + "lng": 8.72 + }, + { + "name": "Wulfen barracks", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 51.705, + "lng": 6.999 + }, + { + "name": "Ayrshire barracks", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 51.171, + "lng": 6.393 + }, + { + "name": "RAF Gibraltar", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 36.152, + "lng": -5.344 + }, + { + "name": "British Gurkha Dharan", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 26.807, + "lng": 87.269 + }, + { + "name": "Headquarters British Gurkhas Nepal", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 27.668, + "lng": 85.317 + }, + { + "name": "Bardufoss Air Station", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 69.052, + "lng": 18.517 + }, + { + "name": "Omani-British Joint Training Area", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 19.014, + "lng": 57.749 + }, + { + "name": "Seeb, Overseas Processing Centre", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 23.675, + "lng": 58.121 + }, + { + "name": "RAF Al Udeid", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 25.11, + "lng": 51.319 + }, + { + "name": "British naval facility, base", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 1.464, + "lng": 103.826 + }, + { + "name": "RAF Mount Pleasant", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": -51.822, + "lng": -58.447 + }, + { + "name": "RAF Ascension", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": -7.969, + "lng": -14.393 + }, + { + "name": "Warwick Camp", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 32.257, + "lng": -64.815 + }, + { + "name": "Cayman Islands Regiment", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 19.293, + "lng": -81.378 + }, + { + "name": "RRH, an early warning and airspace control network", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": -52.153, + "lng": -60.598 + }, + { + "name": "Port Stanley Airport", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": -51.699, + "lng": -57.842 + }, + { + "name": "Jersey Field Squadron", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 49.175, + "lng": -2.108 + }, + { + "name": "Royal Montserrat Defence Force Headquarters", + "country": "United Kingdom", + "operator": "United Kingdoms", + "branch": "army", + "lat": 16.794, + "lng": -62.211 + }, + { + "name": "Robertson Barracks", + "country": "United States", + "operator": "United States", + "branch": "marines", + "lat": -12.44, + "lng": 130.97 + }, + { + "name": "Isa Air Base", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 25.912, + "lng": 50.593 + }, + { + "name": "USAG Brussels", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 50.85, + "lng": 4.349 + }, + { + "name": "Aitos Logistics Center", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 42.7, + "lng": 27.25 + }, + { + "name": "Bezmer", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 42.483, + "lng": 26.5 + }, + { + "name": "Graf Ignatievo", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 42.15, + "lng": 24.75 + }, + { + "name": "Contingency Location Garoua", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 9.333, + "lng": 13.372 + }, + { + "name": "Guantanamo", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 20.144, + "lng": -75.209 + }, + { + "name": "RAF Lakenheath", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 52.417, + "lng": 0.522 + }, + { + "name": "Royal Air Force Alconbury", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 52.369, + "lng": -0.26 + }, + { + "name": "Royal Air Force Croughton", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 52.25, + "lng": -0.833 + }, + { + "name": "RAF Mildenhall", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 51.426, + "lng": -1.7 + }, + { + "name": "Campbell Barracks", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.408, + "lng": 8.691 + }, + { + "name": "Landstuhl Medical Center", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.413, + "lng": 7.57 + }, + { + "name": "USAG Ansbach", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.3, + "lng": 10.583 + }, + { + "name": "USAG Bamberg", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.899, + "lng": 10.901 + }, + { + "name": "USAG Baumholder", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.617, + "lng": 7.334 + }, + { + "name": "USAG Garmisch", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 47.495, + "lng": 11.108 + }, + { + "name": "USAG Grafenwoehr", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.717, + "lng": 11.906 + }, + { + "name": "USAF Hessen", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 50.134, + "lng": 8.914 + }, + { + "name": "USAG Kaiserslautern", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 49.443, + "lng": 7.772 + }, + { + "name": "USAG Schweinfurt", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 50.049, + "lng": 10.222 + }, + { + "name": "USAG Stuttgart", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 48.782, + "lng": 9.177 + }, + { + "name": "USAG Wiesbaden", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 50.083, + "lng": 8.249 + }, + { + "name": "Spangdahlem", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 49.756, + "lng": 6.639 + }, + { + "name": "Panzer Kaserne", + "country": "United States", + "operator": "United States", + "branch": "marines", + "lat": 48.685, + "lng": 9.03 + }, + { + "name": "NSA Gaeta", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 41.214, + "lng": 13.571 + }, + { + "name": "Naval Support Activity", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 41.214, + "lng": 9.408 + }, + { + "name": "Camp Darby", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 43.627, + "lng": 10.292 + }, + { + "name": "Caserma Ederle", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 45.557, + "lng": 11.541 + }, + { + "name": "Aviano", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 46.071, + "lng": 12.595 + }, + { + "name": "Camp Fuji", + "country": "United States", + "operator": "United States", + "branch": "marines", + "lat": 35.317, + "lng": 138.933 + }, + { + "name": "Camp Bondsteel", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 42.367, + "lng": 21.133 + }, + { + "name": "Ali Al Salem Air Base", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 29.349, + "lng": 47.523 + }, + { + "name": "Camp Arifjan", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 28.875, + "lng": 48.159 + }, + { + "name": "Camp Buehring", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 29.695, + "lng": 47.421 + }, + { + "name": "Kuwait Naval Base", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 28.864, + "lng": 48.278 + }, + { + "name": "USAG Schinnen", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 50.943, + "lng": 5.889 + }, + { + "name": "Niger Air Base 201", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 16.921, + "lng": 8.026 + }, + { + "name": "Masirah Aira Base", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 20.667, + "lng": 58.897 + }, + { + "name": "RAFO Thumrait", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 17.664, + "lng": 54.026 + }, + { + "name": "Fort Magsaysay", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 15.435, + "lng": 121.091 + }, + { + "name": "Lumbia Airfield", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 8.405, + "lng": 124.61 + }, + { + "name": "Mactan-Benito Ebuen Air Base", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 10.313, + "lng": 123.978 + }, + { + "name": "Lajes Field", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 38.383, + "lng": -28.267 + }, + { + "name": "Camp Santiago", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 17.977, + "lng": -66.298 + }, + { + "name": "Al Udeid", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 25.279, + "lng": 51.522 + }, + { + "name": "Prince Sultan Air Base", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 24.077, + "lng": 47.564 + }, + { + "name": "COMLOG Westpac", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 1.29, + "lng": 103.85 + }, + { + "name": "Fleet Actvities Chinhae", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 35.103, + "lng": 129.04 + }, + { + "name": "Camp Red Cloud", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 37.742, + "lng": 127.047 + }, + { + "name": "USAG Daegu", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 35.87, + "lng": 128.591 + }, + { + "name": "K-16 Air Base", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 37.438, + "lng": 127.109 + }, + { + "name": "USAG Yongsan", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 37.533, + "lng": 126.983 + }, + { + "name": "U.S. Army Garrison CASEY", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 37.884, + "lng": 127.05 + }, + { + "name": "Naval Station", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 36.622, + "lng": -6.359 + }, + { + "name": "Izmir", + "country": "United States", + "operator": "United States", + "branch": "air_force", + "lat": 38.413, + "lng": 27.138 + }, + { + "name": "Al Dhafra Air Base", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 24.24, + "lng": 54.551 + }, + { + "name": "Port of Jebel Ali", + "country": "United States", + "operator": "United States", + "branch": "army", + "lat": 25.025, + "lng": 55.04 + }, + { + "name": "Fujairah Naval Base", + "country": "United States", + "operator": "United States", + "branch": "navy", + "lat": 25.252, + "lng": 56.365 + }, + { + "name": "PLA Forward Base (Central Asia)", + "country": "China", + "operator": "PLA", + "branch": "army", + "lat": 38.27, + "lng": 64.3 + }, + { + "name": "Ahmadaben Moses Training Garrison", + "country": "Iran", + "operator": "IRGC", + "branch": "army", + "lat": 32.66, + "lng": 51.67 + }, + { + "name": "Lesser Tunb Island Garrison", + "country": "Iran", + "operator": "IRGC Navy", + "branch": "navy", + "lat": 26.27, + "lng": 55.14 + }, + { + "name": "Abu Musa Island Garrison", + "country": "Iran", + "operator": "IRGC Navy", + "branch": "navy", + "lat": 25.876, + "lng": 55.033 + }, + { + "name": "S-300VM SAM Battery (38th AAM Brigade)", + "country": "Russia", + "operator": "VKS / 38th AAM", + "branch": "missile", + "lat": 34.76, + "lng": 36.89 + }, + { + "name": "Privolzhskiy Radar Station", + "country": "Russia", + "operator": "VKS", + "branch": "missile", + "lat": 50.38, + "lng": 45.37 + }, + { + "name": "Anti-Aircraft Artillery Officer Academy", + "country": "North Korea", + "operator": "KPA", + "branch": "army", + "lat": 38.975, + "lng": 125.716 + }, + { + "name": "SRBM Base (Underground)", + "country": "North Korea", + "operator": "KPA Strategic Force", + "branch": "missile", + "lat": 39.21, + "lng": 125.98 + }, + { + "name": "Dimona Radar Facility", + "country": "United States", + "operator": "US / IDF", + "branch": "army", + "lat": 31.022, + "lng": 35.198 + }, + { + "name": "Hatzerim Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 31.233, + "lng": 34.663 + }, + { + "name": "Nevatim Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 31.208, + "lng": 35.012 + }, + { + "name": "Ramat David Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 32.665, + "lng": 35.184 + }, + { + "name": "Tel Nof Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 31.839, + "lng": 34.822 + }, + { + "name": "Hatzor Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 31.762, + "lng": 34.727 + }, + { + "name": "Palmachim Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 31.897, + "lng": 34.691 + }, + { + "name": "Ramon Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 30.776, + "lng": 34.667 + }, + { + "name": "Ovda Air Force Base", + "country": "Israel", + "operator": "IAF", + "branch": "air_force", + "lat": 29.94, + "lng": 34.935 + }, + { + "name": "Haifa Naval Base", + "country": "Israel", + "operator": "Israeli Navy", + "branch": "navy", + "lat": 32.821, + "lng": 34.979 + }, + { + "name": "Atlit Naval Base", + "country": "Israel", + "operator": "Israeli Navy / Shayetet 13", + "branch": "navy", + "lat": 32.695, + "lng": 34.937 + }, + { + "name": "Camp Rabin (Tel HaShomer)", + "country": "Israel", + "operator": "IDF", + "branch": "army", + "lat": 32.052, + "lng": 34.843 + }, + { + "name": "Glilot Intelligence Campus", + "country": "Israel", + "operator": "IDF Unit 8200", + "branch": "army", + "lat": 32.146, + "lng": 34.788 + }, + { + "name": "Base navale de Toulon", + "country": "France", + "operator": "Marine Nationale", + "branch": "navy", + "lat": 43.117, + "lng": 5.928 + }, + { + "name": "Ile Longue SSBN Base", + "country": "France", + "operator": "Marine Nationale", + "branch": "navy", + "lat": 48.312, + "lng": -4.592 + }, + { + "name": "BA 113 Saint-Dizier", + "country": "France", + "operator": "Armee de l'Air", + "branch": "air_force", + "lat": 48.636, + "lng": 4.899 + }, + { + "name": "BA 118 Mont-de-Marsan", + "country": "France", + "operator": "Armee de l'Air", + "branch": "air_force", + "lat": 43.912, + "lng": -0.507 + }, + { + "name": "BA 125 Istres-Le Tube", + "country": "France", + "operator": "Armee de l'Air", + "branch": "air_force", + "lat": 43.523, + "lng": 4.924 + }, + { + "name": "BA 942 Lyon-Mont Verdun", + "country": "France", + "operator": "Armee de l'Air", + "branch": "air_force", + "lat": 45.851, + "lng": 4.765 + }, + { + "name": "BAN Landivisiau", + "country": "France", + "operator": "Marine Nationale", + "branch": "navy", + "lat": 48.531, + "lng": -4.152 + }, + { + "name": "Buchel Air Base", + "country": "Germany", + "operator": "Luftwaffe / NATO", + "branch": "air_force", + "lat": 50.174, + "lng": 7.063 + }, + { + "name": "Norvenich Air Base", + "country": "Germany", + "operator": "Luftwaffe", + "branch": "air_force", + "lat": 50.831, + "lng": 6.658 + }, + { + "name": "Laage Air Base", + "country": "Germany", + "operator": "Luftwaffe", + "branch": "air_force", + "lat": 53.919, + "lng": 12.277 + }, + { + "name": "Kiel Naval Base", + "country": "Germany", + "operator": "Deutsche Marine", + "branch": "navy", + "lat": 54.328, + "lng": 10.163 + }, + { + "name": "Wilhelmshaven Naval Base", + "country": "Germany", + "operator": "Deutsche Marine", + "branch": "navy", + "lat": 53.514, + "lng": 8.149 + }, + { + "name": "Ghedi Air Base", + "country": "Italy", + "operator": "AMI / NATO", + "branch": "air_force", + "lat": 45.432, + "lng": 10.277 + }, + { + "name": "Gioia del Colle Air Base", + "country": "Italy", + "operator": "AMI", + "branch": "air_force", + "lat": 40.768, + "lng": 16.933 + }, + { + "name": "La Spezia Naval Base", + "country": "Italy", + "operator": "Marina Militare", + "branch": "navy", + "lat": 44.103, + "lng": 9.837 + }, + { + "name": "Taranto Naval Base", + "country": "Italy", + "operator": "Marina Militare", + "branch": "navy", + "lat": 40.474, + "lng": 17.234 + }, + { + "name": "Moron Air Base", + "country": "Spain", + "operator": "Ejercito del Aire / USAF", + "branch": "air_force", + "lat": 37.175, + "lng": -5.616 + }, + { + "name": "Zaragoza Air Base", + "country": "Spain", + "operator": "Ejercito del Aire", + "branch": "air_force", + "lat": 41.666, + "lng": -1.042 + }, + { + "name": "Lask Air Base", + "country": "Poland", + "operator": "Polish Air Force", + "branch": "air_force", + "lat": 51.551, + "lng": 19.179 + }, + { + "name": "Powidz Air Base", + "country": "Poland", + "operator": "Polish Air Force", + "branch": "air_force", + "lat": 52.38, + "lng": 17.854 + }, + { + "name": "Redzikowo (Aegis Ashore BMD)", + "country": "Poland", + "operator": "US Navy / NATO", + "branch": "missile", + "lat": 54.479, + "lng": 17.099 + }, + { + "name": "Larissa Air Base", + "country": "Greece", + "operator": "HAF", + "branch": "air_force", + "lat": 39.635, + "lng": 22.465 + }, + { + "name": "Souda Bay Naval Base", + "country": "Greece", + "operator": "Hellenic Navy / NATO", + "branch": "navy", + "lat": 35.483, + "lng": 24.125 + }, + { + "name": "Volkel Air Base", + "country": "Netherlands", + "operator": "RNLAF / NATO", + "branch": "air_force", + "lat": 51.657, + "lng": 5.707 + }, + { + "name": "Den Helder Naval Base", + "country": "Netherlands", + "operator": "RNLN", + "branch": "navy", + "lat": 52.953, + "lng": 4.786 + }, + { + "name": "INS Vikramaditya Home Port (Karwar)", + "country": "India", + "operator": "Indian Navy", + "branch": "navy", + "lat": 14.82, + "lng": 74.124 + }, + { + "name": "Visakhapatnam Naval Base", + "country": "India", + "operator": "Indian Navy Eastern Command", + "branch": "navy", + "lat": 17.686, + "lng": 83.297 + }, + { + "name": "Mumbai Naval Dockyard", + "country": "India", + "operator": "Indian Navy Western Command", + "branch": "navy", + "lat": 18.924, + "lng": 72.842 + }, + { + "name": "INS Rajali (Arakkonam)", + "country": "India", + "operator": "Indian Navy", + "branch": "navy", + "lat": 13.073, + "lng": 79.693 + }, + { + "name": "Ambala Air Force Station", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 30.368, + "lng": 76.817 + }, + { + "name": "Hashimara Air Force Station", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 26.698, + "lng": 89.379 + }, + { + "name": "Thanjavur Air Force Station", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 10.724, + "lng": 79.101 + }, + { + "name": "Kalaikunda Air Force Station", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 22.339, + "lng": 87.221 + }, + { + "name": "Agra Air Force Station", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 27.158, + "lng": 77.961 + }, + { + "name": "Pune Air Force Station", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 18.582, + "lng": 73.919 + }, + { + "name": "Leh Air Force Base", + "country": "India", + "operator": "IAF", + "branch": "air_force", + "lat": 34.136, + "lng": 77.546 + }, + { + "name": "Kahuta Research Laboratories", + "country": "Pakistan", + "operator": "KRL / SPD", + "branch": "nuclear", + "lat": 33.606, + "lng": 73.386 + }, + { + "name": "Kamra Air Complex (PAC)", + "country": "Pakistan", + "operator": "PAF / SPD", + "branch": "air_force", + "lat": 33.87, + "lng": 72.405 + }, + { + "name": "PAF Base Masroor", + "country": "Pakistan", + "operator": "PAF", + "branch": "air_force", + "lat": 24.894, + "lng": 66.939 + }, + { + "name": "PAF Base Sargodha", + "country": "Pakistan", + "operator": "PAF", + "branch": "air_force", + "lat": 32.049, + "lng": 72.671 + }, + { + "name": "PAF Base Shahbaz", + "country": "Pakistan", + "operator": "PAF", + "branch": "air_force", + "lat": 28.284, + "lng": 68.45 + }, + { + "name": "Rawalpindi GHQ", + "country": "Pakistan", + "operator": "Pakistan Army", + "branch": "army", + "lat": 33.597, + "lng": 73.048 + }, + { + "name": "Wah Cantonment Ordnance", + "country": "Pakistan", + "operator": "Pakistan Army", + "branch": "army", + "lat": 33.782, + "lng": 72.747 + }, + { + "name": "Gwadar Naval Base", + "country": "Pakistan", + "operator": "Pakistan Navy", + "branch": "navy", + "lat": 25.13, + "lng": 62.33 + }, + { + "name": "Ormara Naval Base", + "country": "Pakistan", + "operator": "Pakistan Navy", + "branch": "navy", + "lat": 25.21, + "lng": 64.64 + }, + { + "name": "Natanz Uranium Enrichment Facility", + "country": "Iran", + "operator": "AEOI", + "branch": "nuclear", + "lat": 33.724, + "lng": 51.727 + }, + { + "name": "Fordow Underground Enrichment Plant", + "country": "Iran", + "operator": "AEOI", + "branch": "nuclear", + "lat": 34.876, + "lng": 51.602 + }, + { + "name": "Bushehr Nuclear Power Plant", + "country": "Iran", + "operator": "AEOI", + "branch": "nuclear", + "lat": 28.83, + "lng": 50.886 + }, + { + "name": "Parchin Military Complex", + "country": "Iran", + "operator": "MoD / SPND", + "branch": "nuclear", + "lat": 35.518, + "lng": 51.773 + }, + { + "name": "Arak Heavy Water Reactor (IR-40)", + "country": "Iran", + "operator": "AEOI", + "branch": "nuclear", + "lat": 34.046, + "lng": 49.242 + }, + { + "name": "Behjatabad-Abyek Enrichment Facility (Site 311)", + "country": "Iran", + "operator": "AEOI", + "branch": "nuclear", + "lat": 36.045, + "lng": 50.508 + }, + { + "name": "Shahrud Missile Test Site", + "country": "Iran", + "operator": "IRGC Aerospace", + "branch": "missile", + "lat": 36.428, + "lng": 55.059 + }, + { + "name": "Semnan Space Launch Center", + "country": "Iran", + "operator": "IRGC Aerospace", + "branch": "missile", + "lat": 35.235, + "lng": 53.921 + }, + { + "name": "Imam Ali Missile Base (Khorramabad)", + "country": "Iran", + "operator": "IRGC Aerospace", + "branch": "missile", + "lat": 33.442, + "lng": 48.331 + }, + { + "name": "Tabriz Missile Base", + "country": "Iran", + "operator": "IRGC Aerospace", + "branch": "missile", + "lat": 38.065, + "lng": 46.405 + }, + { + "name": "Kermanshah Missile Base", + "country": "Iran", + "operator": "IRGC Aerospace", + "branch": "missile", + "lat": 34.388, + "lng": 47.074 + }, + { + "name": "Bid Kaneh Missile Base", + "country": "Iran", + "operator": "IRGC Aerospace", + "branch": "missile", + "lat": 33.727, + "lng": 47.424 + }, + { + "name": "Khatam al-Anbiya AD HQ (Tehran)", + "country": "Iran", + "operator": "IRGC Air Defense Force", + "branch": "missile", + "lat": 35.7, + "lng": 51.424 + }, + { + "name": "TAB 1 Mehrabad Air Base (Tehran)", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 35.689, + "lng": 51.313 + }, + { + "name": "TAB 3 Nojeh Air Base (Hamadan)", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 34.874, + "lng": 48.654 + }, + { + "name": "TAB 4 Dezful Air Base", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 32.434, + "lng": 48.398 + }, + { + "name": "TAB 6 Bushehr Air Base", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 28.949, + "lng": 50.835 + }, + { + "name": "TAB 7 Shiraz Air Base", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 29.54, + "lng": 52.589 + }, + { + "name": "TAB 8 Khatami Air Base (Isfahan)", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 32.751, + "lng": 51.862 + }, + { + "name": "TAB 9 Bandar Abbas Air Base", + "country": "Iran", + "operator": "IRIAF", + "branch": "air_force", + "lat": 27.217, + "lng": 56.179 + }, + { + "name": "Jask Naval Base", + "country": "Iran", + "operator": "IRIN", + "branch": "navy", + "lat": 25.641, + "lng": 57.772 + }, + { + "name": "Bandar-e Anzali Naval Base", + "country": "Iran", + "operator": "IRIN", + "branch": "navy", + "lat": 37.462, + "lng": 49.477 + }, + { + "name": "Sdot Micha Missile Base", + "country": "Israel", + "operator": "IAF / Strategic Forces", + "branch": "missile", + "lat": 31.73, + "lng": 34.934 + }, + { + "name": "Arrow Battery (Ein Shemer)", + "country": "Israel", + "operator": "IAF Air Defense", + "branch": "missile", + "lat": 32.43, + "lng": 34.93 + }, + { + "name": "Iron Dome Battery (Sderot)", + "country": "Israel", + "operator": "IAF Air Defense", + "branch": "missile", + "lat": 31.524, + "lng": 34.596 + }, + { + "name": "Mount Hermon Intelligence Station", + "country": "Israel", + "operator": "IDF Intelligence", + "branch": "army", + "lat": 33.416, + "lng": 35.857 + }, + { + "name": "Bhabha Atomic Research Centre (BARC)", + "country": "India", + "operator": "DAE", + "branch": "nuclear", + "lat": 19.012, + "lng": 72.921 + }, + { + "name": "Indira Gandhi CARS (Kalpakkam)", + "country": "India", + "operator": "DAE", + "branch": "nuclear", + "lat": 12.556, + "lng": 80.176 + }, + { + "name": "Pokhran Nuclear Test Range", + "country": "India", + "operator": "DRDO / DAE", + "branch": "nuclear", + "lat": 26.731, + "lng": 71.091 + }, + { + "name": "Abdul Kalam Island (Wheeler Island)", + "country": "India", + "operator": "DRDO", + "branch": "missile", + "lat": 20.748, + "lng": 87.075 + }, + { + "name": "Chandipur ITR (Integrated Test Range)", + "country": "India", + "operator": "DRDO", + "branch": "missile", + "lat": 21.452, + "lng": 87.012 + }, + { + "name": "Agni Missile Complex (Secunderabad)", + "country": "India", + "operator": "SFC", + "branch": "missile", + "lat": 17.439, + "lng": 78.498 + }, + { + "name": "BrahMos Aerospace (Hyderabad)", + "country": "India", + "operator": "BrahMos / DRDO", + "branch": "missile", + "lat": 17.472, + "lng": 78.579 + }, + { + "name": "INS Arihant Base (Rambilli)", + "country": "India", + "operator": "Indian Navy SFC", + "branch": "nuclear", + "lat": 17.579, + "lng": 83.262 + }, + { + "name": "Nuclear Fuel Complex (Hyderabad)", + "country": "India", + "operator": "DAE", + "branch": "nuclear", + "lat": 17.503, + "lng": 78.392 + }, + { + "name": "Rare Materials Plant (Mysore)", + "country": "India", + "operator": "DAE", + "branch": "nuclear", + "lat": 12.277, + "lng": 76.636 + }, + { + "name": "Challakere Nuclear City", + "country": "India", + "operator": "DAE / DRDO", + "branch": "nuclear", + "lat": 14.318, + "lng": 76.655 + }, + { + "name": "Sukhna Cantonment (Strike Corps HQ)", + "country": "India", + "operator": "Indian Army", + "branch": "army", + "lat": 30.853, + "lng": 76.862 + }, + { + "name": "Khushab Nuclear Complex", + "country": "Pakistan", + "operator": "PAEC", + "branch": "nuclear", + "lat": 32.025, + "lng": 72.231 + }, + { + "name": "Chagai Nuclear Test Site", + "country": "Pakistan", + "operator": "PAEC", + "branch": "nuclear", + "lat": 28.979, + "lng": 64.946 + }, + { + "name": "National Defence Complex (Fateh Jang)", + "country": "Pakistan", + "operator": "NDC / SPD", + "branch": "missile", + "lat": 33.559, + "lng": 72.371 + }, + { + "name": "Tilla Ranges (Missile Test)", + "country": "Pakistan", + "operator": "Pakistan Army SPD", + "branch": "missile", + "lat": 32.834, + "lng": 72.751 + }, + { + "name": "Sonmiani Test Range", + "country": "Pakistan", + "operator": "SUPARCO / SPD", + "branch": "missile", + "lat": 25.413, + "lng": 66.665 + }, + { + "name": "Hatf Missile Garrison (Gujranwala)", + "country": "Pakistan", + "operator": "Pakistan Army SPD", + "branch": "missile", + "lat": 32.161, + "lng": 74.187 + }, + { + "name": "Pakistan Ordnance Factories (POF)", + "country": "Pakistan", + "operator": "Ministry of Defence", + "branch": "army", + "lat": 33.782, + "lng": 72.447 + }, + { + "name": "HMNB Clyde (Faslane) SSBN Base", + "country": "United Kingdom", + "operator": "Royal Navy", + "branch": "navy", + "lat": 56.067, + "lng": -4.822 + }, + { + "name": "AWE Aldermaston (Warhead Factory)", + "country": "United Kingdom", + "operator": "AWE / MoD", + "branch": "nuclear", + "lat": 51.362, + "lng": -1.152 + }, + { + "name": "HMNB Devonport", + "country": "United Kingdom", + "operator": "Royal Navy", + "branch": "navy", + "lat": 50.383, + "lng": -4.184 + }, + { + "name": "HMNB Portsmouth", + "country": "United Kingdom", + "operator": "Royal Navy", + "branch": "navy", + "lat": 50.8, + "lng": -1.104 + }, + { + "name": "GCHQ Cheltenham", + "country": "United Kingdom", + "operator": "GCHQ / MoD", + "branch": "army", + "lat": 51.899, + "lng": -2.124 + }, + { + "name": "RAF Fylingdales (BMD Radar)", + "country": "United Kingdom", + "operator": "RAF / US Space Force", + "branch": "air_force", + "lat": 54.361, + "lng": -0.669 + }, + { + "name": "RAF Coningsby (Typhoon QRA)", + "country": "United Kingdom", + "operator": "RAF", + "branch": "air_force", + "lat": 53.093, + "lng": -0.166 + }, + { + "name": "RAF Waddington (ISTAR)", + "country": "United Kingdom", + "operator": "RAF", + "branch": "air_force", + "lat": 53.166, + "lng": -0.521 + }, + { + "name": "RAF Brize Norton (Airlift Hub)", + "country": "United Kingdom", + "operator": "RAF", + "branch": "air_force", + "lat": 51.749, + "lng": -1.583 + }, + { + "name": "BAE Barrow Submarine Yard", + "country": "United Kingdom", + "operator": "BAE Systems / MoD", + "branch": "navy", + "lat": 54.107, + "lng": -3.225 + }, + { + "name": "Menwith Hill Station (NSA/GCHQ)", + "country": "United Kingdom", + "operator": "NSA / GCHQ", + "branch": "army", + "lat": 54.015, + "lng": -1.688 + }, + { + "name": "Engels Air Base (Tu-160 Bombers)", + "country": "Russia", + "operator": "VKS Long Range Aviation", + "branch": "air_force", + "lat": 51.475, + "lng": 46.215 + }, + { + "name": "Kozelsk ICBM Base (SS-27 Topol-M)", + "country": "Russia", + "operator": "RVSN", + "branch": "missile", + "lat": 54.05, + "lng": 36.017 + }, + { + "name": "Tatishchevo ICBM Base (SS-27)", + "country": "Russia", + "operator": "RVSN", + "branch": "missile", + "lat": 51.73, + "lng": 45.717 + }, + { + "name": "Dombarovsky ICBM Base (SS-18 Satan)", + "country": "Russia", + "operator": "RVSN", + "branch": "missile", + "lat": 50.757, + "lng": 59.523 + }, + { + "name": "Uzhur ICBM Base (SS-18)", + "country": "Russia", + "operator": "RVSN", + "branch": "missile", + "lat": 55.317, + "lng": 89.783 + }, + { + "name": "Novosibirsk ICBM Base (SS-25)", + "country": "Russia", + "operator": "RVSN", + "branch": "missile", + "lat": 54.853, + "lng": 83.117 + }, + { + "name": "Plesetsk Cosmodrome (ICBM/Space)", + "country": "Russia", + "operator": "VKS / RVSN", + "branch": "missile", + "lat": 62.926, + "lng": 40.577 + }, + { + "name": "Gadzhiyevo SSBN Base (Delta/Borei)", + "country": "Russia", + "operator": "Russian Navy", + "branch": "navy", + "lat": 69.253, + "lng": 33.315 + }, + { + "name": "Novaya Zemlya Nuclear Test Site", + "country": "Russia", + "operator": "12th GUMO", + "branch": "nuclear", + "lat": 73.37, + "lng": 54.97 + }, + { + "name": "Sarov (VNIIEF Nuclear Design Lab)", + "country": "Russia", + "operator": "Rosatom", + "branch": "nuclear", + "lat": 54.933, + "lng": 43.317 + }, + { + "name": "Kaliningrad (Baltic Fleet / Iskander)", + "country": "Russia", + "operator": "Russian Navy / Army", + "branch": "missile", + "lat": 54.71, + "lng": 20.51 + }, + { + "name": "Murmansk-150 / Vidyayevo (SSBN)", + "country": "Russia", + "operator": "Russian Navy", + "branch": "navy", + "lat": 69.317, + "lng": 32.95 + }, + { + "name": "Lida Air Base (Belarus, Su-30SM)", + "country": "Russia", + "operator": "VKS / Belarus AF", + "branch": "air_force", + "lat": 53.88, + "lng": 25.3 + }, + { + "name": "Chernyakhovsk Air Base (Kaliningrad)", + "country": "Russia", + "operator": "VKS", + "branch": "air_force", + "lat": 54.6, + "lng": 21.8 + }, + { + "name": "CEA DAM Ile-de-France (Warhead Design)", + "country": "France", + "operator": "CEA / MoD", + "branch": "nuclear", + "lat": 48.736, + "lng": 2.15 + }, + { + "name": "FOST Brest (SSBN Command)", + "country": "France", + "operator": "Marine Nationale", + "branch": "navy", + "lat": 48.382, + "lng": -4.495 + }, + { + "name": "CEL Biscarrosse (Missile Test Range)", + "country": "France", + "operator": "DGA", + "branch": "missile", + "lat": 44.378, + "lng": -1.232 + }, + { + "name": "BA 120 Cazaux (Weapons Test)", + "country": "France", + "operator": "Armee de l'Air", + "branch": "air_force", + "lat": 44.533, + "lng": -1.125 + }, + { + "name": "BA 116 Luxeuil (Rafale Nuclear Strike)", + "country": "France", + "operator": "Armee de l'Air", + "branch": "air_force", + "lat": 47.788, + "lng": 6.359 + }, + { + "name": "BN Cherbourg (Submarine Construction)", + "country": "France", + "operator": "Naval Group", + "branch": "navy", + "lat": 49.635, + "lng": -1.617 + }, + { + "name": "Mianyang (CAEP Nuclear Weapons Lab)", + "country": "China", + "operator": "CAEP / CMC", + "branch": "nuclear", + "lat": 31.464, + "lng": 104.741 + }, + { + "name": "Xichang Satellite Launch Center", + "country": "China", + "operator": "PLA SSF", + "branch": "missile", + "lat": 28.246, + "lng": 102.027 + }, + { + "name": "Jiuquan Launch Center", + "country": "China", + "operator": "PLA SSF", + "branch": "missile", + "lat": 40.958, + "lng": 100.291 + }, + { + "name": "Wenchang Spacecraft Launch Site", + "country": "China", + "operator": "PLA SSF", + "branch": "missile", + "lat": 19.614, + "lng": 110.951 + }, + { + "name": "Delingha PLARF Base (DF-31)", + "country": "China", + "operator": "PLARF", + "branch": "missile", + "lat": 37.37, + "lng": 97.37 + }, + { + "name": "Datong PLARF Base (DF-41)", + "country": "China", + "operator": "PLARF", + "branch": "missile", + "lat": 40.092, + "lng": 113.291 + }, + { + "name": "Jilantai Missile Training Area", + "country": "China", + "operator": "PLARF", + "branch": "missile", + "lat": 39.81, + "lng": 105.59 + }, + { + "name": "Huludao Submarine Shipyard", + "country": "China", + "operator": "PLAN", + "branch": "navy", + "lat": 40.739, + "lng": 120.853 + }, + { + "name": "Yumen Nuclear Fuel Facility", + "country": "China", + "operator": "CNNC", + "branch": "nuclear", + "lat": 39.833, + "lng": 97.033 + }, + { + "name": "Yongdoktong Missile Base", + "country": "North Korea", + "operator": "KPA Strategic Force", + "branch": "missile", + "lat": 39.95, + "lng": 126.65 + }, + { + "name": "Sakkanmol Missile Base", + "country": "North Korea", + "operator": "KPA Strategic Force", + "branch": "missile", + "lat": 38.28, + "lng": 126.0 + }, + { + "name": "Sino-ri Missile Operating Base", + "country": "North Korea", + "operator": "KPA Strategic Force", + "branch": "missile", + "lat": 39.649, + "lng": 125.866 + }, + { + "name": "Hamhung Chemical Complex", + "country": "North Korea", + "operator": "KPA / 2nd Economic Committee", + "branch": "nuclear", + "lat": 39.931, + "lng": 127.544 + }, + { + "name": "Hsiung Feng Missile Battery (Keelung)", + "country": "Taiwan", + "operator": "ROCN", + "branch": "missile", + "lat": 25.149, + "lng": 121.743 + }, + { + "name": "Tien Kung SAM Site (Taipei)", + "country": "Taiwan", + "operator": "ROCAF", + "branch": "missile", + "lat": 25.074, + "lng": 121.558 + }, + { + "name": "Chungshan Institute of Science (NCSIST)", + "country": "Taiwan", + "operator": "MND / NCSIST", + "branch": "missile", + "lat": 24.851, + "lng": 121.213 + }, + { + "name": "Leshan Radar Station (PAVE PAWS)", + "country": "Taiwan", + "operator": "ROCAF", + "branch": "air_force", + "lat": 23.526, + "lng": 120.401 + }, + { + "name": "Gyeryongdae Military Complex (Joint HQ)", + "country": "South Korea", + "operator": "ROK JCS", + "branch": "army", + "lat": 36.272, + "lng": 127.054 + }, + { + "name": "Jinhae Naval Base (ROKN HQ)", + "country": "South Korea", + "operator": "ROKN", + "branch": "navy", + "lat": 35.147, + "lng": 128.698 + }, + { + "name": "Hyunmu Missile Site (Wonju)", + "country": "South Korea", + "operator": "ROK Army Missile Command", + "branch": "missile", + "lat": 37.332, + "lng": 127.92 + }, + { + "name": "ADD Daejeon (Weapons R&D)", + "country": "South Korea", + "operator": "Agency for Defense Development", + "branch": "missile", + "lat": 36.385, + "lng": 127.372 + }, + { + "name": "Exmouth (Naval Communications Station)", + "country": "Australia", + "operator": "ADF / USN", + "branch": "navy", + "lat": -21.816, + "lng": 114.166 + }, + { + "name": "RAAF Base Amberley", + "country": "Australia", + "operator": "RAAF", + "branch": "air_force", + "lat": -27.637, + "lng": 152.711 + }, + { + "name": "RAAF Base Williamtown", + "country": "Australia", + "operator": "RAAF", + "branch": "air_force", + "lat": -32.793, + "lng": 151.837 + }, + { + "name": "Jagel Air Base (Eurofighter)", + "country": "Germany", + "operator": "Luftwaffe", + "branch": "air_force", + "lat": 54.46, + "lng": 9.52 + }, + { + "name": "Sigonella NAS (NATO AGS)", + "country": "Italy", + "operator": "USN / NATO", + "branch": "navy", + "lat": 37.401, + "lng": 14.922 + }, + { + "name": "Amendola Air Base (F-35)", + "country": "Italy", + "operator": "AMI", + "branch": "air_force", + "lat": 41.541, + "lng": 15.718 + }, + { + "name": "Base Naval de Ferrol", + "country": "Spain", + "operator": "Armada Espanola", + "branch": "navy", + "lat": 43.483, + "lng": -8.233 + }, + { + "name": "Eindhoven Air Base (KDC-10)", + "country": "Netherlands", + "operator": "RNLAF", + "branch": "air_force", + "lat": 51.449, + "lng": 5.374 + }, + { + "name": "Leeuwarden Air Base (F-35)", + "country": "Netherlands", + "operator": "RNLAF", + "branch": "air_force", + "lat": 53.228, + "lng": 5.76 + }, + { + "name": "Araxos Air Base (Nuclear Storage)", + "country": "Greece", + "operator": "HAF / NATO", + "branch": "air_force", + "lat": 38.151, + "lng": 21.425 + }, + { + "name": "Salamis Naval Base (HN HQ)", + "country": "Greece", + "operator": "Hellenic Navy", + "branch": "navy", + "lat": 37.942, + "lng": 23.498 + }, + { + "name": "Negev Nuclear Research Center (Dimona)", + "country": "Israel", + "operator": "IAEC", + "branch": "nuclear", + "lat": 31.001, + "lng": 35.145 + }, + { + "name": "Soreq Nuclear Research Center", + "country": "Israel", + "operator": "IAEC", + "branch": "nuclear", + "lat": 31.767, + "lng": 34.892 + }, + { + "name": "American Samoa", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -14.165, + "lng": -170.42 + }, + { + "name": "Alice Springs", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -25.274, + "lng": 133.775 + }, + { + "name": "Andros Island", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 24.706, + "lng": -78.02 + }, + { + "name": "Chievres", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.586, + "lng": 3.806 + }, + { + "name": "Florennes", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.251, + "lng": 4.605 + }, + { + "name": "Kleine-Brogel", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 51.133, + "lng": 5.454 + }, + { + "name": "Mons", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.454, + "lng": 3.957 + }, + { + "name": "Novo Selo Training Area", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 42.72, + "lng": 26.592 + }, + { + "name": "Navy Med Research Unit 2, Pnom Pehn", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": 11.55, + "lng": 104.917 + }, + { + "name": "Guantanamo Bay\tCuba", + "country": "United States", + "operator": "USN", + "branch": "navy", + "lat": 20.021, + "lng": -75.114 + }, + { + "name": "Comalapa", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.441, + "lng": -89.056 + }, + { + "name": "Thule\tGreenland", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 77.483, + "lng": -69.345 + }, + { + "name": "Soto Cano/Palmerola", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 14.382, + "lng": -87.617 + }, + { + "name": "Grindavik\tIceland", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 63.846, + "lng": -22.445 + }, + { + "name": "Green Zone, Baghdad", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 33.308, + "lng": 44.39 + }, + { + "name": "Catania", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.508, + "lng": 15.083 + }, + { + "name": "Naples", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 40.852, + "lng": 14.268 + }, + { + "name": "Coltano", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 43.619, + "lng": 10.404 + }, + { + "name": "Pordenone", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 45.963, + "lng": 12.655 + }, + { + "name": "Brescia", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 45.541, + "lng": 10.219 + }, + { + "name": "Maniago", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 46.165, + "lng": 12.707 + }, + { + "name": "Pisa", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 43.723, + "lng": 10.402 + }, + { + "name": "San Vito Normanni", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 40.658, + "lng": 17.706 + }, + { + "name": "Niscemi Sicily", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.15, + "lng": 14.383 + }, + { + "name": "Tokyo", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.689, + "lng": 139.692 + }, + { + "name": "Akizuki", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 33.47, + "lng": 130.691 + }, + { + "name": "Henoko Okinawa", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 26.523, + "lng": 128.032 + }, + { + "name": "Sagamihara", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.571, + "lng": 139.373 + }, + { + "name": "Okinawa Island", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 26.501, + "lng": 127.945 + }, + { + "name": "Hachinohe", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 40.512, + "lng": 141.488 + }, + { + "name": "Fukuoka", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 33.59, + "lng": 130.402 + }, + { + "name": "Iwo Jima", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 24.774, + "lng": 141.327 + }, + { + "name": "Kami Seya", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.638, + "lng": 135.187 + }, + { + "name": "Higashi-Hiroshima", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 34.395, + "lng": 132.482 + }, + { + "name": "Kisarazu", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.376, + "lng": 139.917 + }, + { + "name": "Yokohama", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.444, + "lng": 139.638 + }, + { + "name": "Koza", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.374, + "lng": 139.391 + }, + { + "name": "Waco City", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.781, + "lng": 139.606 + }, + { + "name": "Totsuka", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 36.893, + "lng": 140.418 + }, + { + "name": "Japan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 36.205, + "lng": 138.253 + }, + { + "name": "Mombasa\tKenya", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -4.043, + "lng": 39.668 + }, + { + "name": "Chechon", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.124, + "lng": 128.133 + }, + { + "name": "Inchon", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.456, + "lng": 126.705 + }, + { + "name": "Kimhae", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.229, + "lng": 128.889 + }, + { + "name": "Kunsan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.968, + "lng": 126.737 + }, + { + "name": "Kwangju", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.16, + "lng": 126.853 + }, + { + "name": "Masan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.214, + "lng": 128.581 + }, + { + "name": "Munsan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.89, + "lng": 126.699 + }, + { + "name": "Paju", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.76, + "lng": 126.78 + }, + { + "name": "Pochon", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.895, + "lng": 127.2 + }, + { + "name": "Pohang\t(Camp Mujuk)", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 36.019, + "lng": 129.343 + }, + { + "name": "Pyeongtaek", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 36.992, + "lng": 127.113 + }, + { + "name": "Suwon", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.264, + "lng": 127.029 + }, + { + "name": "Taepeak", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.164, + "lng": 128.986 + }, + { + "name": "Waegwan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.989, + "lng": 128.398 + }, + { + "name": "Yechon", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 36.658, + "lng": 128.453 + }, + { + "name": "Yongpyong", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 34.182, + "lng": 126.674 + }, + { + "name": "Al Mubarak Ab @ Kuwait City Iap", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 29.227, + "lng": 47.98 + }, + { + "name": "Khabari/Kheybari Military Crossing", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 29.9, + "lng": 47.183 + }, + { + "name": "Kwajalein R.Reagan Test Site", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 8.72, + "lng": 167.733 + }, + { + "name": "Brunssum", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.949, + "lng": 5.972 + }, + { + "name": "Tinian", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.0, + "lng": 145.633 + }, + { + "name": "Saipan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.183, + "lng": 145.75 + }, + { + "name": "Farallon De Medinilla", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.017, + "lng": 146.059 + }, + { + "name": "Norway", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 67.271, + "lng": 14.383 + }, + { + "name": "Sur Masirah", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 20.421, + "lng": 58.73 + }, + { + "name": "Salalah", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 17.014, + "lng": 54.092 + }, + { + "name": "Mina Qabus, Muscat", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 23.628, + "lng": 58.568 + }, + { + "name": "Raysat", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.934, + "lng": 53.992 + }, + { + "name": "Al Mussanah Ab", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 23.627, + "lng": 57.491 + }, + { + "name": "Lima\tPeru", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -12.048, + "lng": -77.062 + }, + { + "name": "Lajesfield\tAzores", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 38.762, + "lng": -27.095 + }, + { + "name": "Portugal", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": 39.4, + "lng": -8.224 + }, + { + "name": "As Sayliyah", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 25.188, + "lng": 51.412 + }, + { + "name": "Umm Said", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 24.98, + "lng": 51.55 + }, + { + "name": "Falcon-78 Asp", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 25.256, + "lng": 50.958 + }, + { + "name": "Mihail Kogalniceanu (Mk) Ab", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 44.362, + "lng": 28.488 + }, + { + "name": "Deveselu", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 44.083, + "lng": 24.383 + }, + { + "name": "Babadag Training Base", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 44.898, + "lng": 28.742 + }, + { + "name": "Ankara", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 39.921, + "lng": 32.854 + }, + { + "name": "Batman", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 37.881, + "lng": 41.135 + }, + { + "name": "Incirlik Ab, Adana", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 37.0, + "lng": 35.321 + }, + { + "name": "Yumurzalik", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 36.775, + "lng": 35.788 + }, + { + "name": "Fujairah", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 25.1, + "lng": 56.21 + }, + { + "name": "Barford St John", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 51.996, + "lng": -1.362 + }, + { + "name": "Bicester", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 51.9, + "lng": -1.154 + }, + { + "name": "Blenheim Crescent", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 51.569, + "lng": -0.434 + }, + { + "name": "Cambridge", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 52.205, + "lng": 0.122 + }, + { + "name": "Menwith Hill, Harrogate", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 53.992, + "lng": -1.542 + }, + { + "name": "Croughton", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 51.998, + "lng": -1.211 + }, + { + "name": "Ely", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 52.4, + "lng": 0.262 + }, + { + "name": "Fairford", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 51.708, + "lng": -1.785 + }, + { + "name": "Welford", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 52.415, + "lng": -1.056 + }, + { + "name": "St. Croix", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 17.728, + "lng": -64.824 + }, + { + "name": "Fort Buchanan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 18.413, + "lng": -66.122 + }, + { + "name": "Amberg", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.44, + "lng": 11.863 + }, + { + "name": "Bann", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.737, + "lng": 7.098 + }, + { + "name": "Darmstadt", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.878, + "lng": 8.652 + }, + { + "name": "Dexheim", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.846, + "lng": 8.316 + }, + { + "name": "Einsiedlerhof", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.441, + "lng": 7.666 + }, + { + "name": "Geilenkirchen", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.967, + "lng": 6.117 + }, + { + "name": "Germersheim", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.214, + "lng": 8.367 + }, + { + "name": "Giessen", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.587, + "lng": 8.691 + }, + { + "name": "Gruenstadt", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.563, + "lng": 8.168 + }, + { + "name": "Hof", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.314, + "lng": 11.913 + }, + { + "name": "Hohenfels", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.203, + "lng": 11.849 + }, + { + "name": "Illesheim", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.475, + "lng": 10.385 + }, + { + "name": "Kornwestheim", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 48.863, + "lng": 9.181 + }, + { + "name": "Lampertheim", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.6, + "lng": 8.467 + }, + { + "name": "Landshut", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 48.539, + "lng": 12.146 + }, + { + "name": "Langen", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.991, + "lng": 8.663 + }, + { + "name": "Mainz", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.993, + "lng": 8.247 + }, + { + "name": "Mannheim", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.487, + "lng": 8.466 + }, + { + "name": "Miesau", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.405, + "lng": 7.437 + }, + { + "name": "Pirmasens", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.202, + "lng": 7.6 + }, + { + "name": "Schwetzingen", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.385, + "lng": 8.572 + }, + { + "name": "Sembach", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.511, + "lng": 7.865 + }, + { + "name": "Lamu/Manda Bay", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -2.28, + "lng": 40.891 + }, + { + "name": "McMurdo Station", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -77.85, + "lng": 166.667 + }, + { + "name": "Aruba", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 12.521, + "lng": -69.968 + }, + { + "name": "Kojarena, near Geraldton MUOS, ECHELON Intel and Communications Site", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -28.695, + "lng": 114.842 + }, + { + "name": "Zutendaal", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 50.933, + "lng": 5.567 + }, + { + "name": "Gabrone", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -24.55, + "lng": 25.927 + }, + { + "name": "Bezmer Air Base", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 42.455, + "lng": 26.352 + }, + { + "name": "Ouagadougou International Airport", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": 12.353, + "lng": -1.512 + }, + { + "name": "Bujumbura (?)", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -3.383, + "lng": 29.367 + }, + { + "name": "Bangui", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 4.367, + "lng": 18.583 + }, + { + "name": "Djema", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 6.05, + "lng": 25.317 + }, + { + "name": "Palanquero AB/Cpt German Olano Moreno AB", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 5.484, + "lng": -74.657 + }, + { + "name": "Tolemaida AB", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 4.245, + "lng": -74.65 + }, + { + "name": "Larandia", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 1.479, + "lng": -75.487 + }, + { + "name": "Bahia de Malaga", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 4.1, + "lng": -77.35 + }, + { + "name": "Karap\tDenmark", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 56.298, + "lng": 9.099 + }, + { + "name": "Cairo\tEgypt", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 30.044, + "lng": 31.236 + }, + { + "name": "Egypt", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 26.821, + "lng": 30.802 + }, + { + "name": "Dire Dawa", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 9.6, + "lng": 41.867 + }, + { + "name": "Krtsanisi Training Centre, nr. Tblisi", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 41.717, + "lng": 44.783 + }, + { + "name": "Greece", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 39.074, + "lng": 21.824 + }, + { + "name": "Mocoron Tropics Region Testing Center", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.039, + "lng": -84.275 + }, + { + "name": "Papa AB, Papa", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 47.363, + "lng": 17.501 + }, + { + "name": "Shannon Airport", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": 52.702, + "lng": -8.925 + }, + { + "name": "Dimona Radar Base", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 31.067, + "lng": 35.033 + }, + { + "name": "Site 51, IS [loc. Is random in Negev]", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 30.772, + "lng": 34.978 + }, + { + "name": "outside Amman", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 31.98, + "lng": 36.005 + }, + { + "name": "Garissa", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -0.024, + "lng": 37.906 + }, + { + "name": "Monrovia and surrounding", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 6.292, + "lng": -10.756 + }, + { + "name": "Nuakchott (Air Force)", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 18.101, + "lng": -15.952 + }, + { + "name": "Coevorden, NL", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 52.667, + "lng": 6.75 + }, + { + "name": "Rotterdam\tNetherlands", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 51.924, + "lng": 4.482 + }, + { + "name": "Complex Vriezenveen, Almelo", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 52.412, + "lng": 6.626 + }, + { + "name": "Curacao\tNetherlands Antilles", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 12.122, + "lng": -68.882 + }, + { + "name": "Rota", + "country": "United States", + "operator": "USN", + "branch": "navy", + "lat": 14.154, + "lng": 145.203 + }, + { + "name": "Chaklala", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 30.309, + "lng": 72.5 + }, + { + "name": "Tarbela", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 34.089, + "lng": 72.699 + }, + { + "name": "Peshawar", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 34.015, + "lng": 71.571 + }, + { + "name": "Dalbandin", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 28.887, + "lng": 64.399 + }, + { + "name": "Pasni", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 25.295, + "lng": 63.347 + }, + { + "name": "Islamabad", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 33.703, + "lng": 73.084 + }, + { + "name": "IMF Santa Rosa, Iquitos", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -3.744, + "lng": -73.261 + }, + { + "name": "Camp Navarro, Zamboanga City , Mindanao (JSOTF-P)", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 6.952, + "lng": 122.126 + }, + { + "name": "SF FOB Jolo, Sulu (CSL)", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 6.056, + "lng": 121.013 + }, + { + "name": "Camp inside Camp Aguinaldo, Manila", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 14.583, + "lng": 120.967 + }, + { + "name": "Camp Malagutay, Basilan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 6.626, + "lng": 121.942 + }, + { + "name": "Cincu Training Base", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 45.917, + "lng": 24.8 + }, + { + "name": "Smardan Training base", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 45.483, + "lng": 27.933 + }, + { + "name": "Eskan Village, Saudi Arabia", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 24.567, + "lng": 46.85 + }, + { + "name": "Drone base in east, near Yemen", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 18.921, + "lng": 49.838 + }, + { + "name": "Dakar", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 14.728, + "lng": -17.457 + }, + { + "name": "Baledogle Airfield", + "country": "United States", + "operator": "US Military", + "branch": "air_force", + "lat": 2.671, + "lng": 44.793 + }, + { + "name": "Kismayo", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -0.355, + "lng": 42.546 + }, + { + "name": "Nzara", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 4.633, + "lng": 28.255 + }, + { + "name": "Spain", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 40.464, + "lng": -3.749 + }, + { + "name": "U-Tapao", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 12.682, + "lng": 100.998 + }, + { + "name": "naval base [loc?]", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": 12.244, + "lng": 102.393 + }, + { + "name": "CK: Mount Chaambi, Kasserine", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 35.218, + "lng": 8.749 + }, + { + "name": "Incirlik?", + "country": "United States", + "operator": "USAF", + "branch": "air_force", + "lat": 37.002, + "lng": 35.425 + }, + { + "name": "Entebbe", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 0.047, + "lng": 32.446 + }, + { + "name": "Singo Training School", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -0.8, + "lng": 31.498 + }, + { + "name": "Thetford\tUnited Kingdom", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 52.413, + "lng": 0.752 + }, + { + "name": "St. Thomas?", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 18.2, + "lng": -64.55 + }, + { + "name": "Wake Island", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 19.18, + "lng": 166.38 + }, + { + "name": "Dungu", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 3.626, + "lng": 28.561 + }, + { + "name": "Vilseck Germany", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.61, + "lng": 11.806 + }, + { + "name": "Wackernheim Germany", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.974, + "lng": 8.118 + }, + { + "name": "Weisskirchen Germany", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 49.556, + "lng": 6.819 + }, + { + "name": "Chaco", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -27.451, + "lng": -58.987 + }, + { + "name": "Hunting Caye", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.146, + "lng": -88.281 + }, + { + "name": "Hattieville", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 17.45, + "lng": -88.383 + }, + { + "name": "Big Creek FOS", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.514, + "lng": -88.404 + }, + { + "name": "San Pedro Caye", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 17.921, + "lng": -87.961 + }, + { + "name": "Belize City", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 17.505, + "lng": -88.187 + }, + { + "name": "Ambergris Caye", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 18.014, + "lng": -87.931 + }, + { + "name": "Naval Support Detachment Sao Paulo", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": -23.55, + "lng": -46.633 + }, + { + "name": "Fuerte Aguayo NB, Conc\u00d1n", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -32.947, + "lng": -71.452 + }, + { + "name": "Tres Esquinas AB", + "country": "United States", + "operator": "US Military", + "branch": "navy", + "lat": 0.746, + "lng": -75.234 + }, + { + "name": "Caldera", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 9.912, + "lng": -84.714 + }, + { + "name": "Liberia", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 10.633, + "lng": -85.433 + }, + { + "name": "Colorado", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 10.692, + "lng": -83.72 + }, + { + "name": "Flamingo", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 10.434, + "lng": -85.792 + }, + { + "name": "Saona Island", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 18.156, + "lng": -68.699 + }, + { + "name": "Lita, Carchi", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 0.942, + "lng": -78.553 + }, + { + "name": "Miraflores", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.423, + "lng": -88.085 + }, + { + "name": "CEOPAZ Base, Lourdes, Colon", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.717, + "lng": -89.367 + }, + { + "name": "La Union", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.337, + "lng": -87.844 + }, + { + "name": "Cuscatlan", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.673, + "lng": -89.241 + }, + { + "name": "Arba Minch", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 6.04, + "lng": 37.59 + }, + { + "name": "Puerto San Jose", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.92, + "lng": -90.811 + }, + { + "name": "Poptun", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.328, + "lng": -89.408 + }, + { + "name": "Champerico", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 14.294, + "lng": -91.913 + }, + { + "name": "Coban", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.471, + "lng": -90.404 + }, + { + "name": "Santa Ana de Berlin", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 14.7, + "lng": -91.883 + }, + { + "name": "Las Mantanitas", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.856, + "lng": -90.387 + }, + { + "name": "Tecun Uman", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 14.673, + "lng": -92.132 + }, + { + "name": "Puerto Barrios", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.719, + "lng": -88.601 + }, + { + "name": "Guanaja", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.444, + "lng": -85.902 + }, + { + "name": "Puerto Castilla", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 16.011, + "lng": -85.951 + }, + { + "name": "La Venta", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.795, + "lng": -87.29 + }, + { + "name": "El Aguacate", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 13.95, + "lng": -87.132 + }, + { + "name": "Puerto Lempira", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.268, + "lng": -83.771 + }, + { + "name": "La Brea", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 15.793, + "lng": -85.965 + }, + { + "name": "El Bluff", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 12.0, + "lng": -83.683 + }, + { + "name": "Corn Island", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 12.167, + "lng": -83.033 + }, + { + "name": "Corinto", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 12.483, + "lng": -87.183 + }, + { + "name": "Summit", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 9.046, + "lng": -79.513 + }, + { + "name": "Puerto Obaldia", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 8.688, + "lng": -77.513 + }, + { + "name": "Puerto Pina", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 7.583, + "lng": -78.18 + }, + { + "name": "Isla Grande", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 9.593, + "lng": -79.555 + }, + { + "name": "Punta Coco", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 9.3, + "lng": -82.267 + }, + { + "name": "La Palma", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": 8.407, + "lng": -78.142 + }, + { + "name": "Tarapoto", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -6.497, + "lng": -76.372 + }, + { + "name": "Puno", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -15.848, + "lng": -70.019 + }, + { + "name": "14th Battalion, Toledo", + "country": "United States", + "operator": "US Military", + "branch": "army", + "lat": -34.741, + "lng": -56.096 } ] diff --git a/backend/data/sat_gp_cache.json b/backend/data/sat_gp_cache.json index 6fea8cac..521ee6d7 100644 --- a/backend/data/sat_gp_cache.json +++ b/backend/data/sat_gp_cache.json @@ -1 +1 @@ -[{"OBJECT_NAME": "CAPELLA-9-WHITNEY", "NORAD_CAT_ID": 55910, "MEAN_MOTION": 14.99271847, "ECCENTRICITY": 0.0009067, "INCLINATION": 43.9991, "RA_OF_ASC_NODE": 244.4088, "ARG_OF_PERICENTER": 156.0492, "MEAN_ANOMALY": 204.0788, "BSTAR": 0.0013163, "EPOCH": "2023-12-27T21:00:03"}, {"OBJECT_NAME": "CAPELLA-6-WHITNEY", "NORAD_CAT_ID": 48605, "MEAN_MOTION": 15.36992166, "ECCENTRICITY": 0.0006978, "INCLINATION": 53.0293, "RA_OF_ASC_NODE": 151.2332, "ARG_OF_PERICENTER": 275.4158, "MEAN_ANOMALY": 84.6048, "BSTAR": 0.0028544000000000004, "EPOCH": "2023-12-28T11:48:20"}, {"OBJECT_NAME": "CAPELLA-8-WHITNEY", "NORAD_CAT_ID": 51071, "MEAN_MOTION": 16.38892911, "ECCENTRICITY": 0.0015575, "INCLINATION": 97.4006, "RA_OF_ASC_NODE": 322.5782, "ARG_OF_PERICENTER": 273.5387, "MEAN_ANOMALY": 175.0653, "BSTAR": 0.0030023, "EPOCH": "2023-09-06T01:34:15"}, {"OBJECT_NAME": "CAPELLA-7-WHITNEY", "NORAD_CAT_ID": 51072, "MEAN_MOTION": 16.44927659, "ECCENTRICITY": 0.0015288, "INCLINATION": 97.3947, "RA_OF_ASC_NODE": 311.3663, "ARG_OF_PERICENTER": 273.4288, "MEAN_ANOMALY": 179.2482, "BSTAR": 0.0011137, "EPOCH": "2023-08-26T19:44:02"}, {"OBJECT_NAME": "CAPELLA-10-WHITNEY", "NORAD_CAT_ID": 55909, "MEAN_MOTION": 14.95890017, "ECCENTRICITY": 0.0009144, "INCLINATION": 43.9994, "RA_OF_ASC_NODE": 246.9297, "ARG_OF_PERICENTER": 166.1646, "MEAN_ANOMALY": 193.9462, "BSTAR": 0.00065599, "EPOCH": "2023-12-27T21:10:20"}, {"OBJECT_NAME": "CAPELLA-11 (ACADIA-1)", "NORAD_CAT_ID": 57693, "MEAN_MOTION": 14.80571571, "ECCENTRICITY": 0.0002549, "INCLINATION": 53.0051, "RA_OF_ASC_NODE": 241.5, "ARG_OF_PERICENTER": 127.5905, "MEAN_ANOMALY": 232.5304, "BSTAR": 0.00045154, "EPOCH": "2026-03-12T20:38:53"}, {"OBJECT_NAME": "CAPELLA-14 (ACADIA-4)", "NORAD_CAT_ID": 59444, "MEAN_MOTION": 15.10713992, "ECCENTRICITY": 0.0004182, "INCLINATION": 45.6047, "RA_OF_ASC_NODE": 40.8166, "ARG_OF_PERICENTER": 138.8002, "MEAN_ANOMALY": 221.32, "BSTAR": 0.0012364000000000001, "EPOCH": "2026-03-13T08:06:00"}, {"OBJECT_NAME": "CAPELLA-13 (ACADIA-3)", "NORAD_CAT_ID": 60419, "MEAN_MOTION": 14.87642553, "ECCENTRICITY": 0.000131, "INCLINATION": 53.0064, "RA_OF_ASC_NODE": 120.2379, "ARG_OF_PERICENTER": 99.2032, "MEAN_ANOMALY": 260.9096, "BSTAR": -0.0007528599999999999, "EPOCH": "2026-03-12T08:31:00"}, {"OBJECT_NAME": "CAPELLA-15 (ACADIA-5)", "NORAD_CAT_ID": 60544, "MEAN_MOTION": 14.92437598, "ECCENTRICITY": 0.0004118, "INCLINATION": 97.6855, "RA_OF_ASC_NODE": 146.8612, "ARG_OF_PERICENTER": 61.8229, "MEAN_ANOMALY": 298.3406, "BSTAR": -0.00029862, "EPOCH": "2026-03-12T13:48:01"}, {"OBJECT_NAME": "CAPELLA-17 (ACADIA-7)", "NORAD_CAT_ID": 64583, "MEAN_MOTION": 14.90912744, "ECCENTRICITY": 0.0004358, "INCLINATION": 97.7566, "RA_OF_ASC_NODE": 187.4545, "ARG_OF_PERICENTER": 319.2015, "MEAN_ANOMALY": 40.8876, "BSTAR": 0.00098, "EPOCH": "2026-03-13T07:55:00"}, {"OBJECT_NAME": "CAPELLA-16 (ACADIA-6)", "NORAD_CAT_ID": 65318, "MEAN_MOTION": 14.93545595, "ECCENTRICITY": 0.0004913, "INCLINATION": 97.74, "RA_OF_ASC_NODE": 149.4795, "ARG_OF_PERICENTER": 185.0813, "MEAN_ANOMALY": 175.0358, "BSTAR": 0.00082972, "EPOCH": "2026-03-13T22:09:56"}, {"OBJECT_NAME": "CAPELLA-19 (ACADIA-9)", "NORAD_CAT_ID": 67384, "MEAN_MOTION": 14.86660787, "ECCENTRICITY": 0.0001275, "INCLINATION": 97.8129, "RA_OF_ASC_NODE": 71.6661, "ARG_OF_PERICENTER": 133.6197, "MEAN_ANOMALY": 226.5125, "BSTAR": 0.00077301, "EPOCH": "2026-03-12T13:06:13"}, {"OBJECT_NAME": "CAPELLA-18 (ACADIA-8)", "NORAD_CAT_ID": 67385, "MEAN_MOTION": 14.88727587, "ECCENTRICITY": 0.0005576, "INCLINATION": 97.801, "RA_OF_ASC_NODE": 73.1268, "ARG_OF_PERICENTER": 78.4894, "MEAN_ANOMALY": 281.6949, "BSTAR": 0.00074085, "EPOCH": "2026-03-13T22:09:17"}, {"OBJECT_NAME": "PLANETUM1", "NORAD_CAT_ID": 52738, "MEAN_MOTION": 16.30442171, "ECCENTRICITY": 0.0009881, "INCLINATION": 97.5537, "RA_OF_ASC_NODE": 110.7236, "ARG_OF_PERICENTER": 292.3042, "MEAN_ANOMALY": 67.7204, "BSTAR": 0.0013242000000000002, "EPOCH": "2024-11-28T20:09:41"}, {"OBJECT_NAME": "GAOFEN-3", "NORAD_CAT_ID": 41727, "MEAN_MOTION": 14.4221702, "ECCENTRICITY": 0.0001688, "INCLINATION": 98.4058, "RA_OF_ASC_NODE": 82.177, "ARG_OF_PERICENTER": 87.1861, "MEAN_ANOMALY": 272.9522, "BSTAR": -3.0976e-06, "EPOCH": "2026-03-13T23:12:22"}, {"OBJECT_NAME": "GAOFEN-3 03", "NORAD_CAT_ID": 52200, "MEAN_MOTION": 14.42207664, "ECCENTRICITY": 0.0001585, "INCLINATION": 98.4113, "RA_OF_ASC_NODE": 82.9211, "ARG_OF_PERICENTER": 91.9814, "MEAN_ANOMALY": 268.1558, "BSTAR": -8.624900000000001e-06, "EPOCH": "2026-03-13T23:22:03"}, {"OBJECT_NAME": "GAOFEN-6", "NORAD_CAT_ID": 43484, "MEAN_MOTION": 14.76603208, "ECCENTRICITY": 0.0013405, "INCLINATION": 97.7865, "RA_OF_ASC_NODE": 139.0605, "ARG_OF_PERICENTER": 122.1169, "MEAN_ANOMALY": 238.1344, "BSTAR": 7.9892e-05, "EPOCH": "2026-03-13T22:18:48"}, {"OBJECT_NAME": "GAOFEN-3 02", "NORAD_CAT_ID": 49495, "MEAN_MOTION": 14.42216649, "ECCENTRICITY": 0.0001467, "INCLINATION": 98.4137, "RA_OF_ASC_NODE": 82.4363, "ARG_OF_PERICENTER": 286.7274, "MEAN_ANOMALY": 73.3755, "BSTAR": -1.2019000000000001e-05, "EPOCH": "2026-03-14T00:01:22"}, {"OBJECT_NAME": "GAOFEN-2", "NORAD_CAT_ID": 40118, "MEAN_MOTION": 14.8077675, "ECCENTRICITY": 0.0007372, "INCLINATION": 98.0215, "RA_OF_ASC_NODE": 137.2317, "ARG_OF_PERICENTER": 171.0983, "MEAN_ANOMALY": 189.0361, "BSTAR": 1.0704e-05, "EPOCH": "2026-03-13T23:16:48"}, {"OBJECT_NAME": "GAOFEN-8", "NORAD_CAT_ID": 40701, "MEAN_MOTION": 15.42612981, "ECCENTRICITY": 0.0009623, "INCLINATION": 97.6943, "RA_OF_ASC_NODE": 260.8431, "ARG_OF_PERICENTER": 172.7159, "MEAN_ANOMALY": 187.4229, "BSTAR": 0.0005817, "EPOCH": "2026-03-13T23:08:38"}, {"OBJECT_NAME": "GAOFEN-7", "NORAD_CAT_ID": 44703, "MEAN_MOTION": 15.21367581, "ECCENTRICITY": 0.00154, "INCLINATION": 97.254, "RA_OF_ASC_NODE": 137.9616, "ARG_OF_PERICENTER": 15.6623, "MEAN_ANOMALY": 344.5087, "BSTAR": 0.00025296000000000004, "EPOCH": "2026-03-13T23:07:59"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 2B", "NORAD_CAT_ID": 44836, "MEAN_MOTION": 15.19512211, "ECCENTRICITY": 0.0012969, "INCLINATION": 97.496, "RA_OF_ASC_NODE": 145.6035, "ARG_OF_PERICENTER": 293.507, "MEAN_ANOMALY": 66.4801, "BSTAR": 0.00035962, "EPOCH": "2026-03-13T23:07:37"}, {"OBJECT_NAME": "GAOFEN-1 02", "NORAD_CAT_ID": 43259, "MEAN_MOTION": 14.76449151, "ECCENTRICITY": 0.0002195, "INCLINATION": 98.0637, "RA_OF_ASC_NODE": 135.4203, "ARG_OF_PERICENTER": 336.3176, "MEAN_ANOMALY": 23.7932, "BSTAR": 9.090000000000001e-05, "EPOCH": "2026-03-13T16:10:22"}, {"OBJECT_NAME": "GAOFEN-1 04", "NORAD_CAT_ID": 43262, "MEAN_MOTION": 14.76444841, "ECCENTRICITY": 0.0001855, "INCLINATION": 98.0639, "RA_OF_ASC_NODE": 135.7601, "ARG_OF_PERICENTER": 75.9314, "MEAN_ANOMALY": 284.2102, "BSTAR": 7.517899999999999e-05, "EPOCH": "2026-03-13T23:11:53"}, {"OBJECT_NAME": "GAOFEN-10R", "NORAD_CAT_ID": 44622, "MEAN_MOTION": 14.83170684, "ECCENTRICITY": 0.0008263, "INCLINATION": 98.0402, "RA_OF_ASC_NODE": 25.9011, "ARG_OF_PERICENTER": 94.7549, "MEAN_ANOMALY": 265.4608, "BSTAR": 0.00022164, "EPOCH": "2026-03-13T22:46:16"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 2A", "NORAD_CAT_ID": 44777, "MEAN_MOTION": 15.36565084, "ECCENTRICITY": 0.0007548, "INCLINATION": 97.5004, "RA_OF_ASC_NODE": 154.6993, "ARG_OF_PERICENTER": 72.4081, "MEAN_ANOMALY": 287.7987, "BSTAR": 0.00049885, "EPOCH": "2026-03-13T22:46:36"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D18", "NORAD_CAT_ID": 51844, "MEAN_MOTION": 15.77584928, "ECCENTRICITY": 0.0003896, "INCLINATION": 97.3668, "RA_OF_ASC_NODE": 163.7334, "ARG_OF_PERICENTER": 138.3279, "MEAN_ANOMALY": 221.8285, "BSTAR": 0.0010383, "EPOCH": "2026-03-12T06:23:35"}, {"OBJECT_NAME": "GAOFEN DUOMO (GFDM)", "NORAD_CAT_ID": 45856, "MEAN_MOTION": 14.76849797, "ECCENTRICITY": 0.0011845, "INCLINATION": 97.7861, "RA_OF_ASC_NODE": 136.2514, "ARG_OF_PERICENTER": 26.2921, "MEAN_ANOMALY": 333.8888, "BSTAR": 0.00016078999999999998, "EPOCH": "2026-03-13T22:43:13"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 4A", "NORAD_CAT_ID": 52388, "MEAN_MOTION": 16.28950252, "ECCENTRICITY": 0.000805, "INCLINATION": 97.4754, "RA_OF_ASC_NODE": 100.0865, "ARG_OF_PERICENTER": 290.668, "MEAN_ANOMALY": 69.375, "BSTAR": 0.0010643, "EPOCH": "2025-12-16T07:00:53"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D03", "NORAD_CAT_ID": 49006, "MEAN_MOTION": 15.81878461, "ECCENTRICITY": 0.0001474, "INCLINATION": 97.3008, "RA_OF_ASC_NODE": 150.2487, "ARG_OF_PERICENTER": 150.7775, "MEAN_ANOMALY": 209.3576, "BSTAR": 0.0011672, "EPOCH": "2026-03-13T23:41:57"}, {"OBJECT_NAME": "GAOFEN-1 03", "NORAD_CAT_ID": 43260, "MEAN_MOTION": 14.76428287, "ECCENTRICITY": 0.0003234, "INCLINATION": 98.0626, "RA_OF_ASC_NODE": 135.6052, "ARG_OF_PERICENTER": 39.7747, "MEAN_ANOMALY": 320.3699, "BSTAR": 0.00015166, "EPOCH": "2026-03-13T23:46:47"}, {"OBJECT_NAME": "GAOFEN-5 02", "NORAD_CAT_ID": 49122, "MEAN_MOTION": 14.57725614, "ECCENTRICITY": 1.09e-05, "INCLINATION": 98.2672, "RA_OF_ASC_NODE": 147.9575, "ARG_OF_PERICENTER": 265.6614, "MEAN_ANOMALY": 94.4572, "BSTAR": 4.4049000000000005e-05, "EPOCH": "2026-03-13T23:40:13"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3B", "NORAD_CAT_ID": 46454, "MEAN_MOTION": 15.81348232, "ECCENTRICITY": 0.0003185, "INCLINATION": 97.2358, "RA_OF_ASC_NODE": 130.4491, "ARG_OF_PERICENTER": 144.1208, "MEAN_ANOMALY": 216.0275, "BSTAR": 0.0010368, "EPOCH": "2026-03-13T16:43:21"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D54", "NORAD_CAT_ID": 54254, "MEAN_MOTION": 15.69110286, "ECCENTRICITY": 0.0003038, "INCLINATION": 97.6018, "RA_OF_ASC_NODE": 216.0779, "ARG_OF_PERICENTER": 304.0613, "MEAN_ANOMALY": 56.036, "BSTAR": 0.0009132400000000001, "EPOCH": "2026-03-13T22:32:23"}, {"OBJECT_NAME": "ISS (ZARYA)", "NORAD_CAT_ID": 25544, "MEAN_MOTION": 15.48633802, "ECCENTRICITY": 0.0007884, "INCLINATION": 51.6324, "RA_OF_ASC_NODE": 52.4887, "ARG_OF_PERICENTER": 189.2977, "MEAN_ANOMALY": 170.7866, "BSTAR": 0.00022432, "EPOCH": "2026-03-13T16:59:24"}, {"OBJECT_NAME": "ISS (NAUKA)", "NORAD_CAT_ID": 49044, "MEAN_MOTION": 15.48633802, "ECCENTRICITY": 0.0007884, "INCLINATION": 51.6324, "RA_OF_ASC_NODE": 52.4887, "ARG_OF_PERICENTER": 189.2977, "MEAN_ANOMALY": 170.7866, "BSTAR": 0.00022432, "EPOCH": "2026-03-13T16:59:24"}, {"OBJECT_NAME": "SWISSCUBE", "NORAD_CAT_ID": 35932, "MEAN_MOTION": 14.62272355, "ECCENTRICITY": 0.000737, "INCLINATION": 98.4096, "RA_OF_ASC_NODE": 339.2128, "ARG_OF_PERICENTER": 140.2287, "MEAN_ANOMALY": 219.9456, "BSTAR": 0.00027243, "EPOCH": "2026-03-13T23:50:57"}, {"OBJECT_NAME": "AISSAT 1", "NORAD_CAT_ID": 36797, "MEAN_MOTION": 14.96917404, "ECCENTRICITY": 0.0009347, "INCLINATION": 98.1018, "RA_OF_ASC_NODE": 328.8398, "ARG_OF_PERICENTER": 20.3465, "MEAN_ANOMALY": 339.8126, "BSTAR": 0.00026985999999999997, "EPOCH": "2026-03-13T23:18:08"}, {"OBJECT_NAME": "AISSAT 2", "NORAD_CAT_ID": 40075, "MEAN_MOTION": 14.8560182, "ECCENTRICITY": 0.000478, "INCLINATION": 98.3401, "RA_OF_ASC_NODE": 268.4723, "ARG_OF_PERICENTER": 335.0232, "MEAN_ANOMALY": 25.0749, "BSTAR": 0.00040707, "EPOCH": "2023-12-28T11:59:02"}, {"OBJECT_NAME": "ISS OBJECT XK", "NORAD_CAT_ID": 65731, "MEAN_MOTION": 16.39076546, "ECCENTRICITY": 0.0002464, "INCLINATION": 51.6052, "RA_OF_ASC_NODE": 44.2508, "ARG_OF_PERICENTER": 211.8886, "MEAN_ANOMALY": 148.1991, "BSTAR": 0.00075547, "EPOCH": "2026-03-09T22:15:28"}, {"OBJECT_NAME": "ISS OBJECT XU", "NORAD_CAT_ID": 66908, "MEAN_MOTION": 15.70686547, "ECCENTRICITY": 0.0004897, "INCLINATION": 51.6245, "RA_OF_ASC_NODE": 46.3387, "ARG_OF_PERICENTER": 107.7206, "MEAN_ANOMALY": 252.4326, "BSTAR": 0.0013009999999999999, "EPOCH": "2026-03-13T11:07:47"}, {"OBJECT_NAME": "OUTPOST MISSION 2", "NORAD_CAT_ID": 58334, "MEAN_MOTION": 15.51046817, "ECCENTRICITY": 0.0006981, "INCLINATION": 97.3968, "RA_OF_ASC_NODE": 162.7831, "ARG_OF_PERICENTER": 111.1455, "MEAN_ANOMALY": 249.0543, "BSTAR": 0.00065889, "EPOCH": "2026-03-13T22:52:24"}, {"OBJECT_NAME": "ISS (UNITY)", "NORAD_CAT_ID": 25575, "MEAN_MOTION": 15.48633802, "ECCENTRICITY": 0.0007884, "INCLINATION": 51.6324, "RA_OF_ASC_NODE": 52.4887, "ARG_OF_PERICENTER": 189.2977, "MEAN_ANOMALY": 170.7866, "BSTAR": 0.00022432, "EPOCH": "2026-03-13T16:59:24"}, {"OBJECT_NAME": "ISS OBJECT YE", "NORAD_CAT_ID": 67688, "MEAN_MOTION": 15.52874098, "ECCENTRICITY": 0.0008075, "INCLINATION": 51.6312, "RA_OF_ASC_NODE": 53.6814, "ARG_OF_PERICENTER": 164.1724, "MEAN_ANOMALY": 195.952, "BSTAR": 0.0008688000000000001, "EPOCH": "2026-03-13T07:47:10"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 56434, "MEAN_MOTION": 16.34589017, "ECCENTRICITY": 0.0007268, "INCLINATION": 51.614, "RA_OF_ASC_NODE": 110.8216, "ARG_OF_PERICENTER": 312.4976, "MEAN_ANOMALY": 47.5434, "BSTAR": 0.0007953900000000001, "EPOCH": "2023-12-17T13:53:44"}, {"OBJECT_NAME": "ISS DEB [SPX-28 IPA FSE]", "NORAD_CAT_ID": 57212, "MEAN_MOTION": 16.42826151, "ECCENTRICITY": 0.0005882, "INCLINATION": 51.6078, "RA_OF_ASC_NODE": 38.878, "ARG_OF_PERICENTER": 256.2665, "MEAN_ANOMALY": 278.8784, "BSTAR": 0.00035698999999999995, "EPOCH": "2024-05-22T10:35:37"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 58174, "MEAN_MOTION": 16.3176295, "ECCENTRICITY": 0.0009401, "INCLINATION": 51.6127, "RA_OF_ASC_NODE": 333.6363, "ARG_OF_PERICENTER": 246.0894, "MEAN_ANOMALY": 113.9145, "BSTAR": 0.001226, "EPOCH": "2024-03-28T18:25:22"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 58203, "MEAN_MOTION": 15.98175731, "ECCENTRICITY": 0.0014888, "INCLINATION": 51.6276, "RA_OF_ASC_NODE": 303.0132, "ARG_OF_PERICENTER": 126.3295, "MEAN_ANOMALY": 233.9091, "BSTAR": 0.0096833, "EPOCH": "2023-11-14T05:06:41"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 58229, "MEAN_MOTION": 16.43977649, "ECCENTRICITY": 0.0006009, "INCLINATION": 51.6071, "RA_OF_ASC_NODE": 235.9974, "ARG_OF_PERICENTER": 315.3037, "MEAN_ANOMALY": 44.7506, "BSTAR": 0.00037573, "EPOCH": "2024-06-26T14:41:28"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 62376, "MEAN_MOTION": 16.23766836, "ECCENTRICITY": 0.0010647, "INCLINATION": 51.6124, "RA_OF_ASC_NODE": 290.5973, "ARG_OF_PERICENTER": 277.8505, "MEAN_ANOMALY": 82.1305, "BSTAR": 0.0017468, "EPOCH": "2025-04-05T18:57:57"}, {"OBJECT_NAME": "ISS (ZVEZDA)", "NORAD_CAT_ID": 26400, "MEAN_MOTION": 15.48633802, "ECCENTRICITY": 0.0007884, "INCLINATION": 51.6324, "RA_OF_ASC_NODE": 52.4887, "ARG_OF_PERICENTER": 189.2977, "MEAN_ANOMALY": 170.7866, "BSTAR": 0.00022432, "EPOCH": "2026-03-13T16:59:24"}, {"OBJECT_NAME": "ISS (DESTINY)", "NORAD_CAT_ID": 26700, "MEAN_MOTION": 15.48633802, "ECCENTRICITY": 0.0007884, "INCLINATION": 51.6324, "RA_OF_ASC_NODE": 52.4887, "ARG_OF_PERICENTER": 189.2977, "MEAN_ANOMALY": 170.7866, "BSTAR": 0.00022432, "EPOCH": "2026-03-13T16:59:24"}, {"OBJECT_NAME": "ISS OBJECT XT", "NORAD_CAT_ID": 66907, "MEAN_MOTION": 15.70505609, "ECCENTRICITY": 0.0004699, "INCLINATION": 51.6247, "RA_OF_ASC_NODE": 47.0068, "ARG_OF_PERICENTER": 104.7322, "MEAN_ANOMALY": 255.4197, "BSTAR": 0.0013594999999999998, "EPOCH": "2026-03-13T08:07:34"}, {"OBJECT_NAME": "ISS OBJECT XW", "NORAD_CAT_ID": 66910, "MEAN_MOTION": 15.69697037, "ECCENTRICITY": 0.0005316, "INCLINATION": 51.6244, "RA_OF_ASC_NODE": 47.6269, "ARG_OF_PERICENTER": 107.7849, "MEAN_ANOMALY": 252.3729, "BSTAR": 0.0012699, "EPOCH": "2026-03-13T04:59:13"}, {"OBJECT_NAME": "ICEYE-X4", "NORAD_CAT_ID": 44390, "MEAN_MOTION": 15.27095253, "ECCENTRICITY": 0.001218, "INCLINATION": 97.8885, "RA_OF_ASC_NODE": 100.4161, "ARG_OF_PERICENTER": 277.9702, "MEAN_ANOMALY": 82.0154, "BSTAR": 0.0005035300000000001, "EPOCH": "2026-03-13T23:49:55"}, {"OBJECT_NAME": "ICEYE-X9", "NORAD_CAT_ID": 47506, "MEAN_MOTION": 15.28271202, "ECCENTRICITY": 0.0008413, "INCLINATION": 97.3622, "RA_OF_ASC_NODE": 54.3082, "ARG_OF_PERICENTER": 126.886, "MEAN_ANOMALY": 233.3152, "BSTAR": 0.0010076, "EPOCH": "2023-12-28T11:08:25"}, {"OBJECT_NAME": "XR-1 (ICEYE-X10)", "NORAD_CAT_ID": 47507, "MEAN_MOTION": 15.22197725, "ECCENTRICITY": 0.001392, "INCLINATION": 97.3689, "RA_OF_ASC_NODE": 55.932, "ARG_OF_PERICENTER": 134.347, "MEAN_ANOMALY": 225.891, "BSTAR": 0.00062135, "EPOCH": "2023-12-28T04:01:27"}, {"OBJECT_NAME": "ICEYE-X8", "NORAD_CAT_ID": 47510, "MEAN_MOTION": 15.26805421, "ECCENTRICITY": 0.0007492, "INCLINATION": 97.3641, "RA_OF_ASC_NODE": 54.4014, "ARG_OF_PERICENTER": 124.0674, "MEAN_ANOMALY": 236.1278, "BSTAR": 0.00062244, "EPOCH": "2023-12-28T10:34:23"}, {"OBJECT_NAME": "ICEYE-X17", "NORAD_CAT_ID": 52762, "MEAN_MOTION": 15.2124607, "ECCENTRICITY": 0.0013644, "INCLINATION": 97.5558, "RA_OF_ASC_NODE": 120.8021, "ARG_OF_PERICENTER": 1.6767, "MEAN_ANOMALY": 358.4512, "BSTAR": 0.00057881, "EPOCH": "2023-12-28T09:08:20"}, {"OBJECT_NAME": "ICEYE-X55", "NORAD_CAT_ID": 64581, "MEAN_MOTION": 14.94549859, "ECCENTRICITY": 0.0002024, "INCLINATION": 97.7609, "RA_OF_ASC_NODE": 188.6379, "ARG_OF_PERICENTER": 316.5225, "MEAN_ANOMALY": 43.5835, "BSTAR": 0.00046989, "EPOCH": "2026-03-13T21:59:46"}, {"OBJECT_NAME": "ICEYE-X33", "NORAD_CAT_ID": 60548, "MEAN_MOTION": 15.01177965, "ECCENTRICITY": 0.0006008, "INCLINATION": 97.7037, "RA_OF_ASC_NODE": 152.4523, "ARG_OF_PERICENTER": 111.5308, "MEAN_ANOMALY": 248.6557, "BSTAR": 0.00044679, "EPOCH": "2026-03-13T15:27:23"}, {"OBJECT_NAME": "ICEYE-X21", "NORAD_CAT_ID": 55049, "MEAN_MOTION": 15.77464515, "ECCENTRICITY": 0.0003402, "INCLINATION": 97.3379, "RA_OF_ASC_NODE": 152.0526, "ARG_OF_PERICENTER": 60.6638, "MEAN_ANOMALY": 299.4968, "BSTAR": 0.00056355, "EPOCH": "2026-03-13T21:55:17"}, {"OBJECT_NAME": "ICEYE-X7", "NORAD_CAT_ID": 46496, "MEAN_MOTION": 15.2927708, "ECCENTRICITY": 0.0006409, "INCLINATION": 97.7409, "RA_OF_ASC_NODE": 35.3598, "ARG_OF_PERICENTER": 315.7508, "MEAN_ANOMALY": 44.3218, "BSTAR": 0.00054143, "EPOCH": "2026-03-13T22:40:15"}, {"OBJECT_NAME": "ICEYE-X53", "NORAD_CAT_ID": 64584, "MEAN_MOTION": 15.0089958, "ECCENTRICITY": 0.000134, "INCLINATION": 97.7419, "RA_OF_ASC_NODE": 190.1145, "ARG_OF_PERICENTER": 196.1905, "MEAN_ANOMALY": 163.9276, "BSTAR": 0.00046425, "EPOCH": "2026-03-13T22:55:07"}, {"OBJECT_NAME": "ICEYE-X40", "NORAD_CAT_ID": 60549, "MEAN_MOTION": 14.97582986, "ECCENTRICITY": 0.0005926, "INCLINATION": 97.6865, "RA_OF_ASC_NODE": 150.671, "ARG_OF_PERICENTER": 54.553, "MEAN_ANOMALY": 305.6244, "BSTAR": 0.00028059, "EPOCH": "2026-03-13T15:13:46"}, {"OBJECT_NAME": "ICEYE-X49", "NORAD_CAT_ID": 62384, "MEAN_MOTION": 15.1206309, "ECCENTRICITY": 0.0002244, "INCLINATION": 44.9973, "RA_OF_ASC_NODE": 147.332, "ARG_OF_PERICENTER": 261.341, "MEAN_ANOMALY": 98.7213, "BSTAR": 0.0005264600000000001, "EPOCH": "2026-03-13T14:29:43"}, {"OBJECT_NAME": "ICEYE-X27", "NORAD_CAT_ID": 55062, "MEAN_MOTION": 15.21915655, "ECCENTRICITY": 0.0013054, "INCLINATION": 97.4564, "RA_OF_ASC_NODE": 59.6015, "ARG_OF_PERICENTER": 58.5293, "MEAN_ANOMALY": 301.7217, "BSTAR": 0.00058627, "EPOCH": "2023-12-28T03:09:32"}, {"OBJECT_NAME": "ICEYE-X6", "NORAD_CAT_ID": 46497, "MEAN_MOTION": 15.039316, "ECCENTRICITY": 0.0008591, "INCLINATION": 98.0932, "RA_OF_ASC_NODE": 62.1157, "ARG_OF_PERICENTER": 56.1732, "MEAN_ANOMALY": 304.031, "BSTAR": 0.00023758, "EPOCH": "2026-03-13T22:26:47"}, {"OBJECT_NAME": "ICEYE-X47", "NORAD_CAT_ID": 62389, "MEAN_MOTION": 15.12292506, "ECCENTRICITY": 0.0001244, "INCLINATION": 44.9953, "RA_OF_ASC_NODE": 145.4322, "ARG_OF_PERICENTER": 284.6191, "MEAN_ANOMALY": 75.4548, "BSTAR": 0.00041522, "EPOCH": "2026-03-13T22:14:04"}, {"OBJECT_NAME": "ICEYE-X30", "NORAD_CAT_ID": 56947, "MEAN_MOTION": 15.73665321, "ECCENTRICITY": 0.0005197, "INCLINATION": 97.57, "RA_OF_ASC_NODE": 224.5987, "ARG_OF_PERICENTER": 272.4893, "MEAN_ANOMALY": 87.5776, "BSTAR": 0.00087825, "EPOCH": "2026-03-13T23:00:43"}, {"OBJECT_NAME": "ICEYE-X42", "NORAD_CAT_ID": 62698, "MEAN_MOTION": 14.91846966, "ECCENTRICITY": 0.0001233, "INCLINATION": 97.7388, "RA_OF_ASC_NODE": 152.9884, "ARG_OF_PERICENTER": 114.5043, "MEAN_ANOMALY": 245.6305, "BSTAR": 0.00032354, "EPOCH": "2026-03-13T23:08:06"}, {"OBJECT_NAME": "ICEYE-X41", "NORAD_CAT_ID": 62700, "MEAN_MOTION": 14.96111406, "ECCENTRICITY": 0.0001677, "INCLINATION": 97.7117, "RA_OF_ASC_NODE": 153.9314, "ARG_OF_PERICENTER": 40.0054, "MEAN_ANOMALY": 320.1291, "BSTAR": 0.00033944000000000004, "EPOCH": "2026-03-13T23:39:00"}, {"OBJECT_NAME": "ICEYE-X26", "NORAD_CAT_ID": 56961, "MEAN_MOTION": 15.73858643, "ECCENTRICITY": 9.1e-05, "INCLINATION": 97.5762, "RA_OF_ASC_NODE": 228.9322, "ARG_OF_PERICENTER": 244.947, "MEAN_ANOMALY": 115.1699, "BSTAR": 0.00051425, "EPOCH": "2026-03-13T22:45:47"}, {"OBJECT_NAME": "ICEYE-X45", "NORAD_CAT_ID": 62705, "MEAN_MOTION": 15.00865736, "ECCENTRICITY": 9.87e-05, "INCLINATION": 97.8062, "RA_OF_ASC_NODE": 158.3012, "ARG_OF_PERICENTER": 72.3822, "MEAN_ANOMALY": 287.7509, "BSTAR": 0.00031790000000000003, "EPOCH": "2026-03-13T04:34:44"}, {"OBJECT_NAME": "YAOGAN-29", "NORAD_CAT_ID": 41038, "MEAN_MOTION": 14.83002272, "ECCENTRICITY": 0.0001638, "INCLINATION": 98.0146, "RA_OF_ASC_NODE": 101.4189, "ARG_OF_PERICENTER": 30.4553, "MEAN_ANOMALY": 329.6755, "BSTAR": 0.00011476, "EPOCH": "2026-03-13T23:27:57"}, {"OBJECT_NAME": "YAOGAN-3", "NORAD_CAT_ID": 32289, "MEAN_MOTION": 14.90384748, "ECCENTRICITY": 0.0001556, "INCLINATION": 97.8274, "RA_OF_ASC_NODE": 103.8011, "ARG_OF_PERICENTER": 82.5153, "MEAN_ANOMALY": 277.6242, "BSTAR": 0.00015498, "EPOCH": "2026-03-14T00:08:30"}, {"OBJECT_NAME": "YAOGAN-4", "NORAD_CAT_ID": 33446, "MEAN_MOTION": 14.82990653, "ECCENTRICITY": 0.0015102, "INCLINATION": 97.9248, "RA_OF_ASC_NODE": 6.5617, "ARG_OF_PERICENTER": 28.3811, "MEAN_ANOMALY": 331.8222, "BSTAR": 0.00019828, "EPOCH": "2026-03-13T07:06:17"}, {"OBJECT_NAME": "YAOGAN-7", "NORAD_CAT_ID": 36110, "MEAN_MOTION": 14.77900526, "ECCENTRICITY": 0.0024434, "INCLINATION": 98.0203, "RA_OF_ASC_NODE": 319.3604, "ARG_OF_PERICENTER": 349.6706, "MEAN_ANOMALY": 10.3999, "BSTAR": 0.00011518, "EPOCH": "2026-03-13T23:13:20"}, {"OBJECT_NAME": "YAOGAN-21", "NORAD_CAT_ID": 40143, "MEAN_MOTION": 15.2500039, "ECCENTRICITY": 0.0009985, "INCLINATION": 97.1615, "RA_OF_ASC_NODE": 117.3514, "ARG_OF_PERICENTER": 74.4879, "MEAN_ANOMALY": 285.7462, "BSTAR": 0.0003128, "EPOCH": "2026-03-13T23:54:32"}, {"OBJECT_NAME": "YAOGAN-10", "NORAD_CAT_ID": 36834, "MEAN_MOTION": 14.85031759, "ECCENTRICITY": 0.0001558, "INCLINATION": 97.9083, "RA_OF_ASC_NODE": 97.1597, "ARG_OF_PERICENTER": 88.1696, "MEAN_ANOMALY": 328.1159, "BSTAR": 0.00020204, "EPOCH": "2026-03-14T00:26:33"}, {"OBJECT_NAME": "YAOGAN-23", "NORAD_CAT_ID": 40305, "MEAN_MOTION": 16.38800959, "ECCENTRICITY": 0.000344, "INCLINATION": 97.6653, "RA_OF_ASC_NODE": 349.5059, "ARG_OF_PERICENTER": 254.7314, "MEAN_ANOMALY": 115.2384, "BSTAR": 0.00072463, "EPOCH": "2024-12-12T06:42:04"}, {"OBJECT_NAME": "YAOGAN-12", "NORAD_CAT_ID": 37875, "MEAN_MOTION": 15.25448863, "ECCENTRICITY": 0.000959, "INCLINATION": 97.1283, "RA_OF_ASC_NODE": 116.0536, "ARG_OF_PERICENTER": 194.3554, "MEAN_ANOMALY": 165.7414, "BSTAR": 0.00020944, "EPOCH": "2026-03-13T23:41:58"}, {"OBJECT_NAME": "YAOGAN-13", "NORAD_CAT_ID": 37941, "MEAN_MOTION": 16.37208404, "ECCENTRICITY": 0.0005273, "INCLINATION": 97.6847, "RA_OF_ASC_NODE": 66.5996, "ARG_OF_PERICENTER": 226.2633, "MEAN_ANOMALY": 199.5818, "BSTAR": 0.00080897, "EPOCH": "2025-02-22T04:56:09"}, {"OBJECT_NAME": "YAOGAN-31 01A", "NORAD_CAT_ID": 43275, "MEAN_MOTION": 13.45452989, "ECCENTRICITY": 0.0267109, "INCLINATION": 63.398, "RA_OF_ASC_NODE": 336.1881, "ARG_OF_PERICENTER": 7.1349, "MEAN_ANOMALY": 353.3361, "BSTAR": 1.1273e-06, "EPOCH": "2026-03-14T00:03:39"}, {"OBJECT_NAME": "YAOGAN-31 01B", "NORAD_CAT_ID": 43276, "MEAN_MOTION": 13.45449081, "ECCENTRICITY": 0.0267122, "INCLINATION": 63.3982, "RA_OF_ASC_NODE": 336.1885, "ARG_OF_PERICENTER": 7.093, "MEAN_ANOMALY": 353.3758, "BSTAR": 5.9414000000000004e-05, "EPOCH": "2026-03-14T00:04:56"}, {"OBJECT_NAME": "YAOGAN-31 01C", "NORAD_CAT_ID": 43277, "MEAN_MOTION": 13.45433034, "ECCENTRICITY": 0.0267067, "INCLINATION": 63.3991, "RA_OF_ASC_NODE": 335.5353, "ARG_OF_PERICENTER": 6.8355, "MEAN_ANOMALY": 353.6199, "BSTAR": -4.8049e-05, "EPOCH": "2026-03-14T00:07:12"}, {"OBJECT_NAME": "YAOGAN-30 06A", "NORAD_CAT_ID": 44449, "MEAN_MOTION": 15.02133353, "ECCENTRICITY": 0.00037, "INCLINATION": 34.9928, "RA_OF_ASC_NODE": 191.5518, "ARG_OF_PERICENTER": 175.6657, "MEAN_ANOMALY": 184.4083, "BSTAR": 0.00026314, "EPOCH": "2026-03-13T17:58:19"}, {"OBJECT_NAME": "YAOGAN-30 06B", "NORAD_CAT_ID": 44450, "MEAN_MOTION": 15.02122328, "ECCENTRICITY": 0.0009262, "INCLINATION": 34.9916, "RA_OF_ASC_NODE": 190.8407, "ARG_OF_PERICENTER": 315.6605, "MEAN_ANOMALY": 44.3361, "BSTAR": 0.00028546, "EPOCH": "2026-03-13T19:04:54"}, {"OBJECT_NAME": "YAOGAN-30 03A", "NORAD_CAT_ID": 43028, "MEAN_MOTION": 15.02274662, "ECCENTRICITY": 0.0002217, "INCLINATION": 34.9941, "RA_OF_ASC_NODE": 133.0268, "ARG_OF_PERICENTER": 207.1099, "MEAN_ANOMALY": 152.9493, "BSTAR": 0.00029570000000000003, "EPOCH": "2026-03-13T14:23:34"}, {"OBJECT_NAME": "YAOGAN-34 02", "NORAD_CAT_ID": 52084, "MEAN_MOTION": 13.45429288, "ECCENTRICITY": 0.0043567, "INCLINATION": 63.3959, "RA_OF_ASC_NODE": 85.7443, "ARG_OF_PERICENTER": 351.6967, "MEAN_ANOMALY": 8.3333, "BSTAR": 5.2638e-05, "EPOCH": "2026-03-13T23:06:41"}, {"OBJECT_NAME": "YAOGAN-30 06C", "NORAD_CAT_ID": 44451, "MEAN_MOTION": 15.02204198, "ECCENTRICITY": 0.0001127, "INCLINATION": 34.9926, "RA_OF_ASC_NODE": 190.8564, "ARG_OF_PERICENTER": 269.6978, "MEAN_ANOMALY": 90.36, "BSTAR": 0.00028075, "EPOCH": "2026-03-13T18:31:13"}, {"OBJECT_NAME": "YAOGAN-30 03B", "NORAD_CAT_ID": 43029, "MEAN_MOTION": 15.02152547, "ECCENTRICITY": 0.0002692, "INCLINATION": 34.9936, "RA_OF_ASC_NODE": 131.1056, "ARG_OF_PERICENTER": 131.7298, "MEAN_ANOMALY": 228.3641, "BSTAR": 0.00019784, "EPOCH": "2026-03-13T20:21:13"}, {"OBJECT_NAME": "YAOGAN-34 01", "NORAD_CAT_ID": 48340, "MEAN_MOTION": 13.45464033, "ECCENTRICITY": 0.0079282, "INCLINATION": 63.4044, "RA_OF_ASC_NODE": 30.9154, "ARG_OF_PERICENTER": 356.7631, "MEAN_ANOMALY": 3.2873, "BSTAR": 0.00012672, "EPOCH": "2026-03-13T23:46:55"}, {"OBJECT_NAME": "YAOGAN-30 03C", "NORAD_CAT_ID": 43030, "MEAN_MOTION": 15.02202777, "ECCENTRICITY": 0.0002654, "INCLINATION": 34.9945, "RA_OF_ASC_NODE": 132.4019, "ARG_OF_PERICENTER": 283.5763, "MEAN_ANOMALY": 76.4649, "BSTAR": 0.00028654, "EPOCH": "2026-03-13T15:02:11"}, {"OBJECT_NAME": "WORLDVIEW-3 (WV-3)", "NORAD_CAT_ID": 40115, "MEAN_MOTION": 14.84845969, "ECCENTRICITY": 2.33e-05, "INCLINATION": 97.8627, "RA_OF_ASC_NODE": 149.0901, "ARG_OF_PERICENTER": 180.2093, "MEAN_ANOMALY": 179.9122, "BSTAR": 0.00013739, "EPOCH": "2026-03-13T22:53:12"}, {"OBJECT_NAME": "WORLDVIEW-2 (WV-2)", "NORAD_CAT_ID": 35946, "MEAN_MOTION": 14.37912466, "ECCENTRICITY": 0.0005217, "INCLINATION": 98.4686, "RA_OF_ASC_NODE": 147.7074, "ARG_OF_PERICENTER": 134.8366, "MEAN_ANOMALY": 225.3246, "BSTAR": 6.7026e-05, "EPOCH": "2026-03-13T21:38:50"}, {"OBJECT_NAME": "WORLDVIEW-1 (WV-1)", "NORAD_CAT_ID": 32060, "MEAN_MOTION": 15.24615389, "ECCENTRICITY": 0.0003347, "INCLINATION": 97.3824, "RA_OF_ASC_NODE": 194.0222, "ARG_OF_PERICENTER": 128.0866, "MEAN_ANOMALY": 232.0674, "BSTAR": 0.00036453, "EPOCH": "2026-03-13T21:39:56"}, {"OBJECT_NAME": "GEOEYE 1", "NORAD_CAT_ID": 33331, "MEAN_MOTION": 14.64773089, "ECCENTRICITY": 0.0003706, "INCLINATION": 98.12, "RA_OF_ASC_NODE": 148.0546, "ARG_OF_PERICENTER": 16.4564, "MEAN_ANOMALY": 343.6759, "BSTAR": 0.00011921, "EPOCH": "2026-03-13T22:07:31"}, {"OBJECT_NAME": "LUCH DEB", "NORAD_CAT_ID": 44582, "MEAN_MOTION": 1.00701065, "ECCENTRICITY": 0.0017009, "INCLINATION": 14.6048, "RA_OF_ASC_NODE": 354.7817, "ARG_OF_PERICENTER": 87.9758, "MEAN_ANOMALY": 77.8273, "BSTAR": 0.0, "EPOCH": "2026-03-12T17:12:42"}, {"OBJECT_NAME": "LUCH-5X (OLYMP-K 2)", "NORAD_CAT_ID": 55841, "MEAN_MOTION": 1.00270407, "ECCENTRICITY": 0.0001458, "INCLINATION": 0.0279, "RA_OF_ASC_NODE": 89.0645, "ARG_OF_PERICENTER": 119.8966, "MEAN_ANOMALY": 233.8673, "BSTAR": 0.0, "EPOCH": "2026-03-13T13:58:37"}, {"OBJECT_NAME": "LUCH (OLYMP-K 1) DEB", "NORAD_CAT_ID": 67745, "MEAN_MOTION": 0.97993456, "ECCENTRICITY": 0.0520769, "INCLINATION": 1.5795, "RA_OF_ASC_NODE": 85.6956, "ARG_OF_PERICENTER": 329.0115, "MEAN_ANOMALY": 28.4425, "BSTAR": 0.0, "EPOCH": "2026-02-26T11:11:16"}, {"OBJECT_NAME": "LUCH", "NORAD_CAT_ID": 23426, "MEAN_MOTION": 1.00183208, "ECCENTRICITY": 0.0004728, "INCLINATION": 14.7546, "RA_OF_ASC_NODE": 355.344, "ARG_OF_PERICENTER": 224.699, "MEAN_ANOMALY": 135.2716, "BSTAR": 0.0, "EPOCH": "2026-03-12T16:39:38"}, {"OBJECT_NAME": "LUCH-1", "NORAD_CAT_ID": 23680, "MEAN_MOTION": 1.00266907, "ECCENTRICITY": 0.0004384, "INCLINATION": 15.0217, "RA_OF_ASC_NODE": 0.003, "ARG_OF_PERICENTER": 147.6136, "MEAN_ANOMALY": 13.5386, "BSTAR": 0.0, "EPOCH": "2026-03-13T18:16:08"}, {"OBJECT_NAME": "LUCH 5A (SDCM/PRN 140)", "NORAD_CAT_ID": 37951, "MEAN_MOTION": 1.00268953, "ECCENTRICITY": 0.0003411, "INCLINATION": 8.5104, "RA_OF_ASC_NODE": 75.2044, "ARG_OF_PERICENTER": 262.4684, "MEAN_ANOMALY": 295.0348, "BSTAR": 0.0, "EPOCH": "2026-03-13T19:36:38"}, {"OBJECT_NAME": "LUCH 5B (SDCM/PRN 125)", "NORAD_CAT_ID": 38977, "MEAN_MOTION": 1.00271953, "ECCENTRICITY": 0.0003244, "INCLINATION": 10.2781, "RA_OF_ASC_NODE": 50.8078, "ARG_OF_PERICENTER": 227.3464, "MEAN_ANOMALY": 230.5377, "BSTAR": 0.0, "EPOCH": "2026-03-13T23:33:37"}, {"OBJECT_NAME": "LUCH 5V (SDCM/PRN 141)", "NORAD_CAT_ID": 39727, "MEAN_MOTION": 1.00269069, "ECCENTRICITY": 0.000314, "INCLINATION": 4.8975, "RA_OF_ASC_NODE": 70.6733, "ARG_OF_PERICENTER": 297.1295, "MEAN_ANOMALY": 245.4574, "BSTAR": 0.0, "EPOCH": "2026-03-13T23:07:57"}, {"OBJECT_NAME": "LUCH (OLYMP-K 1)", "NORAD_CAT_ID": 40258, "MEAN_MOTION": 0.99116915, "ECCENTRICITY": 0.0003121, "INCLINATION": 1.4978, "RA_OF_ASC_NODE": 84.5508, "ARG_OF_PERICENTER": 187.5711, "MEAN_ANOMALY": 175.9774, "BSTAR": 0.0, "EPOCH": "2026-01-14T19:27:12"}, {"OBJECT_NAME": "TANDEM-X", "NORAD_CAT_ID": 36605, "MEAN_MOTION": 15.19145167, "ECCENTRICITY": 0.0001909, "INCLINATION": 97.4475, "RA_OF_ASC_NODE": 81.6192, "ARG_OF_PERICENTER": 101.4604, "MEAN_ANOMALY": 258.6845, "BSTAR": 8.3268e-05, "EPOCH": "2026-03-13T13:13:24"}, {"OBJECT_NAME": "PAZ", "NORAD_CAT_ID": 43215, "MEAN_MOTION": 15.19158174, "ECCENTRICITY": 0.0001722, "INCLINATION": 97.4462, "RA_OF_ASC_NODE": 81.7323, "ARG_OF_PERICENTER": 88.5098, "MEAN_ANOMALY": 271.6334, "BSTAR": 0.00010771, "EPOCH": "2026-03-13T23:50:12"}, {"OBJECT_NAME": "KONDOR-FKA NO. 1", "NORAD_CAT_ID": 56756, "MEAN_MOTION": 15.19753823, "ECCENTRICITY": 0.0001625, "INCLINATION": 97.44, "RA_OF_ASC_NODE": 268.8789, "ARG_OF_PERICENTER": 82.9203, "MEAN_ANOMALY": 277.2216, "BSTAR": 0.00027241, "EPOCH": "2026-03-13T22:39:44"}, {"OBJECT_NAME": "KONDOR-FKA NO. 2", "NORAD_CAT_ID": 62138, "MEAN_MOTION": 15.19712795, "ECCENTRICITY": 0.0001697, "INCLINATION": 97.4351, "RA_OF_ASC_NODE": 277.7975, "ARG_OF_PERICENTER": 86.2286, "MEAN_ANOMALY": 273.9143, "BSTAR": 0.00012670000000000002, "EPOCH": "2026-03-13T23:15:24"}, {"OBJECT_NAME": "BEIDOU-2 M4 (C12)", "NORAD_CAT_ID": 38251, "MEAN_MOTION": 1.86229915, "ECCENTRICITY": 0.0012191, "INCLINATION": 55.7402, "RA_OF_ASC_NODE": 307.3399, "ARG_OF_PERICENTER": 287.4004, "MEAN_ANOMALY": 72.522, "BSTAR": 0.0, "EPOCH": "2026-03-13T00:07:52"}, {"OBJECT_NAME": "BEIDOU-2 M1", "NORAD_CAT_ID": 31115, "MEAN_MOTION": 1.77349392, "ECCENTRICITY": 0.0002326, "INCLINATION": 50.9679, "RA_OF_ASC_NODE": 222.5681, "ARG_OF_PERICENTER": 33.8688, "MEAN_ANOMALY": 326.1041, "BSTAR": 0.0, "EPOCH": "2026-03-10T03:28:02"}, {"OBJECT_NAME": "BEIDOU-3 M24 (C46)", "NORAD_CAT_ID": 44542, "MEAN_MOTION": 1.86229082, "ECCENTRICITY": 0.0008561, "INCLINATION": 54.3985, "RA_OF_ASC_NODE": 185.9849, "ARG_OF_PERICENTER": 19.8304, "MEAN_ANOMALY": 340.1884, "BSTAR": 0.0, "EPOCH": "2026-03-12T14:10:15"}, {"OBJECT_NAME": "BEIDOU-3 M23 (C45)", "NORAD_CAT_ID": 44543, "MEAN_MOTION": 1.86227389, "ECCENTRICITY": 0.0004976, "INCLINATION": 54.3975, "RA_OF_ASC_NODE": 185.9688, "ARG_OF_PERICENTER": 10.6705, "MEAN_ANOMALY": 318.8703, "BSTAR": 0.0, "EPOCH": "2026-03-12T03:20:09"}, {"OBJECT_NAME": "BEIDOU-3 IGSO-3 (C40)", "NORAD_CAT_ID": 44709, "MEAN_MOTION": 1.00267242, "ECCENTRICITY": 0.0040728, "INCLINATION": 54.9673, "RA_OF_ASC_NODE": 281.9576, "ARG_OF_PERICENTER": 188.7089, "MEAN_ANOMALY": 171.2538, "BSTAR": 0.0, "EPOCH": "2026-03-10T23:36:11"}, {"OBJECT_NAME": "BEIDOU-3 M5 (C23)", "NORAD_CAT_ID": 43581, "MEAN_MOTION": 1.86228824, "ECCENTRICITY": 0.0002388, "INCLINATION": 54.0877, "RA_OF_ASC_NODE": 185.5927, "ARG_OF_PERICENTER": 304.9134, "MEAN_ANOMALY": 55.0505, "BSTAR": 0.0, "EPOCH": "2026-03-12T10:55:13"}, {"OBJECT_NAME": "BEIDOU-3 M22 (C44)", "NORAD_CAT_ID": 44793, "MEAN_MOTION": 1.86232299, "ECCENTRICITY": 0.0007468, "INCLINATION": 54.0225, "RA_OF_ASC_NODE": 303.2142, "ARG_OF_PERICENTER": 38.0258, "MEAN_ANOMALY": 322.0804, "BSTAR": 0.0, "EPOCH": "2026-03-12T23:11:55"}, {"OBJECT_NAME": "BEIDOU-3 M6 (C24)", "NORAD_CAT_ID": 43582, "MEAN_MOTION": 1.86227825, "ECCENTRICITY": 0.0005988, "INCLINATION": 54.0872, "RA_OF_ASC_NODE": 185.5682, "ARG_OF_PERICENTER": 41.7221, "MEAN_ANOMALY": 318.3092, "BSTAR": 0.0, "EPOCH": "2026-03-12T20:31:06"}, {"OBJECT_NAME": "BEIDOU-3 M21 (C43)", "NORAD_CAT_ID": 44794, "MEAN_MOTION": 1.86227925, "ECCENTRICITY": 0.0005425, "INCLINATION": 53.9966, "RA_OF_ASC_NODE": 303.2101, "ARG_OF_PERICENTER": 33.1471, "MEAN_ANOMALY": 326.9381, "BSTAR": 0.0, "EPOCH": "2026-03-12T00:46:41"}, {"OBJECT_NAME": "BEIDOU-2 G8 (C01)", "NORAD_CAT_ID": 44231, "MEAN_MOTION": 1.00273975, "ECCENTRICITY": 0.0011489, "INCLINATION": 1.4155, "RA_OF_ASC_NODE": 74.2587, "ARG_OF_PERICENTER": 274.6914, "MEAN_ANOMALY": 311.0157, "BSTAR": 0.0, "EPOCH": "2026-03-13T22:55:34"}, {"OBJECT_NAME": "BEIDOU-3 IGSO-2 (C39)", "NORAD_CAT_ID": 44337, "MEAN_MOTION": 1.00247364, "ECCENTRICITY": 0.0038734, "INCLINATION": 55.2338, "RA_OF_ASC_NODE": 159.8353, "ARG_OF_PERICENTER": 206.4752, "MEAN_ANOMALY": 153.7223, "BSTAR": 0.0, "EPOCH": "2026-02-26T16:31:51"}, {"OBJECT_NAME": "BEIDOU-3 M10 (C30)", "NORAD_CAT_ID": 43246, "MEAN_MOTION": 1.86229238, "ECCENTRICITY": 0.0005206, "INCLINATION": 54.2796, "RA_OF_ASC_NODE": 303.4048, "ARG_OF_PERICENTER": 25.3248, "MEAN_ANOMALY": 334.7506, "BSTAR": 0.0, "EPOCH": "2026-03-11T03:46:01"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-7 (C16)", "NORAD_CAT_ID": 43539, "MEAN_MOTION": 1.00280738, "ECCENTRICITY": 0.0100561, "INCLINATION": 55.1863, "RA_OF_ASC_NODE": 163.8949, "ARG_OF_PERICENTER": 236.784, "MEAN_ANOMALY": 122.3454, "BSTAR": 0.0, "EPOCH": "2026-03-13T15:56:01"}, {"OBJECT_NAME": "BEIDOU-2 M6 (C14)", "NORAD_CAT_ID": 38775, "MEAN_MOTION": 1.86231021, "ECCENTRICITY": 0.0014127, "INCLINATION": 56.4609, "RA_OF_ASC_NODE": 66.0825, "ARG_OF_PERICENTER": 342.5009, "MEAN_ANOMALY": 17.4715, "BSTAR": 0.0, "EPOCH": "2026-03-12T17:10:29"}, {"OBJECT_NAME": "CHINASAT 31 (BEIDOU-1 *)", "NORAD_CAT_ID": 26643, "MEAN_MOTION": 0.99256234, "ECCENTRICITY": 0.0077094, "INCLINATION": 12.4543, "RA_OF_ASC_NODE": 28.03, "ARG_OF_PERICENTER": 84.8533, "MEAN_ANOMALY": 88.8595, "BSTAR": 0.0, "EPOCH": "2026-03-13T03:19:44"}, {"OBJECT_NAME": "BEIDOU-2 G6 (C02)", "NORAD_CAT_ID": 38953, "MEAN_MOTION": 1.00276062, "ECCENTRICITY": 0.001054, "INCLINATION": 4.1785, "RA_OF_ASC_NODE": 74.5404, "ARG_OF_PERICENTER": 306.4198, "MEAN_ANOMALY": 188.414, "BSTAR": 0.0, "EPOCH": "2026-03-13T20:55:45"}, {"OBJECT_NAME": "BEIDOU-1 G3", "NORAD_CAT_ID": 27813, "MEAN_MOTION": 0.99765897, "ECCENTRICITY": 0.0019498, "INCLINATION": 11.1948, "RA_OF_ASC_NODE": 42.0288, "ARG_OF_PERICENTER": 301.8279, "MEAN_ANOMALY": 234.54, "BSTAR": 0.0, "EPOCH": "2026-03-13T22:33:51"}, {"OBJECT_NAME": "BEIDOU-3S IGSO-1S (C31)", "NORAD_CAT_ID": 40549, "MEAN_MOTION": 1.00272189, "ECCENTRICITY": 0.0035647, "INCLINATION": 49.3736, "RA_OF_ASC_NODE": 296.6132, "ARG_OF_PERICENTER": 190.1678, "MEAN_ANOMALY": 352.9044, "BSTAR": 0.0, "EPOCH": "2026-03-13T14:08:14"}, {"OBJECT_NAME": "BEIDOU-1 G4", "NORAD_CAT_ID": 30323, "MEAN_MOTION": 0.99158371, "ECCENTRICITY": 0.0057038, "INCLINATION": 9.6698, "RA_OF_ASC_NODE": 57.9536, "ARG_OF_PERICENTER": 271.1825, "MEAN_ANOMALY": 91.8212, "BSTAR": 0.0, "EPOCH": "2026-03-13T18:40:28"}, {"OBJECT_NAME": "BEIDOU-3 M12 (C26)", "NORAD_CAT_ID": 43602, "MEAN_MOTION": 1.8622785, "ECCENTRICITY": 0.0008368, "INCLINATION": 54.1816, "RA_OF_ASC_NODE": 184.3275, "ARG_OF_PERICENTER": 38.978, "MEAN_ANOMALY": 321.0706, "BSTAR": 0.0, "EPOCH": "2026-03-12T06:09:12"}, {"OBJECT_NAME": "SHIJIAN-16 (SJ-16)", "NORAD_CAT_ID": 39358, "MEAN_MOTION": 14.92370565, "ECCENTRICITY": 0.001532, "INCLINATION": 74.9729, "RA_OF_ASC_NODE": 176.9542, "ARG_OF_PERICENTER": 101.4806, "MEAN_ANOMALY": 258.8104, "BSTAR": 0.00032554, "EPOCH": "2026-03-13T23:40:36"}, {"OBJECT_NAME": "SHIJIAN-20 (SJ-20)", "NORAD_CAT_ID": 44910, "MEAN_MOTION": 1.00082407, "ECCENTRICITY": 0.0001536, "INCLINATION": 4.6699, "RA_OF_ASC_NODE": 76.2395, "ARG_OF_PERICENTER": 16.315, "MEAN_ANOMALY": 96.486, "BSTAR": 0.0, "EPOCH": "2026-03-12T20:33:00"}, {"OBJECT_NAME": "SHIJIAN-6 01A (SJ-6 01A)", "NORAD_CAT_ID": 28413, "MEAN_MOTION": 15.16526688, "ECCENTRICITY": 0.0008183, "INCLINATION": 97.5978, "RA_OF_ASC_NODE": 99.7091, "ARG_OF_PERICENTER": 104.1313, "MEAN_ANOMALY": 256.0829, "BSTAR": 0.0005259500000000001, "EPOCH": "2026-03-13T19:17:28"}, {"OBJECT_NAME": "SHIJIAN-6 02B (SJ-6 02B)", "NORAD_CAT_ID": 29506, "MEAN_MOTION": 15.01021872, "ECCENTRICITY": 0.0013821, "INCLINATION": 97.7077, "RA_OF_ASC_NODE": 102.6645, "ARG_OF_PERICENTER": 30.0857, "MEAN_ANOMALY": 330.1158, "BSTAR": 0.00018838, "EPOCH": "2026-03-13T23:08:48"}, {"OBJECT_NAME": "SHIJIAN-30A (SJ-30A)", "NORAD_CAT_ID": 66545, "MEAN_MOTION": 15.15423779, "ECCENTRICITY": 0.0011472, "INCLINATION": 51.7981, "RA_OF_ASC_NODE": 265.5259, "ARG_OF_PERICENTER": 306.6475, "MEAN_ANOMALY": 53.3446, "BSTAR": 0.0005244900000000001, "EPOCH": "2026-03-13T22:48:09"}, {"OBJECT_NAME": "SHIJIAN-6 03A (SJ-6 03A)", "NORAD_CAT_ID": 33408, "MEAN_MOTION": 15.16124436, "ECCENTRICITY": 0.0011042, "INCLINATION": 97.8533, "RA_OF_ASC_NODE": 103.0077, "ARG_OF_PERICENTER": 348.3714, "MEAN_ANOMALY": 11.726, "BSTAR": 0.00021966, "EPOCH": "2026-03-13T23:28:43"}, {"OBJECT_NAME": "SHIJIAN-6 04B (SJ-6 04B)", "NORAD_CAT_ID": 37180, "MEAN_MOTION": 14.99126331, "ECCENTRICITY": 0.0009055, "INCLINATION": 97.8747, "RA_OF_ASC_NODE": 77.415, "ARG_OF_PERICENTER": 277.7452, "MEAN_ANOMALY": 82.2742, "BSTAR": 4.071e-05, "EPOCH": "2026-03-13T23:33:25"}, {"OBJECT_NAME": "SHIJIAN-6 02A (SJ-6 02A)", "NORAD_CAT_ID": 29505, "MEAN_MOTION": 15.15768295, "ECCENTRICITY": 0.0004763, "INCLINATION": 97.6496, "RA_OF_ASC_NODE": 113.376, "ARG_OF_PERICENTER": 104.7518, "MEAN_ANOMALY": 255.4243, "BSTAR": 0.00032697, "EPOCH": "2026-03-14T00:03:32"}, {"OBJECT_NAME": "SHIJIAN-6 03B (SJ-6 03B)", "NORAD_CAT_ID": 33409, "MEAN_MOTION": 15.03909215, "ECCENTRICITY": 0.0019698, "INCLINATION": 97.8671, "RA_OF_ASC_NODE": 91.2719, "ARG_OF_PERICENTER": 8.8097, "MEAN_ANOMALY": 351.347, "BSTAR": 0.00026502, "EPOCH": "2026-03-13T23:16:39"}, {"OBJECT_NAME": "SHIJIAN-6 04A (SJ-6 04A)", "NORAD_CAT_ID": 37179, "MEAN_MOTION": 15.1671358, "ECCENTRICITY": 0.0018691, "INCLINATION": 97.8382, "RA_OF_ASC_NODE": 92.9623, "ARG_OF_PERICENTER": 189.0595, "MEAN_ANOMALY": 171.0302, "BSTAR": 0.00034501, "EPOCH": "2026-03-13T23:58:57"}, {"OBJECT_NAME": "SHIJIAN-17 (SJ-17)", "NORAD_CAT_ID": 41838, "MEAN_MOTION": 0.99865501, "ECCENTRICITY": 0.0001148, "INCLINATION": 5.5064, "RA_OF_ASC_NODE": 73.7601, "ARG_OF_PERICENTER": 325.6027, "MEAN_ANOMALY": 47.6368, "BSTAR": 0.0, "EPOCH": "2026-03-12T17:13:52"}, {"OBJECT_NAME": "SHIJIAN-21 (SJ-21)", "NORAD_CAT_ID": 49330, "MEAN_MOTION": 1.00263511, "ECCENTRICITY": 0.0048014, "INCLINATION": 4.8865, "RA_OF_ASC_NODE": 61.7086, "ARG_OF_PERICENTER": 171.424, "MEAN_ANOMALY": 61.5631, "BSTAR": 0.0, "EPOCH": "2026-03-13T23:41:25"}, {"OBJECT_NAME": "SHIJIAN-23 (SJ-23)", "NORAD_CAT_ID": 55131, "MEAN_MOTION": 1.00484158, "ECCENTRICITY": 0.0005724, "INCLINATION": 3.4708, "RA_OF_ASC_NODE": 79.5278, "ARG_OF_PERICENTER": 265.1652, "MEAN_ANOMALY": 240.6077, "BSTAR": 0.0, "EPOCH": "2026-03-13T18:28:41"}, {"OBJECT_NAME": "SHIJIAN-19 (SJ-19)", "NORAD_CAT_ID": 61444, "MEAN_MOTION": 15.77608314, "ECCENTRICITY": 0.0006944, "INCLINATION": 41.601, "RA_OF_ASC_NODE": 76.8644, "ARG_OF_PERICENTER": 34.8351, "MEAN_ANOMALY": 325.295, "BSTAR": 9.0833e-05, "EPOCH": "2024-10-10T17:00:10"}, {"OBJECT_NAME": "SHIJIAN-6 01B (SJ-6 01B)", "NORAD_CAT_ID": 28414, "MEAN_MOTION": 15.05666414, "ECCENTRICITY": 0.0006787, "INCLINATION": 97.6147, "RA_OF_ASC_NODE": 92.0716, "ARG_OF_PERICENTER": 101.7171, "MEAN_ANOMALY": 258.4817, "BSTAR": 0.00011612, "EPOCH": "2026-03-13T23:18:12"}, {"OBJECT_NAME": "SHIJIAN-25 (SJ-25)", "NORAD_CAT_ID": 62485, "MEAN_MOTION": 1.0027194, "ECCENTRICITY": 0.0048627, "INCLINATION": 4.8863, "RA_OF_ASC_NODE": 61.7143, "ARG_OF_PERICENTER": 166.3864, "MEAN_ANOMALY": 329.1967, "BSTAR": 0.0, "EPOCH": "2026-03-13T17:14:33"}, {"OBJECT_NAME": "SHIJIAN-26 (SJ-26)", "NORAD_CAT_ID": 64199, "MEAN_MOTION": 15.22615444, "ECCENTRICITY": 0.0018329, "INCLINATION": 97.4607, "RA_OF_ASC_NODE": 152.108, "ARG_OF_PERICENTER": 330.5834, "MEAN_ANOMALY": 29.4369, "BSTAR": 0.00030253, "EPOCH": "2026-03-13T22:31:08"}, {"OBJECT_NAME": "SHIJIAN-30B (SJ-30B)", "NORAD_CAT_ID": 66546, "MEAN_MOTION": 15.15360631, "ECCENTRICITY": 0.0009402, "INCLINATION": 51.7965, "RA_OF_ASC_NODE": 265.5138, "ARG_OF_PERICENTER": 308.952, "MEAN_ANOMALY": 51.0618, "BSTAR": 0.00055464, "EPOCH": "2026-03-13T22:49:39"}, {"OBJECT_NAME": "SHIJIAN-30C (SJ-30C)", "NORAD_CAT_ID": 66547, "MEAN_MOTION": 15.15463214, "ECCENTRICITY": 0.0010197, "INCLINATION": 51.7973, "RA_OF_ASC_NODE": 265.5296, "ARG_OF_PERICENTER": 299.7369, "MEAN_ANOMALY": 60.2592, "BSTAR": 0.0005682199999999999, "EPOCH": "2026-03-13T22:50:54"}, {"OBJECT_NAME": "SHIJIAN-28 (SJ-28)", "NORAD_CAT_ID": 66549, "MEAN_MOTION": 1.00269894, "ECCENTRICITY": 6.05e-05, "INCLINATION": 4.7853, "RA_OF_ASC_NODE": 273.572, "ARG_OF_PERICENTER": 175.7995, "MEAN_ANOMALY": 180.8826, "BSTAR": 0.0, "EPOCH": "2026-03-13T22:47:42"}, {"OBJECT_NAME": "GSAT0101 (GALILEO-PFM)", "NORAD_CAT_ID": 37846, "MEAN_MOTION": 1.70475478, "ECCENTRICITY": 0.0003975, "INCLINATION": 57.0318, "RA_OF_ASC_NODE": 344.8758, "ARG_OF_PERICENTER": 357.0591, "MEAN_ANOMALY": 2.9533, "BSTAR": 0.0, "EPOCH": "2026-02-22T11:08:37"}, {"OBJECT_NAME": "GSAT0219 (GALILEO 23)", "NORAD_CAT_ID": 43566, "MEAN_MOTION": 1.70474822, "ECCENTRICITY": 0.0004318, "INCLINATION": 57.205, "RA_OF_ASC_NODE": 344.5202, "ARG_OF_PERICENTER": 344.0327, "MEAN_ANOMALY": 15.9782, "BSTAR": 0.0, "EPOCH": "2026-03-08T00:37:58"}, {"OBJECT_NAME": "GSAT0207 (GALILEO 15)", "NORAD_CAT_ID": 41859, "MEAN_MOTION": 1.70474457, "ECCENTRICITY": 0.0005361, "INCLINATION": 55.4358, "RA_OF_ASC_NODE": 103.7267, "ARG_OF_PERICENTER": 309.9075, "MEAN_ANOMALY": 50.1195, "BSTAR": 0.0, "EPOCH": "2026-03-13T17:15:10"}, {"OBJECT_NAME": "GSAT0103 (GALILEO-FM3)", "NORAD_CAT_ID": 38857, "MEAN_MOTION": 1.70473527, "ECCENTRICITY": 0.000557, "INCLINATION": 55.7441, "RA_OF_ASC_NODE": 104.2408, "ARG_OF_PERICENTER": 286.1218, "MEAN_ANOMALY": 73.8831, "BSTAR": 0.0, "EPOCH": "2026-03-10T22:17:59"}, {"OBJECT_NAME": "GSAT0232 (GALILEO 32)", "NORAD_CAT_ID": 61182, "MEAN_MOTION": 1.70443939, "ECCENTRICITY": 0.00036, "INCLINATION": 55.2225, "RA_OF_ASC_NODE": 224.5881, "ARG_OF_PERICENTER": 296.6314, "MEAN_ANOMALY": 116.1306, "BSTAR": 0.0, "EPOCH": "2026-02-17T15:52:34"}, {"OBJECT_NAME": "GSAT0226 (GALILEO 31)", "NORAD_CAT_ID": 61183, "MEAN_MOTION": 1.70474359, "ECCENTRICITY": 0.0001584, "INCLINATION": 55.2155, "RA_OF_ASC_NODE": 223.9355, "ARG_OF_PERICENTER": 137.4612, "MEAN_ANOMALY": 46.3182, "BSTAR": 0.0, "EPOCH": "2026-03-13T18:37:21"}, {"OBJECT_NAME": "GSAT0212 (GALILEO 16)", "NORAD_CAT_ID": 41860, "MEAN_MOTION": 1.70474723, "ECCENTRICITY": 0.0003884, "INCLINATION": 55.4332, "RA_OF_ASC_NODE": 103.7422, "ARG_OF_PERICENTER": 334.9903, "MEAN_ANOMALY": 25.0622, "BSTAR": 0.0, "EPOCH": "2026-03-12T23:39:35"}, {"OBJECT_NAME": "GSAT0233 (GALILEO 33)", "NORAD_CAT_ID": 67160, "MEAN_MOTION": 1.70474578, "ECCENTRICITY": 0.0003011, "INCLINATION": 54.3935, "RA_OF_ASC_NODE": 105.2952, "ARG_OF_PERICENTER": 207.0582, "MEAN_ANOMALY": 359.4109, "BSTAR": 0.0, "EPOCH": "2026-02-13T12:00:00"}, {"OBJECT_NAME": "GSAT0213 (GALILEO 17)", "NORAD_CAT_ID": 41861, "MEAN_MOTION": 1.70475014, "ECCENTRICITY": 0.0005437, "INCLINATION": 55.4354, "RA_OF_ASC_NODE": 103.7885, "ARG_OF_PERICENTER": 293.5263, "MEAN_ANOMALY": 66.4835, "BSTAR": 0.0, "EPOCH": "2026-03-11T14:14:38"}, {"OBJECT_NAME": "GSAT0234 (GALILEO 34)", "NORAD_CAT_ID": 67162, "MEAN_MOTION": 1.70475046, "ECCENTRICITY": 0.0002786, "INCLINATION": 54.2701, "RA_OF_ASC_NODE": 104.8613, "ARG_OF_PERICENTER": 225.0665, "MEAN_ANOMALY": 135.9201, "BSTAR": 0.0, "EPOCH": "2026-03-13T07:42:19"}, {"OBJECT_NAME": "GSAT0214 (GALILEO 18)", "NORAD_CAT_ID": 41862, "MEAN_MOTION": 1.70474834, "ECCENTRICITY": 0.0004476, "INCLINATION": 55.4337, "RA_OF_ASC_NODE": 103.7448, "ARG_OF_PERICENTER": 297.9628, "MEAN_ANOMALY": 62.0628, "BSTAR": 0.0, "EPOCH": "2026-03-12T21:54:20"}, {"OBJECT_NAME": "GSAT0102 (GALILEO-FM2)", "NORAD_CAT_ID": 37847, "MEAN_MOTION": 1.70475778, "ECCENTRICITY": 0.0005422, "INCLINATION": 57.0301, "RA_OF_ASC_NODE": 344.6971, "ARG_OF_PERICENTER": 341.2092, "MEAN_ANOMALY": 154.1953, "BSTAR": 0.0, "EPOCH": "2026-03-01T01:30:59"}, {"OBJECT_NAME": "GSAT0215 (GALILEO 19)", "NORAD_CAT_ID": 43055, "MEAN_MOTION": 1.70474604, "ECCENTRICITY": 5.03e-05, "INCLINATION": 55.1052, "RA_OF_ASC_NODE": 224.1555, "ARG_OF_PERICENTER": 355.2258, "MEAN_ANOMALY": 4.7321, "BSTAR": 0.0, "EPOCH": "2026-03-12T19:35:18"}, {"OBJECT_NAME": "GSAT0216 (GALILEO 20)", "NORAD_CAT_ID": 43056, "MEAN_MOTION": 1.70474728, "ECCENTRICITY": 0.0001729, "INCLINATION": 55.1057, "RA_OF_ASC_NODE": 224.1638, "ARG_OF_PERICENTER": 313.2085, "MEAN_ANOMALY": 46.7359, "BSTAR": 0.0, "EPOCH": "2026-03-12T12:29:59"}, {"OBJECT_NAME": "GALILEO104 [GAL]", "NORAD_CAT_ID": 38858, "MEAN_MOTION": 1.64592169, "ECCENTRICITY": 0.0001481, "INCLINATION": 55.4156, "RA_OF_ASC_NODE": 121.7581, "ARG_OF_PERICENTER": 313.3657, "MEAN_ANOMALY": 182.5612, "BSTAR": 0.0, "EPOCH": "2024-06-18T00:14:41"}, {"OBJECT_NAME": "GSAT0201 (GALILEO 5)", "NORAD_CAT_ID": 40128, "MEAN_MOTION": 1.8552017, "ECCENTRICITY": 0.1658807, "INCLINATION": 48.9815, "RA_OF_ASC_NODE": 277.4406, "ARG_OF_PERICENTER": 175.5932, "MEAN_ANOMALY": 186.0859, "BSTAR": 0.0, "EPOCH": "2026-03-11T07:43:01"}, {"OBJECT_NAME": "GSAT0217 (GALILEO 21)", "NORAD_CAT_ID": 43057, "MEAN_MOTION": 1.70474639, "ECCENTRICITY": 0.0001948, "INCLINATION": 55.1048, "RA_OF_ASC_NODE": 224.1555, "ARG_OF_PERICENTER": 350.8522, "MEAN_ANOMALY": 9.1027, "BSTAR": 0.0, "EPOCH": "2026-03-12T17:47:52"}, {"OBJECT_NAME": "GSAT0202 (GALILEO 6)", "NORAD_CAT_ID": 40129, "MEAN_MOTION": 1.85520847, "ECCENTRICITY": 0.1659654, "INCLINATION": 48.9971, "RA_OF_ASC_NODE": 276.5149, "ARG_OF_PERICENTER": 176.3418, "MEAN_ANOMALY": 185.0558, "BSTAR": 0.0, "EPOCH": "2026-03-11T13:51:31"}, {"OBJECT_NAME": "GSAT0218 (GALILEO 22)", "NORAD_CAT_ID": 43058, "MEAN_MOTION": 1.70474604, "ECCENTRICITY": 0.0002024, "INCLINATION": 55.1035, "RA_OF_ASC_NODE": 224.1311, "ARG_OF_PERICENTER": 298.9535, "MEAN_ANOMALY": 60.9828, "BSTAR": 0.0, "EPOCH": "2026-03-13T13:09:25"}, {"OBJECT_NAME": "GSAT0203 (GALILEO 7)", "NORAD_CAT_ID": 40544, "MEAN_MOTION": 1.70475763, "ECCENTRICITY": 0.0004663, "INCLINATION": 56.8181, "RA_OF_ASC_NODE": 344.2043, "ARG_OF_PERICENTER": 298.714, "MEAN_ANOMALY": 61.2673, "BSTAR": 0.0, "EPOCH": "2026-03-13T14:23:31"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-1 (C06)", "NORAD_CAT_ID": 36828, "MEAN_MOTION": 1.00250201, "ECCENTRICITY": 0.0054469, "INCLINATION": 54.2923, "RA_OF_ASC_NODE": 163.8253, "ARG_OF_PERICENTER": 219.3556, "MEAN_ANOMALY": 215.6821, "BSTAR": 0.0, "EPOCH": "2026-03-13T21:32:09"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-2 (C07)", "NORAD_CAT_ID": 37256, "MEAN_MOTION": 1.00257833, "ECCENTRICITY": 0.004749, "INCLINATION": 47.7215, "RA_OF_ASC_NODE": 273.0501, "ARG_OF_PERICENTER": 213.2827, "MEAN_ANOMALY": 326.5355, "BSTAR": 0.0, "EPOCH": "2026-03-10T11:51:55"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-3 (C08)", "NORAD_CAT_ID": 37384, "MEAN_MOTION": 1.00293315, "ECCENTRICITY": 0.0035062, "INCLINATION": 62.2726, "RA_OF_ASC_NODE": 41.1265, "ARG_OF_PERICENTER": 190.7769, "MEAN_ANOMALY": 297.326, "BSTAR": 0.0, "EPOCH": "2026-03-05T17:18:14"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-4 (C09)", "NORAD_CAT_ID": 37763, "MEAN_MOTION": 1.00264423, "ECCENTRICITY": 0.0155085, "INCLINATION": 54.5894, "RA_OF_ASC_NODE": 166.5549, "ARG_OF_PERICENTER": 231.062, "MEAN_ANOMALY": 149.1617, "BSTAR": 0.0, "EPOCH": "2026-03-11T18:52:33"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-5 (C10)", "NORAD_CAT_ID": 37948, "MEAN_MOTION": 1.00270695, "ECCENTRICITY": 0.0108864, "INCLINATION": 47.8524, "RA_OF_ASC_NODE": 272.7266, "ARG_OF_PERICENTER": 221.072, "MEAN_ANOMALY": 318.7247, "BSTAR": 0.0, "EPOCH": "2026-03-13T12:25:15"}, {"OBJECT_NAME": "BEIDOU-3S IGSO-2S (C56)", "NORAD_CAT_ID": 40938, "MEAN_MOTION": 1.00254296, "ECCENTRICITY": 0.0061515, "INCLINATION": 49.4373, "RA_OF_ASC_NODE": 260.4556, "ARG_OF_PERICENTER": 187.1638, "MEAN_ANOMALY": 17.4903, "BSTAR": 0.0, "EPOCH": "2026-01-27T16:14:57"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-6 (C13)", "NORAD_CAT_ID": 41434, "MEAN_MOTION": 1.00277411, "ECCENTRICITY": 0.0059566, "INCLINATION": 60.0969, "RA_OF_ASC_NODE": 39.005, "ARG_OF_PERICENTER": 233.2714, "MEAN_ANOMALY": 308.7278, "BSTAR": 0.0, "EPOCH": "2026-03-11T21:02:52"}, {"OBJECT_NAME": "BEIDOU-3 IGSO-1 (C38)", "NORAD_CAT_ID": 44204, "MEAN_MOTION": 1.00268377, "ECCENTRICITY": 0.0025635, "INCLINATION": 58.5672, "RA_OF_ASC_NODE": 38.6298, "ARG_OF_PERICENTER": 236.0438, "MEAN_ANOMALY": 335.2844, "BSTAR": 0.0, "EPOCH": "2026-03-13T21:18:00"}, {"OBJECT_NAME": "COSMO-SKYMED 1", "NORAD_CAT_ID": 31598, "MEAN_MOTION": 14.96574619, "ECCENTRICITY": 0.0001578, "INCLINATION": 97.8882, "RA_OF_ASC_NODE": 264.0096, "ARG_OF_PERICENTER": 95.4766, "MEAN_ANOMALY": 264.6635, "BSTAR": 0.00042908000000000003, "EPOCH": "2026-03-13T22:38:48"}, {"OBJECT_NAME": "COSMO-SKYMED 2", "NORAD_CAT_ID": 32376, "MEAN_MOTION": 14.8214924, "ECCENTRICITY": 0.0001327, "INCLINATION": 97.8868, "RA_OF_ASC_NODE": 257.9453, "ARG_OF_PERICENTER": 83.9144, "MEAN_ANOMALY": 276.2221, "BSTAR": 8.912600000000001e-05, "EPOCH": "2026-03-13T23:18:30"}, {"OBJECT_NAME": "COSMO-SKYMED 4", "NORAD_CAT_ID": 37216, "MEAN_MOTION": 14.82144204, "ECCENTRICITY": 0.0001495, "INCLINATION": 97.8871, "RA_OF_ASC_NODE": 257.9577, "ARG_OF_PERICENTER": 83.0447, "MEAN_ANOMALY": 277.0936, "BSTAR": 6.1181e-05, "EPOCH": "2026-03-13T23:36:43"}, {"OBJECT_NAME": "COSMO-SKYMED 3", "NORAD_CAT_ID": 33412, "MEAN_MOTION": 15.06287447, "ECCENTRICITY": 0.0014664, "INCLINATION": 97.8416, "RA_OF_ASC_NODE": 289.9339, "ARG_OF_PERICENTER": 225.8003, "MEAN_ANOMALY": 134.202, "BSTAR": 0.00029846, "EPOCH": "2026-03-13T23:18:31"}, {"OBJECT_NAME": "NAVSTAR 66 (USA 232)", "NORAD_CAT_ID": 37753, "MEAN_MOTION": 2.00563331, "ECCENTRICITY": 0.0140225, "INCLINATION": 56.6074, "RA_OF_ASC_NODE": 335.748, "ARG_OF_PERICENTER": 60.959, "MEAN_ANOMALY": 141.4014, "BSTAR": 0.0, "EPOCH": "2026-03-13T10:01:31"}, {"OBJECT_NAME": "NAVSTAR 46 (USA 145)", "NORAD_CAT_ID": 25933, "MEAN_MOTION": 2.00567343, "ECCENTRICITY": 0.0106389, "INCLINATION": 51.5489, "RA_OF_ASC_NODE": 299.64, "ARG_OF_PERICENTER": 171.929, "MEAN_ANOMALY": 9.148, "BSTAR": 0.0, "EPOCH": "2026-03-13T19:41:18"}, {"OBJECT_NAME": "NAVSTAR 49 (USA 154)", "NORAD_CAT_ID": 26605, "MEAN_MOTION": 2.00570193, "ECCENTRICITY": 0.0173455, "INCLINATION": 55.5687, "RA_OF_ASC_NODE": 98.7251, "ARG_OF_PERICENTER": 265.6826, "MEAN_ANOMALY": 103.4412, "BSTAR": 0.0, "EPOCH": "2026-03-13T16:25:19"}, {"OBJECT_NAME": "NAVSTAR 52 (USA 168)", "NORAD_CAT_ID": 27704, "MEAN_MOTION": 1.94743161, "ECCENTRICITY": 0.0004627, "INCLINATION": 54.8698, "RA_OF_ASC_NODE": 328.8449, "ARG_OF_PERICENTER": 343.01, "MEAN_ANOMALY": 204.2637, "BSTAR": 0.0, "EPOCH": "2026-03-11T13:01:38"}, {"OBJECT_NAME": "NAVSTAR 53 (USA 175)", "NORAD_CAT_ID": 28129, "MEAN_MOTION": 1.92678601, "ECCENTRICITY": 0.000316, "INCLINATION": 54.9721, "RA_OF_ASC_NODE": 26.8461, "ARG_OF_PERICENTER": 314.5189, "MEAN_ANOMALY": 45.4493, "BSTAR": 0.0, "EPOCH": "2026-03-12T12:24:25"}, {"OBJECT_NAME": "NAVSTAR 55 (USA 178)", "NORAD_CAT_ID": 28361, "MEAN_MOTION": 2.00552384, "ECCENTRICITY": 0.0159871, "INCLINATION": 54.7964, "RA_OF_ASC_NODE": 90.5792, "ARG_OF_PERICENTER": 298.9108, "MEAN_ANOMALY": 53.8796, "BSTAR": 0.0, "EPOCH": "2026-03-13T09:35:38"}, {"OBJECT_NAME": "NAVSTAR 63 (USA 203)", "NORAD_CAT_ID": 34661, "MEAN_MOTION": 2.00555628, "ECCENTRICITY": 0.0144936, "INCLINATION": 54.5029, "RA_OF_ASC_NODE": 214.4867, "ARG_OF_PERICENTER": 61.4833, "MEAN_ANOMALY": 120.7701, "BSTAR": 0.0, "EPOCH": "2026-03-13T13:47:46"}, {"OBJECT_NAME": "DMSP 5D-3 F18 (USA 210)", "NORAD_CAT_ID": 35951, "MEAN_MOTION": 14.14838953, "ECCENTRICITY": 0.0011368, "INCLINATION": 98.8968, "RA_OF_ASC_NODE": 53.9706, "ARG_OF_PERICENTER": 156.3337, "MEAN_ANOMALY": 203.8361, "BSTAR": 0.00018149000000000003, "EPOCH": "2026-03-13T22:55:56"}, {"OBJECT_NAME": "AEHF-1 (USA 214)", "NORAD_CAT_ID": 36868, "MEAN_MOTION": 1.00269989, "ECCENTRICITY": 0.0004323, "INCLINATION": 7.9828, "RA_OF_ASC_NODE": 69.2901, "ARG_OF_PERICENTER": 282.4183, "MEAN_ANOMALY": 82.5563, "BSTAR": 0.0, "EPOCH": "2026-03-13T15:32:11"}, {"OBJECT_NAME": "AEHF-2 (USA 235)", "NORAD_CAT_ID": 38254, "MEAN_MOTION": 1.0027274, "ECCENTRICITY": 0.0003126, "INCLINATION": 6.4295, "RA_OF_ASC_NODE": 58.1783, "ARG_OF_PERICENTER": 283.2868, "MEAN_ANOMALY": 94.5435, "BSTAR": 0.0, "EPOCH": "2026-03-13T18:44:20"}, {"OBJECT_NAME": "AEHF-5 (USA 292)", "NORAD_CAT_ID": 44481, "MEAN_MOTION": 1.00269143, "ECCENTRICITY": 0.0071181, "INCLINATION": 1.5525, "RA_OF_ASC_NODE": 329.312, "ARG_OF_PERICENTER": 3.3698, "MEAN_ANOMALY": 272.1078, "BSTAR": 0.0, "EPOCH": "2026-03-13T22:40:43"}, {"OBJECT_NAME": "AEHF-4 (USA 288)", "NORAD_CAT_ID": 43651, "MEAN_MOTION": 1.00270052, "ECCENTRICITY": 0.0063991, "INCLINATION": 1.422, "RA_OF_ASC_NODE": 355.4529, "ARG_OF_PERICENTER": 359.1371, "MEAN_ANOMALY": 78.2689, "BSTAR": 0.0, "EPOCH": "2026-03-13T21:55:24"}, {"OBJECT_NAME": "WGS F7 (USA 263)", "NORAD_CAT_ID": 40746, "MEAN_MOTION": 1.00272244, "ECCENTRICITY": 2.25e-05, "INCLINATION": 0.0131, "RA_OF_ASC_NODE": 135.051, "ARG_OF_PERICENTER": 270.5458, "MEAN_ANOMALY": 158.3684, "BSTAR": 0.0, "EPOCH": "2026-03-13T14:30:54"}, {"OBJECT_NAME": "SBSS (USA 216)", "NORAD_CAT_ID": 37168, "MEAN_MOTION": 15.15755191, "ECCENTRICITY": 0.0082509, "INCLINATION": 97.7475, "RA_OF_ASC_NODE": 315.5523, "ARG_OF_PERICENTER": 174.1816, "MEAN_ANOMALY": 311.3892, "BSTAR": 0.00025053, "EPOCH": "2026-03-12T23:42:37"}, {"OBJECT_NAME": "USA 81", "NORAD_CAT_ID": 21949, "MEAN_MOTION": 14.32369362, "ECCENTRICITY": 0.0002353, "INCLINATION": 85.0076, "RA_OF_ASC_NODE": 129.041, "ARG_OF_PERICENTER": 74.3719, "MEAN_ANOMALY": 285.7733, "BSTAR": 5.7183e-05, "EPOCH": "2026-03-13T20:24:23"}, {"OBJECT_NAME": "NUSAT-8 (MARIE)", "NORAD_CAT_ID": 45018, "MEAN_MOTION": 16.42177442, "ECCENTRICITY": 0.0011737, "INCLINATION": 97.1425, "RA_OF_ASC_NODE": 344.4413, "ARG_OF_PERICENTER": 268.3663, "MEAN_ANOMALY": 91.6293, "BSTAR": 0.00041399000000000004, "EPOCH": "2023-09-30T17:45:37"}, {"OBJECT_NAME": "NEUSAR", "NORAD_CAT_ID": 52937, "MEAN_MOTION": 15.07997321, "ECCENTRICITY": 0.0001716, "INCLINATION": 9.9786, "RA_OF_ASC_NODE": 24.8027, "ARG_OF_PERICENTER": 85.5833, "MEAN_ANOMALY": 274.4578, "BSTAR": 0.00069195, "EPOCH": "2023-12-28T07:08:57"}, {"OBJECT_NAME": "CINEMA-3 (KHUSAT-2)", "NORAD_CAT_ID": 39426, "MEAN_MOTION": 14.78011842, "ECCENTRICITY": 0.0090675, "INCLINATION": 97.8433, "RA_OF_ASC_NODE": 346.9945, "ARG_OF_PERICENTER": 162.1567, "MEAN_ANOMALY": 198.2864, "BSTAR": 0.00021754, "EPOCH": "2026-03-13T06:57:51"}, {"OBJECT_NAME": "NUSAT-6 (HYPATIA)", "NORAD_CAT_ID": 46272, "MEAN_MOTION": 16.40036993, "ECCENTRICITY": 0.0013029, "INCLINATION": 97.2026, "RA_OF_ASC_NODE": 336.2175, "ARG_OF_PERICENTER": 264.7707, "MEAN_ANOMALY": 95.2106, "BSTAR": 0.00055612, "EPOCH": "2024-09-13T16:58:59"}, {"OBJECT_NAME": "DTUSAT-2", "NORAD_CAT_ID": 40030, "MEAN_MOTION": 15.13620769, "ECCENTRICITY": 0.0007591, "INCLINATION": 98.085, "RA_OF_ASC_NODE": 64.7507, "ARG_OF_PERICENTER": 185.8583, "MEAN_ANOMALY": 174.2559, "BSTAR": 0.0005200399999999999, "EPOCH": "2026-03-13T22:57:22"}, {"OBJECT_NAME": "NUSAT-12 (DOROTHY)", "NORAD_CAT_ID": 46827, "MEAN_MOTION": 16.36178297, "ECCENTRICITY": 0.0011945, "INCLINATION": 97.1044, "RA_OF_ASC_NODE": 344.9336, "ARG_OF_PERICENTER": 264.565, "MEAN_ANOMALY": 95.4286, "BSTAR": 0.0008634300000000001, "EPOCH": "2023-09-26T20:09:18"}, {"OBJECT_NAME": "NUSAT-4 (ADA)", "NORAD_CAT_ID": 43195, "MEAN_MOTION": 16.38072031, "ECCENTRICITY": 0.0007046, "INCLINATION": 97.5559, "RA_OF_ASC_NODE": 181.2615, "ARG_OF_PERICENTER": 278.8217, "MEAN_ANOMALY": 81.2283, "BSTAR": 0.00073412, "EPOCH": "2024-01-04T03:41:36"}, {"OBJECT_NAME": "NUSAT-9 (ALICE)", "NORAD_CAT_ID": 46828, "MEAN_MOTION": 16.33943729, "ECCENTRICITY": 0.001099, "INCLINATION": 97.0919, "RA_OF_ASC_NODE": 350.4269, "ARG_OF_PERICENTER": 266.8171, "MEAN_ANOMALY": 93.1869, "BSTAR": 0.0009418100000000001, "EPOCH": "2023-10-02T20:48:12"}, {"OBJECT_NAME": "NUSAT-5 (MARYAM)", "NORAD_CAT_ID": 43204, "MEAN_MOTION": 16.4605451, "ECCENTRICITY": 0.0008666, "INCLINATION": 97.5583, "RA_OF_ASC_NODE": 170.606, "ARG_OF_PERICENTER": 279.6992, "MEAN_ANOMALY": 80.3331, "BSTAR": 0.00028268, "EPOCH": "2023-12-24T01:49:10"}, {"OBJECT_NAME": "CHUBUSAT-2", "NORAD_CAT_ID": 41338, "MEAN_MOTION": 15.19817765, "ECCENTRICITY": 0.0008656, "INCLINATION": 30.9967, "RA_OF_ASC_NODE": 226.7504, "ARG_OF_PERICENTER": 182.5168, "MEAN_ANOMALY": 177.543, "BSTAR": 0.00035058, "EPOCH": "2026-03-13T19:40:59"}, {"OBJECT_NAME": "O/OREOS (USA 219)", "NORAD_CAT_ID": 37224, "MEAN_MOTION": 14.93303315, "ECCENTRICITY": 0.0016748, "INCLINATION": 71.9708, "RA_OF_ASC_NODE": 296.7525, "ARG_OF_PERICENTER": 250.1651, "MEAN_ANOMALY": 109.7714, "BSTAR": 0.00026673999999999996, "EPOCH": "2026-03-13T14:48:38"}, {"OBJECT_NAME": "SPOT 5", "NORAD_CAT_ID": 27421, "MEAN_MOTION": 14.54665613, "ECCENTRICITY": 0.0129098, "INCLINATION": 97.9979, "RA_OF_ASC_NODE": 116.0092, "ARG_OF_PERICENTER": 212.1013, "MEAN_ANOMALY": 147.2278, "BSTAR": 0.00010395, "EPOCH": "2026-03-13T23:24:59"}, {"OBJECT_NAME": "SPOT 6", "NORAD_CAT_ID": 38755, "MEAN_MOTION": 14.58545193, "ECCENTRICITY": 0.0001519, "INCLINATION": 98.2212, "RA_OF_ASC_NODE": 141.2962, "ARG_OF_PERICENTER": 92.5216, "MEAN_ANOMALY": 267.6158, "BSTAR": 6.4742e-05, "EPOCH": "2026-03-13T21:49:58"}, {"OBJECT_NAME": "SPOT 7", "NORAD_CAT_ID": 40053, "MEAN_MOTION": 14.60866373, "ECCENTRICITY": 0.0001432, "INCLINATION": 98.0737, "RA_OF_ASC_NODE": 137.8192, "ARG_OF_PERICENTER": 99.1404, "MEAN_ANOMALY": 260.9959, "BSTAR": 9.7991e-05, "EPOCH": "2026-03-13T21:51:07"}, {"OBJECT_NAME": "GLONASS125 [COD]", "NORAD_CAT_ID": 37372, "MEAN_MOTION": 2.13104226, "ECCENTRICITY": 0.000871, "INCLINATION": 64.8285, "RA_OF_ASC_NODE": 104.8188, "ARG_OF_PERICENTER": 293.3373, "MEAN_ANOMALY": 12.0047, "BSTAR": 0.0, "EPOCH": "2023-11-08T23:59:42"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D48", "NORAD_CAT_ID": 54695, "MEAN_MOTION": 15.33068034, "ECCENTRICITY": 0.0007644, "INCLINATION": 97.6778, "RA_OF_ASC_NODE": 238.63, "ARG_OF_PERICENTER": 99.1246, "MEAN_ANOMALY": 261.086, "BSTAR": 0.00044999, "EPOCH": "2026-03-13T23:22:13"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D28", "NORAD_CAT_ID": 52445, "MEAN_MOTION": 15.60632762, "ECCENTRICITY": 0.0002663, "INCLINATION": 97.5574, "RA_OF_ASC_NODE": 179.5289, "ARG_OF_PERICENTER": 66.9948, "MEAN_ANOMALY": 293.159, "BSTAR": 0.00097274, "EPOCH": "2026-03-13T21:49:31"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D14", "NORAD_CAT_ID": 51831, "MEAN_MOTION": 15.81049188, "ECCENTRICITY": 0.0003043, "INCLINATION": 97.3688, "RA_OF_ASC_NODE": 164.6872, "ARG_OF_PERICENTER": 152.8576, "MEAN_ANOMALY": 207.2851, "BSTAR": 0.0011202, "EPOCH": "2026-03-12T06:18:07"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D16", "NORAD_CAT_ID": 51834, "MEAN_MOTION": 15.89787455, "ECCENTRICITY": 0.0002752, "INCLINATION": 97.3683, "RA_OF_ASC_NODE": 167.8466, "ARG_OF_PERICENTER": 210.2494, "MEAN_ANOMALY": 149.862, "BSTAR": 0.0012409, "EPOCH": "2026-03-13T22:19:50"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3C", "NORAD_CAT_ID": 46455, "MEAN_MOTION": 16.4332105, "ECCENTRICITY": 0.0008886, "INCLINATION": 97.224, "RA_OF_ASC_NODE": 102.405, "ARG_OF_PERICENTER": 228.0638, "MEAN_ANOMALY": 131.9907, "BSTAR": 0.00043027, "EPOCH": "2026-02-10T19:53:25"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D11", "NORAD_CAT_ID": 51835, "MEAN_MOTION": 15.78195757, "ECCENTRICITY": 0.0003472, "INCLINATION": 97.3721, "RA_OF_ASC_NODE": 164.6089, "ARG_OF_PERICENTER": 168.3789, "MEAN_ANOMALY": 191.7558, "BSTAR": 0.0010862, "EPOCH": "2026-03-12T05:09:01"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D", "NORAD_CAT_ID": 46456, "MEAN_MOTION": 15.21343265, "ECCENTRICITY": 0.0011568, "INCLINATION": 97.3728, "RA_OF_ASC_NODE": 46.7737, "ARG_OF_PERICENTER": 112.6941, "MEAN_ANOMALY": 247.5519, "BSTAR": 0.00055382, "EPOCH": "2023-12-28T11:38:25"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3E", "NORAD_CAT_ID": 46457, "MEAN_MOTION": 15.54182685, "ECCENTRICITY": 0.0015199, "INCLINATION": 97.2426, "RA_OF_ASC_NODE": 123.9773, "ARG_OF_PERICENTER": 150.3093, "MEAN_ANOMALY": 209.9027, "BSTAR": 0.000728, "EPOCH": "2026-03-13T22:48:07"}, {"OBJECT_NAME": "JILIN-1 03", "NORAD_CAT_ID": 41914, "MEAN_MOTION": 15.39113047, "ECCENTRICITY": 0.0007394, "INCLINATION": 97.1997, "RA_OF_ASC_NODE": 81.5648, "ARG_OF_PERICENTER": 140.8128, "MEAN_ANOMALY": 219.3654, "BSTAR": 0.00040647000000000003, "EPOCH": "2026-03-13T23:42:15"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D32", "NORAD_CAT_ID": 52449, "MEAN_MOTION": 15.2053231, "ECCENTRICITY": 0.0011601, "INCLINATION": 97.6185, "RA_OF_ASC_NODE": 74.9126, "ARG_OF_PERICENTER": 14.3554, "MEAN_ANOMALY": 345.8008, "BSTAR": 0.00056673, "EPOCH": "2023-12-28T10:45:20"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D15", "NORAD_CAT_ID": 51839, "MEAN_MOTION": 15.70184009, "ECCENTRICITY": 0.0003813, "INCLINATION": 97.3687, "RA_OF_ASC_NODE": 164.3937, "ARG_OF_PERICENTER": 164.698, "MEAN_ANOMALY": 195.4398, "BSTAR": 0.0010919, "EPOCH": "2026-03-13T22:49:37"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3F", "NORAD_CAT_ID": 46458, "MEAN_MOTION": 15.52566429, "ECCENTRICITY": 0.0016742, "INCLINATION": 97.2381, "RA_OF_ASC_NODE": 121.635, "ARG_OF_PERICENTER": 139.6856, "MEAN_ANOMALY": 220.5642, "BSTAR": 0.00070215, "EPOCH": "2026-03-13T21:52:55"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3G", "NORAD_CAT_ID": 46459, "MEAN_MOTION": 15.58394288, "ECCENTRICITY": 0.0012688, "INCLINATION": 97.2458, "RA_OF_ASC_NODE": 127.2844, "ARG_OF_PERICENTER": 52.8372, "MEAN_ANOMALY": 307.4041, "BSTAR": 0.00076823, "EPOCH": "2026-03-13T22:57:05"}, {"OBJECT_NAME": "JILIN-1 04", "NORAD_CAT_ID": 43022, "MEAN_MOTION": 15.39855409, "ECCENTRICITY": 0.0004764, "INCLINATION": 97.4764, "RA_OF_ASC_NODE": 180.2825, "ARG_OF_PERICENTER": 164.4367, "MEAN_ANOMALY": 195.7026, "BSTAR": 0.00043830000000000003, "EPOCH": "2026-03-13T23:15:06"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D33", "NORAD_CAT_ID": 52450, "MEAN_MOTION": 15.20833127, "ECCENTRICITY": 0.0013609, "INCLINATION": 97.6181, "RA_OF_ASC_NODE": 74.9956, "ARG_OF_PERICENTER": 10.9961, "MEAN_ANOMALY": 349.1568, "BSTAR": 0.00056472, "EPOCH": "2023-12-28T10:56:13"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3H", "NORAD_CAT_ID": 46460, "MEAN_MOTION": 15.56778448, "ECCENTRICITY": 0.0006484, "INCLINATION": 97.2402, "RA_OF_ASC_NODE": 123.983, "ARG_OF_PERICENTER": 347.5654, "MEAN_ANOMALY": 12.5441, "BSTAR": 0.0007107900000000001, "EPOCH": "2026-03-13T21:54:49"}, {"OBJECT_NAME": "JILIN-1 05", "NORAD_CAT_ID": 43023, "MEAN_MOTION": 15.32516093, "ECCENTRICITY": 0.0004951, "INCLINATION": 97.4781, "RA_OF_ASC_NODE": 177.9842, "ARG_OF_PERICENTER": 190.8207, "MEAN_ANOMALY": 169.2929, "BSTAR": 0.00024848000000000003, "EPOCH": "2026-03-13T23:06:06"}, {"OBJECT_NAME": "SBIRS GEO-3 (USA 282)", "NORAD_CAT_ID": 43162, "MEAN_MOTION": 1.00272787, "ECCENTRICITY": 0.0002207, "INCLINATION": 2.1264, "RA_OF_ASC_NODE": 7.356, "ARG_OF_PERICENTER": 350.0556, "MEAN_ANOMALY": 311.1915, "BSTAR": 0.0, "EPOCH": "2026-03-13T19:46:14"}, {"OBJECT_NAME": "SBIRS GEO-4 (USA 273)", "NORAD_CAT_ID": 41937, "MEAN_MOTION": 1.00272716, "ECCENTRICITY": 0.0002291, "INCLINATION": 2.0648, "RA_OF_ASC_NODE": 40.3594, "ARG_OF_PERICENTER": 316.9988, "MEAN_ANOMALY": 136.2516, "BSTAR": 0.0, "EPOCH": "2026-03-13T20:05:01"}, {"OBJECT_NAME": "SBIRS GEO-5 (USA 315)", "NORAD_CAT_ID": 48618, "MEAN_MOTION": 1.00271905, "ECCENTRICITY": 0.0001187, "INCLINATION": 5.2257, "RA_OF_ASC_NODE": 328.3918, "ARG_OF_PERICENTER": 28.3379, "MEAN_ANOMALY": 212.5906, "BSTAR": 0.0, "EPOCH": "2026-03-13T20:16:17"}, {"OBJECT_NAME": "SBIRS GEO-6 (USA 336)", "NORAD_CAT_ID": 53355, "MEAN_MOTION": 1.00271656, "ECCENTRICITY": 0.0002092, "INCLINATION": 3.4222, "RA_OF_ASC_NODE": 315.954, "ARG_OF_PERICENTER": 39.3161, "MEAN_ANOMALY": 298.8075, "BSTAR": 0.0, "EPOCH": "2026-03-13T22:54:04"}, {"OBJECT_NAME": "SBIRS GEO-1 (USA 230)", "NORAD_CAT_ID": 37481, "MEAN_MOTION": 1.00272453, "ECCENTRICITY": 0.0002318, "INCLINATION": 4.3256, "RA_OF_ASC_NODE": 53.0987, "ARG_OF_PERICENTER": 296.7572, "MEAN_ANOMALY": 287.6177, "BSTAR": 0.0, "EPOCH": "2026-03-13T03:39:35"}, {"OBJECT_NAME": "SBIRS GEO-2 (USA 241)", "NORAD_CAT_ID": 39120, "MEAN_MOTION": 1.00271486, "ECCENTRICITY": 0.000228, "INCLINATION": 4.2817, "RA_OF_ASC_NODE": 52.3935, "ARG_OF_PERICENTER": 297.8022, "MEAN_ANOMALY": 281.3772, "BSTAR": 0.0, "EPOCH": "2026-03-13T10:01:21"}, {"OBJECT_NAME": "SKYSAT-C17", "NORAD_CAT_ID": 46179, "MEAN_MOTION": 15.7911991, "ECCENTRICITY": 0.0006126, "INCLINATION": 52.97, "RA_OF_ASC_NODE": 214.8332, "ARG_OF_PERICENTER": 89.9494, "MEAN_ANOMALY": 270.2227, "BSTAR": 0.00064037, "EPOCH": "2023-12-27T20:37:45"}, {"OBJECT_NAME": "SKYSAT-C18", "NORAD_CAT_ID": 46180, "MEAN_MOTION": 16.35747614, "ECCENTRICITY": 0.0017318, "INCLINATION": 52.9561, "RA_OF_ASC_NODE": 11.8431, "ARG_OF_PERICENTER": 265.4879, "MEAN_ANOMALY": 104.1762, "BSTAR": 0.0015942999999999999, "EPOCH": "2023-06-25T20:09:47"}, {"OBJECT_NAME": "SKYSAT-C19", "NORAD_CAT_ID": 46235, "MEAN_MOTION": 15.89514628, "ECCENTRICITY": 0.0002149, "INCLINATION": 52.9675, "RA_OF_ASC_NODE": 201.9813, "ARG_OF_PERICENTER": 132.3648, "MEAN_ANOMALY": 227.7558, "BSTAR": 0.0007014899999999999, "EPOCH": "2023-12-27T23:17:19"}, {"OBJECT_NAME": "SKYSAT-C14", "NORAD_CAT_ID": 45788, "MEAN_MOTION": 15.53833173, "ECCENTRICITY": 0.001113, "INCLINATION": 52.9777, "RA_OF_ASC_NODE": 149.837, "ARG_OF_PERICENTER": 213.0476, "MEAN_ANOMALY": 146.9838, "BSTAR": 0.00062044, "EPOCH": "2023-12-27T15:51:00"}, {"OBJECT_NAME": "SKYSAT-C16", "NORAD_CAT_ID": 45789, "MEAN_MOTION": 15.53303284, "ECCENTRICITY": 0.000551, "INCLINATION": 52.9788, "RA_OF_ASC_NODE": 133.7921, "ARG_OF_PERICENTER": 2.6753, "MEAN_ANOMALY": 357.4284, "BSTAR": 0.0005727, "EPOCH": "2023-12-28T12:24:10"}, {"OBJECT_NAME": "SKYSAT-C15", "NORAD_CAT_ID": 45790, "MEAN_MOTION": 15.47687342, "ECCENTRICITY": 0.0003931, "INCLINATION": 52.9782, "RA_OF_ASC_NODE": 154.0912, "ARG_OF_PERICENTER": 180.8503, "MEAN_ANOMALY": 179.2497, "BSTAR": 0.00058075, "EPOCH": "2023-12-27T15:49:43"}, {"OBJECT_NAME": "SKYSAT-C9", "NORAD_CAT_ID": 42989, "MEAN_MOTION": 15.39018182, "ECCENTRICITY": 0.0009866, "INCLINATION": 97.4262, "RA_OF_ASC_NODE": 208.551, "ARG_OF_PERICENTER": 120.4629, "MEAN_ANOMALY": 239.7592, "BSTAR": 0.00040079, "EPOCH": "2026-03-13T08:30:09"}, {"OBJECT_NAME": "SKYSAT-C5", "NORAD_CAT_ID": 41772, "MEAN_MOTION": 15.3435329, "ECCENTRICITY": 0.0001665, "INCLINATION": 97.0615, "RA_OF_ASC_NODE": 116.5685, "ARG_OF_PERICENTER": 86.9951, "MEAN_ANOMALY": 273.1483, "BSTAR": 0.00028166000000000004, "EPOCH": "2026-03-12T15:07:05"}, {"OBJECT_NAME": "SKYSAT-C2", "NORAD_CAT_ID": 41773, "MEAN_MOTION": 15.398215, "ECCENTRICITY": 0.0004736, "INCLINATION": 97.0279, "RA_OF_ASC_NODE": 98.5898, "ARG_OF_PERICENTER": 59.4654, "MEAN_ANOMALY": 300.706, "BSTAR": 0.00036845, "EPOCH": "2026-03-13T19:15:38"}, {"OBJECT_NAME": "SKYSAT-C3", "NORAD_CAT_ID": 41774, "MEAN_MOTION": 15.48522832, "ECCENTRICITY": 0.0001744, "INCLINATION": 96.9178, "RA_OF_ASC_NODE": 107.5203, "ARG_OF_PERICENTER": 112.5905, "MEAN_ANOMALY": 247.5532, "BSTAR": 0.00047518, "EPOCH": "2026-03-13T19:45:36"}, {"OBJECT_NAME": "SKYSAT-C11", "NORAD_CAT_ID": 42987, "MEAN_MOTION": 15.52704181, "ECCENTRICITY": 4.71e-05, "INCLINATION": 97.4206, "RA_OF_ASC_NODE": 218.0313, "ARG_OF_PERICENTER": 77.4255, "MEAN_ANOMALY": 282.705, "BSTAR": 0.00054886, "EPOCH": "2026-03-13T21:44:48"}, {"OBJECT_NAME": "SKYSAT-C10", "NORAD_CAT_ID": 42988, "MEAN_MOTION": 15.32791548, "ECCENTRICITY": 9.97e-05, "INCLINATION": 97.4337, "RA_OF_ASC_NODE": 209.0417, "ARG_OF_PERICENTER": 94.2944, "MEAN_ANOMALY": 265.8412, "BSTAR": 0.00041338000000000003, "EPOCH": "2026-03-13T21:50:28"}, {"OBJECT_NAME": "SKYSAT-C8", "NORAD_CAT_ID": 42990, "MEAN_MOTION": 15.34894571, "ECCENTRICITY": 0.0001228, "INCLINATION": 97.4354, "RA_OF_ASC_NODE": 208.539, "ARG_OF_PERICENTER": 95.1778, "MEAN_ANOMALY": 264.9605, "BSTAR": 0.0004103, "EPOCH": "2026-03-13T22:53:35"}, {"OBJECT_NAME": "SKYSAT-C7", "NORAD_CAT_ID": 42991, "MEAN_MOTION": 15.3484588, "ECCENTRICITY": 0.0007304, "INCLINATION": 97.4366, "RA_OF_ASC_NODE": 209.5123, "ARG_OF_PERICENTER": 90.9557, "MEAN_ANOMALY": 269.2523, "BSTAR": 0.00042582, "EPOCH": "2026-03-13T22:39:31"}, {"OBJECT_NAME": "SKYSAT-C6", "NORAD_CAT_ID": 42992, "MEAN_MOTION": 15.32609953, "ECCENTRICITY": 0.0003737, "INCLINATION": 97.44, "RA_OF_ASC_NODE": 209.7875, "ARG_OF_PERICENTER": 117.9663, "MEAN_ANOMALY": 242.1957, "BSTAR": 0.00040575, "EPOCH": "2026-03-13T22:43:08"}, {"OBJECT_NAME": "SKYSAT-C12", "NORAD_CAT_ID": 43797, "MEAN_MOTION": 15.45770367, "ECCENTRICITY": 0.0005874, "INCLINATION": 96.9381, "RA_OF_ASC_NODE": 100.1924, "ARG_OF_PERICENTER": 126.9428, "MEAN_ANOMALY": 233.2361, "BSTAR": 0.00040636000000000003, "EPOCH": "2026-03-13T23:11:55"}, {"OBJECT_NAME": "SKYSAT-C13", "NORAD_CAT_ID": 43802, "MEAN_MOTION": 15.77539348, "ECCENTRICITY": 0.0004556, "INCLINATION": 96.9399, "RA_OF_ASC_NODE": 120.5628, "ARG_OF_PERICENTER": 185.5887, "MEAN_ANOMALY": 174.533, "BSTAR": 0.0006055899999999999, "EPOCH": "2026-03-13T21:59:14"}, {"OBJECT_NAME": "SKYSAT-A", "NORAD_CAT_ID": 39418, "MEAN_MOTION": 15.12387974, "ECCENTRICITY": 0.0019985, "INCLINATION": 97.395, "RA_OF_ASC_NODE": 125.2838, "ARG_OF_PERICENTER": 284.9639, "MEAN_ANOMALY": 74.938, "BSTAR": 0.00021779000000000001, "EPOCH": "2026-03-13T20:52:44"}, {"OBJECT_NAME": "SKYSAT-B", "NORAD_CAT_ID": 40072, "MEAN_MOTION": 14.8774521, "ECCENTRICITY": 0.000457, "INCLINATION": 98.3728, "RA_OF_ASC_NODE": 26.3696, "ARG_OF_PERICENTER": 217.1835, "MEAN_ANOMALY": 142.9064, "BSTAR": 0.00023938, "EPOCH": "2026-03-13T18:21:32"}, {"OBJECT_NAME": "SKYSAT-C1", "NORAD_CAT_ID": 41601, "MEAN_MOTION": 15.34599505, "ECCENTRICITY": 0.0003252, "INCLINATION": 96.9701, "RA_OF_ASC_NODE": 109.504, "ARG_OF_PERICENTER": 45.2329, "MEAN_ANOMALY": 314.9179, "BSTAR": 0.00033023000000000004, "EPOCH": "2026-03-13T13:09:19"}, {"OBJECT_NAME": "PLEIADES NEO 3", "NORAD_CAT_ID": 48268, "MEAN_MOTION": 14.81671937, "ECCENTRICITY": 0.000128, "INCLINATION": 97.893, "RA_OF_ASC_NODE": 149.098, "ARG_OF_PERICENTER": 94.2351, "MEAN_ANOMALY": 265.9009, "BSTAR": 2.3185000000000002e-05, "EPOCH": "2026-03-13T23:09:22"}, {"OBJECT_NAME": "PLEIADES NEO 4", "NORAD_CAT_ID": 49070, "MEAN_MOTION": 14.81671436, "ECCENTRICITY": 0.0001336, "INCLINATION": 97.893, "RA_OF_ASC_NODE": 149.0607, "ARG_OF_PERICENTER": 92.4826, "MEAN_ANOMALY": 267.654, "BSTAR": -7.0122e-05, "EPOCH": "2026-03-13T22:20:43"}, {"OBJECT_NAME": "PLEIADES 1B", "NORAD_CAT_ID": 39019, "MEAN_MOTION": 14.58530816, "ECCENTRICITY": 0.0001594, "INCLINATION": 98.2006, "RA_OF_ASC_NODE": 149.2705, "ARG_OF_PERICENTER": 84.011, "MEAN_ANOMALY": 276.127, "BSTAR": 7.3644e-05, "EPOCH": "2026-03-13T22:17:39"}, {"OBJECT_NAME": "PLEIADES 1A", "NORAD_CAT_ID": 38012, "MEAN_MOTION": 14.58543149, "ECCENTRICITY": 0.0001693, "INCLINATION": 98.2035, "RA_OF_ASC_NODE": 149.3549, "ARG_OF_PERICENTER": 87.1605, "MEAN_ANOMALY": 85.9887, "BSTAR": 3.6172e-05, "EPOCH": "2026-03-13T23:54:34"}, {"OBJECT_NAME": "PLEIADES YEARLING", "NORAD_CAT_ID": 56207, "MEAN_MOTION": 15.36797276, "ECCENTRICITY": 0.0012954, "INCLINATION": 97.388, "RA_OF_ASC_NODE": 256.6744, "ARG_OF_PERICENTER": 51.7533, "MEAN_ANOMALY": 308.4875, "BSTAR": 0.0010478, "EPOCH": "2023-12-28T09:58:50"}, {"OBJECT_NAME": "CSO-3", "NORAD_CAT_ID": 63156, "MEAN_MOTION": 14.34022377, "ECCENTRICITY": 0.0001932, "INCLINATION": 98.606, "RA_OF_ASC_NODE": 14.3262, "ARG_OF_PERICENTER": 17.427, "MEAN_ANOMALY": 342.6981, "BSTAR": 0.00016071, "EPOCH": "2025-03-13T22:37:36"}] \ No newline at end of file +[{"OBJECT_NAME": "WORLDVIEW-3 (WV-3)", "NORAD_CAT_ID": 40115, "MEAN_MOTION": 14.84866273, "ECCENTRICITY": 0.0001046, "INCLINATION": 97.8612, "RA_OF_ASC_NODE": 158.9303, "ARG_OF_PERICENTER": 163.4248, "MEAN_ANOMALY": 196.7001, "BSTAR": 0.00013244, "EPOCH": "2026-03-23T22:14:54"}, {"OBJECT_NAME": "WORLDVIEW-2 (WV-2)", "NORAD_CAT_ID": 35946, "MEAN_MOTION": 14.37916335, "ECCENTRICITY": 0.0005529, "INCLINATION": 98.467, "RA_OF_ASC_NODE": 157.5833, "ARG_OF_PERICENTER": 110.9181, "MEAN_ANOMALY": 249.2598, "BSTAR": 7.3348e-05, "EPOCH": "2026-03-23T22:08:03"}, {"OBJECT_NAME": "WORLDVIEW-1 (WV-1)", "NORAD_CAT_ID": 32060, "MEAN_MOTION": 15.23842027, "ECCENTRICITY": 0.0002465, "INCLINATION": 97.3829, "RA_OF_ASC_NODE": 203.8473, "ARG_OF_PERICENTER": 27.3344, "MEAN_ANOMALY": 332.8023, "BSTAR": 0.00033418, "EPOCH": "2026-03-23T21:13:37"}, {"OBJECT_NAME": "PLEIADES NEO 4", "NORAD_CAT_ID": 49070, "MEAN_MOTION": 14.81675608, "ECCENTRICITY": 0.0001306, "INCLINATION": 97.8932, "RA_OF_ASC_NODE": 157.181, "ARG_OF_PERICENTER": 94.4254, "MEAN_ANOMALY": 265.7108, "BSTAR": 8.6816e-05, "EPOCH": "2026-03-22T04:04:49"}, {"OBJECT_NAME": "PLEIADES NEO 3", "NORAD_CAT_ID": 48268, "MEAN_MOTION": 14.81669651, "ECCENTRICITY": 0.0001343, "INCLINATION": 97.8927, "RA_OF_ASC_NODE": 158.949, "ARG_OF_PERICENTER": 91.4181, "MEAN_ANOMALY": 268.7186, "BSTAR": -0.00024907, "EPOCH": "2026-03-23T23:01:53"}, {"OBJECT_NAME": "PLEIADES 1A", "NORAD_CAT_ID": 38012, "MEAN_MOTION": 14.58551306, "ECCENTRICITY": 0.00016, "INCLINATION": 98.2017, "RA_OF_ASC_NODE": 159.1587, "ARG_OF_PERICENTER": 78.3578, "MEAN_ANOMALY": 9.8748, "BSTAR": 8.183600000000001e-05, "EPOCH": "2026-03-23T22:15:22"}, {"OBJECT_NAME": "PLEIADES 1B", "NORAD_CAT_ID": 39019, "MEAN_MOTION": 14.58538759, "ECCENTRICITY": 0.0001481, "INCLINATION": 98.1987, "RA_OF_ASC_NODE": 159.1544, "ARG_OF_PERICENTER": 79.8677, "MEAN_ANOMALY": 280.2689, "BSTAR": 9.6175e-05, "EPOCH": "2026-03-23T22:40:39"}, {"OBJECT_NAME": "PLEIADES YEARLING", "NORAD_CAT_ID": 56207, "MEAN_MOTION": 15.36797276, "ECCENTRICITY": 0.0012954, "INCLINATION": 97.388, "RA_OF_ASC_NODE": 256.6744, "ARG_OF_PERICENTER": 51.7533, "MEAN_ANOMALY": 308.4875, "BSTAR": 0.0010478, "EPOCH": "2023-12-28T09:58:50"}, {"OBJECT_NAME": "NAVSTAR 66 (USA 232)", "NORAD_CAT_ID": 37753, "MEAN_MOTION": 2.00562992, "ECCENTRICITY": 0.014066, "INCLINATION": 56.6025, "RA_OF_ASC_NODE": 335.3388, "ARG_OF_PERICENTER": 61.1241, "MEAN_ANOMALY": 166.3638, "BSTAR": 0.0, "EPOCH": "2026-03-23T22:08:50"}, {"OBJECT_NAME": "NAVSTAR 55 (USA 178)", "NORAD_CAT_ID": 28361, "MEAN_MOTION": 2.00552574, "ECCENTRICITY": 0.0160035, "INCLINATION": 54.8016, "RA_OF_ASC_NODE": 90.3066, "ARG_OF_PERICENTER": 299.1359, "MEAN_ANOMALY": 240.0198, "BSTAR": 0.0, "EPOCH": "2026-03-20T03:21:14"}, {"OBJECT_NAME": "NAVSTAR 52 (USA 168)", "NORAD_CAT_ID": 27704, "MEAN_MOTION": 1.9474293, "ECCENTRICITY": 0.0004456, "INCLINATION": 54.8635, "RA_OF_ASC_NODE": 328.4349, "ARG_OF_PERICENTER": 347.0054, "MEAN_ANOMALY": 191.4599, "BSTAR": 0.0, "EPOCH": "2026-03-22T07:31:16"}, {"OBJECT_NAME": "NAVSTAR 53 (USA 175)", "NORAD_CAT_ID": 28129, "MEAN_MOTION": 1.92678335, "ECCENTRICITY": 0.0003099, "INCLINATION": 54.9773, "RA_OF_ASC_NODE": 26.5196, "ARG_OF_PERICENTER": 317.9046, "MEAN_ANOMALY": 42.0616, "BSTAR": 0.0, "EPOCH": "2026-03-21T08:09:11"}, {"OBJECT_NAME": "NAVSTAR 63 (USA 203)", "NORAD_CAT_ID": 34661, "MEAN_MOTION": 2.00555218, "ECCENTRICITY": 0.0144775, "INCLINATION": 54.5006, "RA_OF_ASC_NODE": 214.1187, "ARG_OF_PERICENTER": 61.5232, "MEAN_ANOMALY": 271.4716, "BSTAR": 0.0, "EPOCH": "2026-03-22T18:12:03"}, {"OBJECT_NAME": "NAVSTAR 46 (USA 145)", "NORAD_CAT_ID": 25933, "MEAN_MOTION": 2.00566586, "ECCENTRICITY": 0.010638, "INCLINATION": 51.5403, "RA_OF_ASC_NODE": 299.2015, "ARG_OF_PERICENTER": 172.1852, "MEAN_ANOMALY": 38.7108, "BSTAR": 0.0, "EPOCH": "2026-03-23T19:59:26"}, {"OBJECT_NAME": "NAVSTAR 49 (USA 154)", "NORAD_CAT_ID": 26605, "MEAN_MOTION": 2.00569544, "ECCENTRICITY": 0.0173718, "INCLINATION": 55.5752, "RA_OF_ASC_NODE": 98.3329, "ARG_OF_PERICENTER": 265.8473, "MEAN_ANOMALY": 65.9305, "BSTAR": 0.0, "EPOCH": "2026-03-23T14:29:29"}, {"OBJECT_NAME": "ICEYE-X8", "NORAD_CAT_ID": 47510, "MEAN_MOTION": 15.26805421, "ECCENTRICITY": 0.0007492, "INCLINATION": 97.3641, "RA_OF_ASC_NODE": 54.4014, "ARG_OF_PERICENTER": 124.0674, "MEAN_ANOMALY": 236.1278, "BSTAR": 0.00062244, "EPOCH": "2023-12-28T10:34:23"}, {"OBJECT_NAME": "ICEYE-X12", "NORAD_CAT_ID": 48914, "MEAN_MOTION": 15.19543383, "ECCENTRICITY": 0.0001769, "INCLINATION": 97.6141, "RA_OF_ASC_NODE": 134.5462, "ARG_OF_PERICENTER": 318.3531, "MEAN_ANOMALY": 41.7569, "BSTAR": 0.0007832000000000001, "EPOCH": "2023-12-28T10:18:56"}, {"OBJECT_NAME": "ICEYE-X14", "NORAD_CAT_ID": 51070, "MEAN_MOTION": 15.19599986, "ECCENTRICITY": 0.0006515, "INCLINATION": 97.418, "RA_OF_ASC_NODE": 67.3206, "ARG_OF_PERICENTER": 19.2413, "MEAN_ANOMALY": 340.9067, "BSTAR": 0.00061029, "EPOCH": "2023-12-28T10:10:51"}, {"OBJECT_NAME": "ICEYE-X1", "NORAD_CAT_ID": 43114, "MEAN_MOTION": 15.77777582, "ECCENTRICITY": 0.0001643, "INCLINATION": 97.2952, "RA_OF_ASC_NODE": 83.4991, "ARG_OF_PERICENTER": 155.7939, "MEAN_ANOMALY": 204.3405, "BSTAR": 0.0011651, "EPOCH": "2023-12-28T10:38:05"}, {"OBJECT_NAME": "ICEYE-X2", "NORAD_CAT_ID": 43800, "MEAN_MOTION": 15.16627646, "ECCENTRICITY": 0.0009126, "INCLINATION": 97.4457, "RA_OF_ASC_NODE": 138.3006, "ARG_OF_PERICENTER": 232.3621, "MEAN_ANOMALY": 127.6785, "BSTAR": 0.00031575, "EPOCH": "2026-03-23T22:18:50"}, {"OBJECT_NAME": "ICEYE-X6", "NORAD_CAT_ID": 46497, "MEAN_MOTION": 15.0399879, "ECCENTRICITY": 0.0007849, "INCLINATION": 98.0961, "RA_OF_ASC_NODE": 72.5567, "ARG_OF_PERICENTER": 25.6767, "MEAN_ANOMALY": 334.4846, "BSTAR": 0.00029317, "EPOCH": "2026-03-23T21:57:45"}, {"OBJECT_NAME": "ICEYE-X21", "NORAD_CAT_ID": 55049, "MEAN_MOTION": 15.79237861, "ECCENTRICITY": 0.0002471, "INCLINATION": 97.3343, "RA_OF_ASC_NODE": 162.6877, "ARG_OF_PERICENTER": 14.0044, "MEAN_ANOMALY": 346.1291, "BSTAR": 0.00059244, "EPOCH": "2026-03-23T22:20:31"}, {"OBJECT_NAME": "ICEYE-X62", "NORAD_CAT_ID": 66755, "MEAN_MOTION": 15.12221504, "ECCENTRICITY": 8.9e-05, "INCLINATION": 97.4248, "RA_OF_ASC_NODE": 157.748, "ARG_OF_PERICENTER": 336.8961, "MEAN_ANOMALY": 23.223, "BSTAR": 0.00017105, "EPOCH": "2026-03-23T22:19:01"}, {"OBJECT_NAME": "ICEYE-X50", "NORAD_CAT_ID": 63255, "MEAN_MOTION": 14.94969678, "ECCENTRICITY": 0.0003043, "INCLINATION": 97.6981, "RA_OF_ASC_NODE": 336.4695, "ARG_OF_PERICENTER": 114.5539, "MEAN_ANOMALY": 245.6, "BSTAR": 0.00020511, "EPOCH": "2026-03-23T23:05:30"}, {"OBJECT_NAME": "ICEYE-X34", "NORAD_CAT_ID": 58294, "MEAN_MOTION": 15.53760527, "ECCENTRICITY": 0.0003836, "INCLINATION": 97.3894, "RA_OF_ASC_NODE": 170.7975, "ARG_OF_PERICENTER": 119.8248, "MEAN_ANOMALY": 240.3387, "BSTAR": 0.00086317, "EPOCH": "2026-03-23T22:38:07"}, {"OBJECT_NAME": "ICEYE-X51", "NORAD_CAT_ID": 63257, "MEAN_MOTION": 15.00912938, "ECCENTRICITY": 0.000212, "INCLINATION": 97.7374, "RA_OF_ASC_NODE": 338.3, "ARG_OF_PERICENTER": 63.617, "MEAN_ANOMALY": 296.5272, "BSTAR": 0.00039815, "EPOCH": "2026-03-23T22:07:25"}, {"OBJECT_NAME": "ICEYE-X35", "NORAD_CAT_ID": 58302, "MEAN_MOTION": 15.45563489, "ECCENTRICITY": 0.0010946, "INCLINATION": 97.3827, "RA_OF_ASC_NODE": 165.6267, "ARG_OF_PERICENTER": 146.3105, "MEAN_ANOMALY": 213.8842, "BSTAR": 0.00014358, "EPOCH": "2026-03-23T23:08:34"}, {"OBJECT_NAME": "ICEYE-X46", "NORAD_CAT_ID": 63258, "MEAN_MOTION": 14.95314993, "ECCENTRICITY": 0.0001601, "INCLINATION": 97.6977, "RA_OF_ASC_NODE": 335.9236, "ARG_OF_PERICENTER": 79.8175, "MEAN_ANOMALY": 280.3227, "BSTAR": 0.00029341, "EPOCH": "2026-03-23T10:26:01"}, {"OBJECT_NAME": "ICEYE-X38", "NORAD_CAT_ID": 59100, "MEAN_MOTION": 15.11963966, "ECCENTRICITY": 0.0002925, "INCLINATION": 97.8269, "RA_OF_ASC_NODE": 224.4216, "ARG_OF_PERICENTER": 44.1878, "MEAN_ANOMALY": 315.9585, "BSTAR": 0.00034938000000000005, "EPOCH": "2026-03-23T21:41:26"}, {"OBJECT_NAME": "ICEYE-X37", "NORAD_CAT_ID": 59102, "MEAN_MOTION": 15.05352183, "ECCENTRICITY": 0.0014387, "INCLINATION": 97.8173, "RA_OF_ASC_NODE": 218.6944, "ARG_OF_PERICENTER": 299.25, "MEAN_ANOMALY": 60.7287, "BSTAR": 0.00040508, "EPOCH": "2026-03-23T22:06:00"}, {"OBJECT_NAME": "ICEYE-X52", "NORAD_CAT_ID": 64572, "MEAN_MOTION": 14.93206865, "ECCENTRICITY": 9.66e-05, "INCLINATION": 97.7627, "RA_OF_ASC_NODE": 198.1706, "ARG_OF_PERICENTER": 216.5012, "MEAN_ANOMALY": 143.6142, "BSTAR": 0.00025127, "EPOCH": "2026-03-23T21:49:18"}, {"OBJECT_NAME": "ICEYE-X36", "NORAD_CAT_ID": 59103, "MEAN_MOTION": 15.06119786, "ECCENTRICITY": 0.0004816, "INCLINATION": 97.8236, "RA_OF_ASC_NODE": 219.6471, "ARG_OF_PERICENTER": 238.3735, "MEAN_ANOMALY": 121.7022, "BSTAR": 0.00037749, "EPOCH": "2026-03-23T23:14:04"}, {"OBJECT_NAME": "ICEYE-X56", "NORAD_CAT_ID": 64574, "MEAN_MOTION": 14.95043392, "ECCENTRICITY": 0.0001157, "INCLINATION": 97.7623, "RA_OF_ASC_NODE": 198.6301, "ARG_OF_PERICENTER": 222.2153, "MEAN_ANOMALY": 137.8979, "BSTAR": 0.00040664, "EPOCH": "2026-03-23T21:57:54"}, {"OBJECT_NAME": "ICEYE-X43", "NORAD_CAT_ID": 60539, "MEAN_MOTION": 14.99446128, "ECCENTRICITY": 0.0002659, "INCLINATION": 97.681, "RA_OF_ASC_NODE": 161.0759, "ARG_OF_PERICENTER": 50.9638, "MEAN_ANOMALY": 309.1822, "BSTAR": 0.00037373, "EPOCH": "2026-03-23T22:57:52"}, {"OBJECT_NAME": "ICEYE-X57", "NORAD_CAT_ID": 64578, "MEAN_MOTION": 15.00885756, "ECCENTRICITY": 0.0002434, "INCLINATION": 97.7664, "RA_OF_ASC_NODE": 199.2878, "ARG_OF_PERICENTER": 64.9021, "MEAN_ANOMALY": 295.2455, "BSTAR": 0.00026517, "EPOCH": "2026-03-23T22:18:45"}, {"OBJECT_NAME": "USA 81", "NORAD_CAT_ID": 21949, "MEAN_MOTION": 14.32372337, "ECCENTRICITY": 0.0001997, "INCLINATION": 85.0057, "RA_OF_ASC_NODE": 123.2759, "ARG_OF_PERICENTER": 68.9246, "MEAN_ANOMALY": 291.216, "BSTAR": 5.148000000000001e-05, "EPOCH": "2026-03-23T20:09:27"}, {"OBJECT_NAME": "AAUSAT 4", "NORAD_CAT_ID": 41460, "MEAN_MOTION": 16.30498134, "ECCENTRICITY": 0.0006472, "INCLINATION": 98.1243, "RA_OF_ASC_NODE": 117.1757, "ARG_OF_PERICENTER": 164.1838, "MEAN_ANOMALY": 195.9657, "BSTAR": 0.0017281, "EPOCH": "2023-09-06T08:03:39"}, {"OBJECT_NAME": "DMSP 5D-3 F15 (USA 147)", "NORAD_CAT_ID": 25991, "MEAN_MOTION": 14.17530173, "ECCENTRICITY": 0.0009132, "INCLINATION": 99.0071, "RA_OF_ASC_NODE": 120.4111, "ARG_OF_PERICENTER": 241.9618, "MEAN_ANOMALY": 182.1731, "BSTAR": 0.00011215, "EPOCH": "2026-03-23T23:01:15"}, {"OBJECT_NAME": "PAUSAT-1", "NORAD_CAT_ID": 62653, "MEAN_MOTION": 15.2287681, "ECCENTRICITY": 3.93e-05, "INCLINATION": 97.398, "RA_OF_ASC_NODE": 163.8053, "ARG_OF_PERICENTER": 166.1144, "MEAN_ANOMALY": 194.0104, "BSTAR": 0.00022203000000000001, "EPOCH": "2026-03-23T21:51:05"}, {"OBJECT_NAME": "AAUSAT 3", "NORAD_CAT_ID": 39087, "MEAN_MOTION": 14.39821779, "ECCENTRICITY": 0.0012555, "INCLINATION": 98.3696, "RA_OF_ASC_NODE": 268.5179, "ARG_OF_PERICENTER": 107.7486, "MEAN_ANOMALY": 252.5074, "BSTAR": 0.00024019, "EPOCH": "2026-03-23T18:39:01"}, {"OBJECT_NAME": "SNUSAT-2", "NORAD_CAT_ID": 43782, "MEAN_MOTION": 15.20430429, "ECCENTRICITY": 0.0009483, "INCLINATION": 97.416, "RA_OF_ASC_NODE": 137.3613, "ARG_OF_PERICENTER": 228.8671, "MEAN_ANOMALY": 131.1746, "BSTAR": 0.00042971, "EPOCH": "2026-03-23T14:53:14"}, {"OBJECT_NAME": "LUSAT (LO-19)", "NORAD_CAT_ID": 20442, "MEAN_MOTION": 14.3409352, "ECCENTRICITY": 0.0012702, "INCLINATION": 98.8925, "RA_OF_ASC_NODE": 101.9231, "ARG_OF_PERICENTER": 97.1458, "MEAN_ANOMALY": 263.1171, "BSTAR": 6.4529e-05, "EPOCH": "2026-03-23T12:07:35"}, {"OBJECT_NAME": "DMSP 5D-3 F17 (USA 191)", "NORAD_CAT_ID": 29522, "MEAN_MOTION": 14.14970238, "ECCENTRICITY": 0.0009071, "INCLINATION": 98.7412, "RA_OF_ASC_NODE": 91.2675, "ARG_OF_PERICENTER": 303.4042, "MEAN_ANOMALY": 56.6264, "BSTAR": 0.00010219000000000001, "EPOCH": "2026-03-23T23:05:12"}, {"OBJECT_NAME": "TURKSAT-3USAT", "NORAD_CAT_ID": 39152, "MEAN_MOTION": 14.95851472, "ECCENTRICITY": 0.0012727, "INCLINATION": 97.7913, "RA_OF_ASC_NODE": 173.9063, "ARG_OF_PERICENTER": 33.3146, "MEAN_ANOMALY": 326.8873, "BSTAR": 0.00040783000000000003, "EPOCH": "2026-03-23T16:13:14"}, {"OBJECT_NAME": "JINJUSAT-1B", "NORAD_CAT_ID": 63210, "MEAN_MOTION": 15.30728901, "ECCENTRICITY": 0.0004839, "INCLINATION": 97.4093, "RA_OF_ASC_NODE": 339.7522, "ARG_OF_PERICENTER": 50.313, "MEAN_ANOMALY": 309.8537, "BSTAR": 0.00064421, "EPOCH": "2026-03-23T23:34:24"}, {"OBJECT_NAME": "CINEMA-3 (KHUSAT-2)", "NORAD_CAT_ID": 39426, "MEAN_MOTION": 14.78042054, "ECCENTRICITY": 0.00914, "INCLINATION": 97.8428, "RA_OF_ASC_NODE": 357.146, "ARG_OF_PERICENTER": 128.6544, "MEAN_ANOMALY": 232.2892, "BSTAR": 0.00020388, "EPOCH": "2026-03-23T17:10:55"}, {"OBJECT_NAME": "DMSP 5D-3 F18 (USA 210)", "NORAD_CAT_ID": 35951, "MEAN_MOTION": 14.1484466, "ECCENTRICITY": 0.0011854, "INCLINATION": 98.8984, "RA_OF_ASC_NODE": 63.9105, "ARG_OF_PERICENTER": 129.3914, "MEAN_ANOMALY": 230.831, "BSTAR": 0.00015031, "EPOCH": "2026-03-23T22:14:41"}, {"OBJECT_NAME": "DTUSAT-2", "NORAD_CAT_ID": 40030, "MEAN_MOTION": 15.13780924, "ECCENTRICITY": 0.000865, "INCLINATION": 98.0866, "RA_OF_ASC_NODE": 75.3412, "ARG_OF_PERICENTER": 153.9454, "MEAN_ANOMALY": 206.2213, "BSTAR": 0.0004703, "EPOCH": "2026-03-23T22:31:11"}, {"OBJECT_NAME": "MUSAT-2", "NORAD_CAT_ID": 59099, "MEAN_MOTION": 15.0740969, "ECCENTRICITY": 0.000645, "INCLINATION": 97.8464, "RA_OF_ASC_NODE": 219.7209, "ARG_OF_PERICENTER": 259.9759, "MEAN_ANOMALY": 100.074, "BSTAR": 0.00059893, "EPOCH": "2026-03-23T22:14:33"}, {"OBJECT_NAME": "CHUBUSAT-2", "NORAD_CAT_ID": 41338, "MEAN_MOTION": 15.1994134, "ECCENTRICITY": 0.0008714, "INCLINATION": 30.9972, "RA_OF_ASC_NODE": 163.0639, "ARG_OF_PERICENTER": 281.6929, "MEAN_ANOMALY": 78.2734, "BSTAR": 0.00029645, "EPOCH": "2026-03-23T12:57:00"}, {"OBJECT_NAME": "CHUBUSAT-3", "NORAD_CAT_ID": 41339, "MEAN_MOTION": 15.17790783, "ECCENTRICITY": 0.0009095, "INCLINATION": 31.0043, "RA_OF_ASC_NODE": 168.101, "ARG_OF_PERICENTER": 275.9386, "MEAN_ANOMALY": 84.0217, "BSTAR": 0.00030377, "EPOCH": "2026-03-23T22:23:42"}, {"OBJECT_NAME": "O/OREOS (USA 219)", "NORAD_CAT_ID": 37224, "MEAN_MOTION": 14.93351824, "ECCENTRICITY": 0.0016402, "INCLINATION": 71.9693, "RA_OF_ASC_NODE": 273.3972, "ARG_OF_PERICENTER": 232.5648, "MEAN_ANOMALY": 127.4031, "BSTAR": 0.00027423, "EPOCH": "2026-03-23T22:23:59"}, {"OBJECT_NAME": "DMSP 5D-3 F16 (USA 172)", "NORAD_CAT_ID": 28054, "MEAN_MOTION": 14.14467391, "ECCENTRICITY": 0.0007632, "INCLINATION": 98.9976, "RA_OF_ASC_NODE": 106.1546, "ARG_OF_PERICENTER": 36.6673, "MEAN_ANOMALY": 353.3775, "BSTAR": 8.8911e-05, "EPOCH": "2026-03-23T22:05:44"}, {"OBJECT_NAME": "UFO 11 (USA 174)", "NORAD_CAT_ID": 28117, "MEAN_MOTION": 1.0027229, "ECCENTRICITY": 0.0003244, "INCLINATION": 8.8778, "RA_OF_ASC_NODE": 34.236, "ARG_OF_PERICENTER": 312.1031, "MEAN_ANOMALY": 233.1226, "BSTAR": 0.0, "EPOCH": "2026-03-23T21:30:38"}, {"OBJECT_NAME": "TANDEM-X", "NORAD_CAT_ID": 36605, "MEAN_MOTION": 15.19155413, "ECCENTRICITY": 0.000188, "INCLINATION": 97.4472, "RA_OF_ASC_NODE": 91.4886, "ARG_OF_PERICENTER": 95.4532, "MEAN_ANOMALY": 264.6917, "BSTAR": 6.508000000000001e-05, "EPOCH": "2026-03-23T13:30:39"}, {"OBJECT_NAME": "GAOFEN-3", "NORAD_CAT_ID": 41727, "MEAN_MOTION": 14.42219275, "ECCENTRICITY": 0.0001696, "INCLINATION": 98.4051, "RA_OF_ASC_NODE": 92.0187, "ARG_OF_PERICENTER": 79.4554, "MEAN_ANOMALY": 280.6827, "BSTAR": -5.2673e-06, "EPOCH": "2026-03-23T22:58:33"}, {"OBJECT_NAME": "GAOFEN-3 03", "NORAD_CAT_ID": 52200, "MEAN_MOTION": 14.42209809, "ECCENTRICITY": 0.0001666, "INCLINATION": 98.4104, "RA_OF_ASC_NODE": 92.769, "ARG_OF_PERICENTER": 86.1274, "MEAN_ANOMALY": 274.0106, "BSTAR": -9.983400000000002e-06, "EPOCH": "2026-03-23T23:08:21"}, {"OBJECT_NAME": "GAOFEN-4", "NORAD_CAT_ID": 41194, "MEAN_MOTION": 1.00268371, "ECCENTRICITY": 0.0005814, "INCLINATION": 0.3588, "RA_OF_ASC_NODE": 82.9448, "ARG_OF_PERICENTER": 89.3201, "MEAN_ANOMALY": 269.7225, "BSTAR": 0.0, "EPOCH": "2026-03-23T10:21:34"}, {"OBJECT_NAME": "GAOFEN-1 02", "NORAD_CAT_ID": 43259, "MEAN_MOTION": 14.76465516, "ECCENTRICITY": 0.0001644, "INCLINATION": 98.0609, "RA_OF_ASC_NODE": 145.7047, "ARG_OF_PERICENTER": 286.1799, "MEAN_ANOMALY": 73.923, "BSTAR": -7.158799999999999e-05, "EPOCH": "2026-03-23T23:24:00"}, {"OBJECT_NAME": "GAOFEN-11 02", "NORAD_CAT_ID": 46396, "MEAN_MOTION": 15.23212665, "ECCENTRICITY": 0.001875, "INCLINATION": 97.5004, "RA_OF_ASC_NODE": 208.9239, "ARG_OF_PERICENTER": 243.6211, "MEAN_ANOMALY": 116.3101, "BSTAR": 0.000322, "EPOCH": "2026-03-23T22:37:23"}, {"OBJECT_NAME": "GAOFEN-7", "NORAD_CAT_ID": 44703, "MEAN_MOTION": 15.21461215, "ECCENTRICITY": 0.0014323, "INCLINATION": 97.251, "RA_OF_ASC_NODE": 147.5953, "ARG_OF_PERICENTER": 340.0442, "MEAN_ANOMALY": 20.0232, "BSTAR": 0.00022482, "EPOCH": "2026-03-23T23:03:50"}, {"OBJECT_NAME": "GAOFEN-2", "NORAD_CAT_ID": 40118, "MEAN_MOTION": 14.80791256, "ECCENTRICITY": 0.000813, "INCLINATION": 98.0189, "RA_OF_ASC_NODE": 147.2324, "ARG_OF_PERICENTER": 140.7409, "MEAN_ANOMALY": 219.4394, "BSTAR": 4.516e-05, "EPOCH": "2026-03-23T23:17:55"}, {"OBJECT_NAME": "GAOFEN-8", "NORAD_CAT_ID": 40701, "MEAN_MOTION": 15.42958648, "ECCENTRICITY": 0.001014, "INCLINATION": 97.6941, "RA_OF_ASC_NODE": 271.3873, "ARG_OF_PERICENTER": 138.0452, "MEAN_ANOMALY": 222.1574, "BSTAR": 0.00054055, "EPOCH": "2026-03-23T22:52:04"}, {"OBJECT_NAME": "GAOFEN-6", "NORAD_CAT_ID": 43484, "MEAN_MOTION": 14.76621861, "ECCENTRICITY": 0.0013691, "INCLINATION": 97.7838, "RA_OF_ASC_NODE": 148.7333, "ARG_OF_PERICENTER": 92.9574, "MEAN_ANOMALY": 267.3204, "BSTAR": 0.00017874, "EPOCH": "2026-03-23T23:00:38"}, {"OBJECT_NAME": "GAOFEN-3 02", "NORAD_CAT_ID": 49495, "MEAN_MOTION": 14.42209942, "ECCENTRICITY": 0.0001499, "INCLINATION": 98.4129, "RA_OF_ASC_NODE": 92.2186, "ARG_OF_PERICENTER": 229.5097, "MEAN_ANOMALY": 130.5962, "BSTAR": 7.057600000000001e-06, "EPOCH": "2026-03-23T22:07:42"}, {"OBJECT_NAME": "GAOFEN-10R", "NORAD_CAT_ID": 44622, "MEAN_MOTION": 14.82842154, "ECCENTRICITY": 0.0006961, "INCLINATION": 98.0417, "RA_OF_ASC_NODE": 35.9458, "ARG_OF_PERICENTER": 60.6716, "MEAN_ANOMALY": 299.5191, "BSTAR": 0.00015433, "EPOCH": "2026-03-23T22:25:42"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D16", "NORAD_CAT_ID": 51834, "MEAN_MOTION": 15.98144909, "ECCENTRICITY": 0.0003354, "INCLINATION": 97.3634, "RA_OF_ASC_NODE": 178.7994, "ARG_OF_PERICENTER": 237.7426, "MEAN_ANOMALY": 122.3526, "BSTAR": 0.0012393999999999999, "EPOCH": "2026-03-23T23:28:18"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3G", "NORAD_CAT_ID": 46459, "MEAN_MOTION": 15.59488718, "ECCENTRICITY": 0.0011046, "INCLINATION": 97.2423, "RA_OF_ASC_NODE": 137.4838, "ARG_OF_PERICENTER": 15.4776, "MEAN_ANOMALY": 344.6817, "BSTAR": 0.0006899300000000001, "EPOCH": "2026-03-23T23:16:38"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3H", "NORAD_CAT_ID": 46460, "MEAN_MOTION": 15.57743023, "ECCENTRICITY": 0.0005696, "INCLINATION": 97.2369, "RA_OF_ASC_NODE": 134.16, "ARG_OF_PERICENTER": 298.9229, "MEAN_ANOMALY": 61.1455, "BSTAR": 0.00065964, "EPOCH": "2026-03-23T22:29:53"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D48", "NORAD_CAT_ID": 54695, "MEAN_MOTION": 15.3332151, "ECCENTRICITY": 0.0007431, "INCLINATION": 97.6802, "RA_OF_ASC_NODE": 248.9993, "ARG_OF_PERICENTER": 68.8434, "MEAN_ANOMALY": 291.3601, "BSTAR": 0.00038528, "EPOCH": "2026-03-23T23:01:27"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D11", "NORAD_CAT_ID": 51835, "MEAN_MOTION": 15.8286677, "ECCENTRICITY": 0.0002122, "INCLINATION": 97.3709, "RA_OF_ASC_NODE": 176.9403, "ARG_OF_PERICENTER": 171.7584, "MEAN_ANOMALY": 188.3721, "BSTAR": 0.0011271, "EPOCH": "2026-03-23T17:43:22"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 2A", "NORAD_CAT_ID": 44777, "MEAN_MOTION": 15.36878118, "ECCENTRICITY": 0.0006843, "INCLINATION": 97.4984, "RA_OF_ASC_NODE": 164.8593, "ARG_OF_PERICENTER": 41.3667, "MEAN_ANOMALY": 318.8094, "BSTAR": 0.00046902, "EPOCH": "2026-03-23T21:52:57"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3J", "NORAD_CAT_ID": 46462, "MEAN_MOTION": 15.80104291, "ECCENTRICITY": 0.0002684, "INCLINATION": 97.2352, "RA_OF_ASC_NODE": 140.4914, "ARG_OF_PERICENTER": 120.428, "MEAN_ANOMALY": 239.7253, "BSTAR": 0.0010071, "EPOCH": "2026-03-23T22:54:33"}, {"OBJECT_NAME": "GAOFEN-1", "NORAD_CAT_ID": 39150, "MEAN_MOTION": 14.76504812, "ECCENTRICITY": 0.0017257, "INCLINATION": 97.9146, "RA_OF_ASC_NODE": 156.7257, "ARG_OF_PERICENTER": 176.0755, "MEAN_ANOMALY": 184.0593, "BSTAR": 9.0946e-05, "EPOCH": "2026-03-23T22:43:48"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D15", "NORAD_CAT_ID": 51839, "MEAN_MOTION": 15.72614376, "ECCENTRICITY": 0.0003208, "INCLINATION": 97.3671, "RA_OF_ASC_NODE": 174.945, "ARG_OF_PERICENTER": 150.7744, "MEAN_ANOMALY": 209.3699, "BSTAR": 0.0010687000000000001, "EPOCH": "2026-03-23T22:47:02"}, {"OBJECT_NAME": "GLONASS125 [COD]", "NORAD_CAT_ID": 37372, "MEAN_MOTION": 2.13104226, "ECCENTRICITY": 0.000871, "INCLINATION": 64.8285, "RA_OF_ASC_NODE": 104.8188, "ARG_OF_PERICENTER": 293.3373, "MEAN_ANOMALY": 12.0047, "BSTAR": 0.0, "EPOCH": "2023-11-08T23:59:42"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 2D", "NORAD_CAT_ID": 49256, "MEAN_MOTION": 15.09880194, "ECCENTRICITY": 0.0035009, "INCLINATION": 97.6345, "RA_OF_ASC_NODE": 205.8699, "ARG_OF_PERICENTER": 77.4136, "MEAN_ANOMALY": 283.1005, "BSTAR": 0.00038712, "EPOCH": "2026-03-23T22:56:31"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 2F", "NORAD_CAT_ID": 49338, "MEAN_MOTION": 15.07085573, "ECCENTRICITY": 0.0019822, "INCLINATION": 97.6567, "RA_OF_ASC_NODE": 209.457, "ARG_OF_PERICENTER": 153.8209, "MEAN_ANOMALY": 206.4025, "BSTAR": 0.00032739, "EPOCH": "2026-03-23T22:29:43"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 2B", "NORAD_CAT_ID": 44836, "MEAN_MOTION": 15.19648813, "ECCENTRICITY": 0.0012973, "INCLINATION": 97.4933, "RA_OF_ASC_NODE": 155.5415, "ARG_OF_PERICENTER": 254.515, "MEAN_ANOMALY": 105.4652, "BSTAR": 0.0003231, "EPOCH": "2026-03-23T23:20:48"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D27", "NORAD_CAT_ID": 52444, "MEAN_MOTION": 15.60685938, "ECCENTRICITY": 0.0003524, "INCLINATION": 97.5633, "RA_OF_ASC_NODE": 190.2659, "ARG_OF_PERICENTER": 348.3539, "MEAN_ANOMALY": 11.7635, "BSTAR": 0.00086866, "EPOCH": "2026-03-23T22:12:34"}, {"OBJECT_NAME": "JILIN-1 KUANFU 01", "NORAD_CAT_ID": 45016, "MEAN_MOTION": 15.10185296, "ECCENTRICITY": 0.0008768, "INCLINATION": 97.5071, "RA_OF_ASC_NODE": 158.5283, "ARG_OF_PERICENTER": 229.3324, "MEAN_ANOMALY": 130.7144, "BSTAR": 0.00028474, "EPOCH": "2026-03-23T21:27:20"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3D28", "NORAD_CAT_ID": 52445, "MEAN_MOTION": 15.6209442, "ECCENTRICITY": 0.0001895, "INCLINATION": 97.5576, "RA_OF_ASC_NODE": 190.2591, "ARG_OF_PERICENTER": 18.2209, "MEAN_ANOMALY": 341.9116, "BSTAR": 0.00086381, "EPOCH": "2026-03-23T23:19:00"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D14", "NORAD_CAT_ID": 51831, "MEAN_MOTION": 15.86805562, "ECCENTRICITY": 0.0001296, "INCLINATION": 97.3675, "RA_OF_ASC_NODE": 177.2506, "ARG_OF_PERICENTER": 193.5898, "MEAN_ANOMALY": 166.5339, "BSTAR": 0.0011907, "EPOCH": "2026-03-23T22:50:27"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3B", "NORAD_CAT_ID": 46454, "MEAN_MOTION": 15.8567357, "ECCENTRICITY": 0.0002578, "INCLINATION": 97.2304, "RA_OF_ASC_NODE": 141.2448, "ARG_OF_PERICENTER": 150.4835, "MEAN_ANOMALY": 209.6582, "BSTAR": 0.0010021, "EPOCH": "2026-03-23T22:26:33"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3E", "NORAD_CAT_ID": 46457, "MEAN_MOTION": 15.55062298, "ECCENTRICITY": 0.0015272, "INCLINATION": 97.2394, "RA_OF_ASC_NODE": 134.0693, "ARG_OF_PERICENTER": 118.0185, "MEAN_ANOMALY": 242.2616, "BSTAR": 0.00067247, "EPOCH": "2026-03-23T22:15:00"}, {"OBJECT_NAME": "JILIN-1", "NORAD_CAT_ID": 40961, "MEAN_MOTION": 14.78016522, "ECCENTRICITY": 0.0017043, "INCLINATION": 97.6521, "RA_OF_ASC_NODE": 86.5283, "ARG_OF_PERICENTER": 175.8515, "MEAN_ANOMALY": 184.2841, "BSTAR": 0.00010638000000000001, "EPOCH": "2026-03-23T22:03:17"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 3F", "NORAD_CAT_ID": 46458, "MEAN_MOTION": 15.53373641, "ECCENTRICITY": 0.0016791, "INCLINATION": 97.2346, "RA_OF_ASC_NODE": 131.772, "ARG_OF_PERICENTER": 106.9766, "MEAN_ANOMALY": 253.333, "BSTAR": 0.0006441200000000001, "EPOCH": "2026-03-23T23:07:50"}, {"OBJECT_NAME": "JILIN-1 03", "NORAD_CAT_ID": 41914, "MEAN_MOTION": 15.39390729, "ECCENTRICITY": 0.0008156, "INCLINATION": 97.1996, "RA_OF_ASC_NODE": 91.3404, "ARG_OF_PERICENTER": 113.5588, "MEAN_ANOMALY": 246.6515, "BSTAR": 0.00036260999999999997, "EPOCH": "2026-03-23T22:25:07"}, {"OBJECT_NAME": "JILIN-1 04", "NORAD_CAT_ID": 43022, "MEAN_MOTION": 15.40163411, "ECCENTRICITY": 0.0005319, "INCLINATION": 97.477, "RA_OF_ASC_NODE": 190.4406, "ARG_OF_PERICENTER": 135.2123, "MEAN_ANOMALY": 224.9553, "BSTAR": 0.00040168, "EPOCH": "2026-03-23T21:50:51"}, {"OBJECT_NAME": "JILIN-1 GAOFEN 03D10", "NORAD_CAT_ID": 51840, "MEAN_MOTION": 15.82477917, "ECCENTRICITY": 0.000321, "INCLINATION": 97.367, "RA_OF_ASC_NODE": 176.3909, "ARG_OF_PERICENTER": 229.0296, "MEAN_ANOMALY": 131.0695, "BSTAR": 0.0012097, "EPOCH": "2026-03-23T05:46:41"}, {"OBJECT_NAME": "YAOGAN-29", "NORAD_CAT_ID": 41038, "MEAN_MOTION": 14.8301076, "ECCENTRICITY": 8.73e-05, "INCLINATION": 98.012, "RA_OF_ASC_NODE": 111.4297, "ARG_OF_PERICENTER": 12.0769, "MEAN_ANOMALY": 348.0465, "BSTAR": 0.00010119, "EPOCH": "2026-03-23T23:07:31"}, {"OBJECT_NAME": "YAOGAN-7", "NORAD_CAT_ID": 36110, "MEAN_MOTION": 14.77914979, "ECCENTRICITY": 0.0023717, "INCLINATION": 98.0187, "RA_OF_ASC_NODE": 329.2653, "ARG_OF_PERICENTER": 316.814, "MEAN_ANOMALY": 43.121, "BSTAR": 0.00010288, "EPOCH": "2026-03-23T22:04:56"}, {"OBJECT_NAME": "YAOGAN-10", "NORAD_CAT_ID": 36834, "MEAN_MOTION": 14.85070177, "ECCENTRICITY": 0.0001602, "INCLINATION": 97.9063, "RA_OF_ASC_NODE": 107.0589, "ARG_OF_PERICENTER": 83.7106, "MEAN_ANOMALY": 339.4485, "BSTAR": 0.00018409, "EPOCH": "2026-03-23T23:48:12"}, {"OBJECT_NAME": "YAOGAN-21", "NORAD_CAT_ID": 40143, "MEAN_MOTION": 15.25135654, "ECCENTRICITY": 0.0009465, "INCLINATION": 97.1585, "RA_OF_ASC_NODE": 126.8918, "ARG_OF_PERICENTER": 44.5649, "MEAN_ANOMALY": 315.6349, "BSTAR": 0.00018427, "EPOCH": "2026-03-23T23:15:56"}, {"OBJECT_NAME": "YAOGAN-12", "NORAD_CAT_ID": 37875, "MEAN_MOTION": 15.25544934, "ECCENTRICITY": 0.0010655, "INCLINATION": 97.1251, "RA_OF_ASC_NODE": 125.5536, "ARG_OF_PERICENTER": 158.9887, "MEAN_ANOMALY": 201.1792, "BSTAR": 0.00020247, "EPOCH": "2026-03-23T22:59:23"}, {"OBJECT_NAME": "YAOGAN-3", "NORAD_CAT_ID": 32289, "MEAN_MOTION": 14.90433622, "ECCENTRICITY": 0.0001421, "INCLINATION": 97.8247, "RA_OF_ASC_NODE": 113.6453, "ARG_OF_PERICENTER": 77.8247, "MEAN_ANOMALY": 282.313, "BSTAR": 0.00031403, "EPOCH": "2026-03-23T22:36:44"}, {"OBJECT_NAME": "YAOGAN-4", "NORAD_CAT_ID": 33446, "MEAN_MOTION": 14.83017577, "ECCENTRICITY": 0.0014238, "INCLINATION": 97.9259, "RA_OF_ASC_NODE": 16.9309, "ARG_OF_PERICENTER": 355.2266, "MEAN_ANOMALY": 4.8809, "BSTAR": 0.00015872, "EPOCH": "2026-03-23T18:05:58"}, {"OBJECT_NAME": "YAOGAN-23", "NORAD_CAT_ID": 40305, "MEAN_MOTION": 16.38800959, "ECCENTRICITY": 0.000344, "INCLINATION": 97.6653, "RA_OF_ASC_NODE": 349.5059, "ARG_OF_PERICENTER": 254.7314, "MEAN_ANOMALY": 115.2384, "BSTAR": 0.00072463, "EPOCH": "2024-12-12T06:42:04"}, {"OBJECT_NAME": "YAOGAN-13", "NORAD_CAT_ID": 37941, "MEAN_MOTION": 16.37208404, "ECCENTRICITY": 0.0005273, "INCLINATION": 97.6847, "RA_OF_ASC_NODE": 66.5996, "ARG_OF_PERICENTER": 226.2633, "MEAN_ANOMALY": 199.5818, "BSTAR": 0.00080897, "EPOCH": "2025-02-22T04:56:09"}, {"OBJECT_NAME": "YAOGAN-31 03B", "NORAD_CAT_ID": 47693, "MEAN_MOTION": 13.45442996, "ECCENTRICITY": 0.0169955, "INCLINATION": 63.4022, "RA_OF_ASC_NODE": 57.7991, "ARG_OF_PERICENTER": 0.5805, "MEAN_ANOMALY": 359.5388, "BSTAR": 0.00024432, "EPOCH": "2026-03-23T21:58:16"}, {"OBJECT_NAME": "YAOGAN-39 03C", "NORAD_CAT_ID": 57990, "MEAN_MOTION": 15.17173493, "ECCENTRICITY": 0.0009347, "INCLINATION": 34.99, "RA_OF_ASC_NODE": 110.2552, "ARG_OF_PERICENTER": 97.7788, "MEAN_ANOMALY": 262.3987, "BSTAR": 0.00025358000000000005, "EPOCH": "2026-03-23T13:00:22"}, {"OBJECT_NAME": "YAOGAN-32 A", "NORAD_CAT_ID": 43642, "MEAN_MOTION": 14.63057334, "ECCENTRICITY": 0.0001836, "INCLINATION": 98.1571, "RA_OF_ASC_NODE": 110.8971, "ARG_OF_PERICENTER": 85.3542, "MEAN_ANOMALY": 274.787, "BSTAR": 0.00022994, "EPOCH": "2026-03-22T08:08:25"}, {"OBJECT_NAME": "YAOGAN-36 05A", "NORAD_CAT_ID": 57452, "MEAN_MOTION": 15.23047151, "ECCENTRICITY": 0.0001726, "INCLINATION": 35.0037, "RA_OF_ASC_NODE": 252.8039, "ARG_OF_PERICENTER": 322.7123, "MEAN_ANOMALY": 37.3471, "BSTAR": 4.1984e-06, "EPOCH": "2026-03-20T21:52:13"}, {"OBJECT_NAME": "YAOGAN-30 01", "NORAD_CAT_ID": 41473, "MEAN_MOTION": 14.76294995, "ECCENTRICITY": 0.0016884, "INCLINATION": 97.9116, "RA_OF_ASC_NODE": 134.6566, "ARG_OF_PERICENTER": 145.3029, "MEAN_ANOMALY": 214.9286, "BSTAR": 0.00010307, "EPOCH": "2026-03-23T23:00:19"}, {"OBJECT_NAME": "YAOGAN-35 05B", "NORAD_CAT_ID": 53761, "MEAN_MOTION": 15.35242952, "ECCENTRICITY": 0.0006432, "INCLINATION": 34.9772, "RA_OF_ASC_NODE": 137.5029, "ARG_OF_PERICENTER": 124.4469, "MEAN_ANOMALY": 235.6858, "BSTAR": 0.00057286, "EPOCH": "2026-03-23T20:55:06"}, {"OBJECT_NAME": "YAOGAN-40 03B", "NORAD_CAT_ID": 65545, "MEAN_MOTION": 14.12430415, "ECCENTRICITY": 0.000242, "INCLINATION": 85.9852, "RA_OF_ASC_NODE": 82.3477, "ARG_OF_PERICENTER": 81.0208, "MEAN_ANOMALY": 279.1249, "BSTAR": 6.1769e-05, "EPOCH": "2026-03-23T22:58:38"}, {"OBJECT_NAME": "YAOGAN-17 01C", "NORAD_CAT_ID": 39241, "MEAN_MOTION": 13.45470368, "ECCENTRICITY": 0.041961, "INCLINATION": 63.3884, "RA_OF_ASC_NODE": 309.5698, "ARG_OF_PERICENTER": 11.103, "MEAN_ANOMALY": 349.891, "BSTAR": 1.3643e-05, "EPOCH": "2026-03-23T22:09:59"}, {"OBJECT_NAME": "YAOGAN-32 02B", "NORAD_CAT_ID": 49384, "MEAN_MOTION": 14.64068404, "ECCENTRICITY": 0.0001707, "INCLINATION": 98.2314, "RA_OF_ASC_NODE": 216.7349, "ARG_OF_PERICENTER": 114.7127, "MEAN_ANOMALY": 245.4253, "BSTAR": 0.00019789999999999999, "EPOCH": "2026-03-23T22:48:26"}, {"OBJECT_NAME": "YAOGAN-43 01G", "NORAD_CAT_ID": 60464, "MEAN_MOTION": 15.2859006, "ECCENTRICITY": 0.0005184, "INCLINATION": 35.0008, "RA_OF_ASC_NODE": 330.0452, "ARG_OF_PERICENTER": 79.941, "MEAN_ANOMALY": 280.1891, "BSTAR": 0.0003062, "EPOCH": "2026-03-23T22:39:09"}, {"OBJECT_NAME": "YAOGAN-8", "NORAD_CAT_ID": 36121, "MEAN_MOTION": 13.05075063, "ECCENTRICITY": 0.0021154, "INCLINATION": 100.2297, "RA_OF_ASC_NODE": 331.9228, "ARG_OF_PERICENTER": 154.5335, "MEAN_ANOMALY": 205.6819, "BSTAR": -0.0011011999999999999, "EPOCH": "2026-03-23T22:37:20"}, {"OBJECT_NAME": "SKYSAT-A", "NORAD_CAT_ID": 39418, "MEAN_MOTION": 15.12453089, "ECCENTRICITY": 0.0020077, "INCLINATION": 97.392, "RA_OF_ASC_NODE": 134.9626, "ARG_OF_PERICENTER": 247.4521, "MEAN_ANOMALY": 112.4585, "BSTAR": 0.00018955, "EPOCH": "2026-03-23T20:38:51"}, {"OBJECT_NAME": "SKYSAT-C13", "NORAD_CAT_ID": 43802, "MEAN_MOTION": 15.79392393, "ECCENTRICITY": 0.0005361, "INCLINATION": 96.9365, "RA_OF_ASC_NODE": 130.2421, "ARG_OF_PERICENTER": 159.4035, "MEAN_ANOMALY": 200.7451, "BSTAR": 0.00049242, "EPOCH": "2026-03-23T13:15:41"}, {"OBJECT_NAME": "SKYSAT-C11", "NORAD_CAT_ID": 42987, "MEAN_MOTION": 15.53334958, "ECCENTRICITY": 2.09e-05, "INCLINATION": 97.4235, "RA_OF_ASC_NODE": 228.428, "ARG_OF_PERICENTER": 140.593, "MEAN_ANOMALY": 219.5338, "BSTAR": 0.00045133, "EPOCH": "2026-03-23T22:59:03"}, {"OBJECT_NAME": "SKYSAT-C12", "NORAD_CAT_ID": 43797, "MEAN_MOTION": 15.46109074, "ECCENTRICITY": 0.0006297, "INCLINATION": 96.9359, "RA_OF_ASC_NODE": 109.6058, "ARG_OF_PERICENTER": 103.3947, "MEAN_ANOMALY": 256.8006, "BSTAR": 0.00033057, "EPOCH": "2026-03-23T19:19:43"}, {"OBJECT_NAME": "SKYSAT-C1", "NORAD_CAT_ID": 41601, "MEAN_MOTION": 15.34798685, "ECCENTRICITY": 0.0002151, "INCLINATION": 96.9672, "RA_OF_ASC_NODE": 119.2387, "ARG_OF_PERICENTER": 23.9381, "MEAN_ANOMALY": 336.1963, "BSTAR": 0.00030816, "EPOCH": "2026-03-23T20:24:06"}, {"OBJECT_NAME": "SKYSAT-C2", "NORAD_CAT_ID": 41773, "MEAN_MOTION": 15.40074201, "ECCENTRICITY": 0.000371, "INCLINATION": 97.0265, "RA_OF_ASC_NODE": 108.2009, "ARG_OF_PERICENTER": 35.3703, "MEAN_ANOMALY": 324.7789, "BSTAR": 0.00031195, "EPOCH": "2026-03-23T19:25:36"}, {"OBJECT_NAME": "SKYSAT-C3", "NORAD_CAT_ID": 41774, "MEAN_MOTION": 15.4894052, "ECCENTRICITY": 0.0002154, "INCLINATION": 96.9151, "RA_OF_ASC_NODE": 116.8045, "ARG_OF_PERICENTER": 117.6083, "MEAN_ANOMALY": 242.5389, "BSTAR": 0.00035385, "EPOCH": "2026-03-23T12:21:47"}, {"OBJECT_NAME": "SKYSAT-C16", "NORAD_CAT_ID": 45789, "MEAN_MOTION": 15.53303284, "ECCENTRICITY": 0.000551, "INCLINATION": 52.9788, "RA_OF_ASC_NODE": 133.7921, "ARG_OF_PERICENTER": 2.6753, "MEAN_ANOMALY": 357.4284, "BSTAR": 0.0005727, "EPOCH": "2023-12-28T12:24:10"}, {"OBJECT_NAME": "SKYSAT-C15", "NORAD_CAT_ID": 45790, "MEAN_MOTION": 15.47687342, "ECCENTRICITY": 0.0003931, "INCLINATION": 52.9782, "RA_OF_ASC_NODE": 154.0912, "ARG_OF_PERICENTER": 180.8503, "MEAN_ANOMALY": 179.2497, "BSTAR": 0.00058075, "EPOCH": "2023-12-27T15:49:43"}, {"OBJECT_NAME": "SKYSAT-B", "NORAD_CAT_ID": 40072, "MEAN_MOTION": 14.87776582, "ECCENTRICITY": 0.0005302, "INCLINATION": 98.3752, "RA_OF_ASC_NODE": 36.9436, "ARG_OF_PERICENTER": 181.09, "MEAN_ANOMALY": 179.0304, "BSTAR": 0.00018048, "EPOCH": "2026-03-23T18:51:57"}, {"OBJECT_NAME": "SKYSAT-C18", "NORAD_CAT_ID": 46180, "MEAN_MOTION": 16.35747614, "ECCENTRICITY": 0.0017318, "INCLINATION": 52.9561, "RA_OF_ASC_NODE": 11.8431, "ARG_OF_PERICENTER": 265.4879, "MEAN_ANOMALY": 104.1762, "BSTAR": 0.0015942999999999999, "EPOCH": "2023-06-25T20:09:47"}, {"OBJECT_NAME": "SKYSAT-C19", "NORAD_CAT_ID": 46235, "MEAN_MOTION": 15.89514628, "ECCENTRICITY": 0.0002149, "INCLINATION": 52.9675, "RA_OF_ASC_NODE": 201.9813, "ARG_OF_PERICENTER": 132.3648, "MEAN_ANOMALY": 227.7558, "BSTAR": 0.0007014899999999999, "EPOCH": "2023-12-27T23:17:19"}, {"OBJECT_NAME": "SKYSAT-C10", "NORAD_CAT_ID": 42988, "MEAN_MOTION": 15.33010812, "ECCENTRICITY": 0.0001045, "INCLINATION": 97.4367, "RA_OF_ASC_NODE": 219.0815, "ARG_OF_PERICENTER": 102.1325, "MEAN_ANOMALY": 258.0034, "BSTAR": 0.00034211, "EPOCH": "2026-03-23T21:32:31"}, {"OBJECT_NAME": "SKYSAT-C14", "NORAD_CAT_ID": 45788, "MEAN_MOTION": 15.53833173, "ECCENTRICITY": 0.001113, "INCLINATION": 52.9777, "RA_OF_ASC_NODE": 149.837, "ARG_OF_PERICENTER": 213.0476, "MEAN_ANOMALY": 146.9838, "BSTAR": 0.00062044, "EPOCH": "2023-12-27T15:51:00"}, {"OBJECT_NAME": "SKYSAT-C17", "NORAD_CAT_ID": 46179, "MEAN_MOTION": 15.7911991, "ECCENTRICITY": 0.0006126, "INCLINATION": 52.97, "RA_OF_ASC_NODE": 214.8332, "ARG_OF_PERICENTER": 89.9494, "MEAN_ANOMALY": 270.2227, "BSTAR": 0.00064037, "EPOCH": "2023-12-27T20:37:45"}, {"OBJECT_NAME": "SKYSAT-C4", "NORAD_CAT_ID": 41771, "MEAN_MOTION": 15.45874285, "ECCENTRICITY": 0.0002105, "INCLINATION": 96.9272, "RA_OF_ASC_NODE": 108.4278, "ARG_OF_PERICENTER": 133.9438, "MEAN_ANOMALY": 226.1987, "BSTAR": 0.00035497, "EPOCH": "2026-03-23T23:07:06"}, {"OBJECT_NAME": "SKYSAT-C5", "NORAD_CAT_ID": 41772, "MEAN_MOTION": 15.34255747, "ECCENTRICITY": 0.0001037, "INCLINATION": 97.0583, "RA_OF_ASC_NODE": 126.8628, "ARG_OF_PERICENTER": 148.154, "MEAN_ANOMALY": 211.9767, "BSTAR": 0.00031937, "EPOCH": "2026-03-23T09:24:31"}, {"OBJECT_NAME": "SKYSAT-C9", "NORAD_CAT_ID": 42989, "MEAN_MOTION": 15.39335921, "ECCENTRICITY": 0.00099, "INCLINATION": 97.429, "RA_OF_ASC_NODE": 219.228, "ARG_OF_PERICENTER": 87.3297, "MEAN_ANOMALY": 272.9081, "BSTAR": 0.00038876, "EPOCH": "2026-03-23T21:16:10"}, {"OBJECT_NAME": "SKYSAT-C8", "NORAD_CAT_ID": 42990, "MEAN_MOTION": 15.35136827, "ECCENTRICITY": 0.0001261, "INCLINATION": 97.4381, "RA_OF_ASC_NODE": 218.5996, "ARG_OF_PERICENTER": 95.8657, "MEAN_ANOMALY": 264.273, "BSTAR": 0.00036858, "EPOCH": "2026-03-23T22:15:51"}, {"OBJECT_NAME": "SKYSAT-C7", "NORAD_CAT_ID": 42991, "MEAN_MOTION": 15.35089267, "ECCENTRICITY": 0.0006938, "INCLINATION": 97.4393, "RA_OF_ASC_NODE": 219.5738, "ARG_OF_PERICENTER": 60.8557, "MEAN_ANOMALY": 299.338, "BSTAR": 0.00037338, "EPOCH": "2026-03-23T22:02:13"}, {"OBJECT_NAME": "CAPELLA-8-WHITNEY", "NORAD_CAT_ID": 51071, "MEAN_MOTION": 16.38892911, "ECCENTRICITY": 0.0015575, "INCLINATION": 97.4006, "RA_OF_ASC_NODE": 322.5782, "ARG_OF_PERICENTER": 273.5387, "MEAN_ANOMALY": 175.0653, "BSTAR": 0.0030023, "EPOCH": "2023-09-06T01:34:15"}, {"OBJECT_NAME": "CAPELLA-6-WHITNEY", "NORAD_CAT_ID": 48605, "MEAN_MOTION": 15.36992166, "ECCENTRICITY": 0.0006978, "INCLINATION": 53.0293, "RA_OF_ASC_NODE": 151.2332, "ARG_OF_PERICENTER": 275.4158, "MEAN_ANOMALY": 84.6048, "BSTAR": 0.0028544000000000004, "EPOCH": "2023-12-28T11:48:20"}, {"OBJECT_NAME": "CAPELLA-10-WHITNEY", "NORAD_CAT_ID": 55909, "MEAN_MOTION": 14.95890017, "ECCENTRICITY": 0.0009144, "INCLINATION": 43.9994, "RA_OF_ASC_NODE": 246.9297, "ARG_OF_PERICENTER": 166.1646, "MEAN_ANOMALY": 193.9462, "BSTAR": 0.00065599, "EPOCH": "2023-12-27T21:10:20"}, {"OBJECT_NAME": "CAPELLA-9-WHITNEY", "NORAD_CAT_ID": 55910, "MEAN_MOTION": 14.99271847, "ECCENTRICITY": 0.0009067, "INCLINATION": 43.9991, "RA_OF_ASC_NODE": 244.4088, "ARG_OF_PERICENTER": 156.0492, "MEAN_ANOMALY": 204.0788, "BSTAR": 0.0013163, "EPOCH": "2023-12-27T21:00:03"}, {"OBJECT_NAME": "CAPELLA-7-WHITNEY", "NORAD_CAT_ID": 51072, "MEAN_MOTION": 16.44927659, "ECCENTRICITY": 0.0015288, "INCLINATION": 97.3947, "RA_OF_ASC_NODE": 311.3663, "ARG_OF_PERICENTER": 273.4288, "MEAN_ANOMALY": 179.2482, "BSTAR": 0.0011137, "EPOCH": "2023-08-26T19:44:02"}, {"OBJECT_NAME": "CAPELLA-11 (ACADIA-1)", "NORAD_CAT_ID": 57693, "MEAN_MOTION": 14.80964113, "ECCENTRICITY": 0.0001804, "INCLINATION": 53.0062, "RA_OF_ASC_NODE": 194.5789, "ARG_OF_PERICENTER": 151.5429, "MEAN_ANOMALY": 208.5647, "BSTAR": 0.00040513, "EPOCH": "2026-03-23T17:27:17"}, {"OBJECT_NAME": "CAPELLA-14 (ACADIA-4)", "NORAD_CAT_ID": 59444, "MEAN_MOTION": 15.11124421, "ECCENTRICITY": 0.0005656, "INCLINATION": 45.6046, "RA_OF_ASC_NODE": 345.8216, "ARG_OF_PERICENTER": 198.6037, "MEAN_ANOMALY": 161.4643, "BSTAR": 0.0010071, "EPOCH": "2026-03-23T18:49:21"}, {"OBJECT_NAME": "CAPELLA-15 (ACADIA-5)", "NORAD_CAT_ID": 60544, "MEAN_MOTION": 14.92446729, "ECCENTRICITY": 0.0003341, "INCLINATION": 97.6812, "RA_OF_ASC_NODE": 157.7205, "ARG_OF_PERICENTER": 19.6622, "MEAN_ANOMALY": 340.4726, "BSTAR": 0.00080892, "EPOCH": "2026-03-23T16:54:43"}, {"OBJECT_NAME": "CAPELLA-17 (ACADIA-7)", "NORAD_CAT_ID": 64583, "MEAN_MOTION": 14.91090217, "ECCENTRICITY": 0.0004328, "INCLINATION": 97.7583, "RA_OF_ASC_NODE": 197.8128, "ARG_OF_PERICENTER": 273.8465, "MEAN_ANOMALY": 86.2258, "BSTAR": 0.00094777, "EPOCH": "2026-03-23T20:47:21"}, {"OBJECT_NAME": "CAPELLA-13 (ACADIA-3)", "NORAD_CAT_ID": 60419, "MEAN_MOTION": 14.87644967, "ECCENTRICITY": 0.0001416, "INCLINATION": 53.0046, "RA_OF_ASC_NODE": 72.7299, "ARG_OF_PERICENTER": 80.6134, "MEAN_ANOMALY": 279.5006, "BSTAR": 0.00046405999999999997, "EPOCH": "2026-03-23T05:43:32"}, {"OBJECT_NAME": "CAPELLA-16 (ACADIA-6)", "NORAD_CAT_ID": 65318, "MEAN_MOTION": 14.93702919, "ECCENTRICITY": 0.0005696, "INCLINATION": 97.7388, "RA_OF_ASC_NODE": 159.1119, "ARG_OF_PERICENTER": 153.6438, "MEAN_ANOMALY": 206.5072, "BSTAR": 0.0007525699999999999, "EPOCH": "2026-03-23T16:54:33"}, {"OBJECT_NAME": "CAPELLA-19 (ACADIA-9)", "NORAD_CAT_ID": 67384, "MEAN_MOTION": 14.86656972, "ECCENTRICITY": 0.0001664, "INCLINATION": 97.8025, "RA_OF_ASC_NODE": 82.8515, "ARG_OF_PERICENTER": 91.9799, "MEAN_ANOMALY": 268.1607, "BSTAR": 0.0006203300000000001, "EPOCH": "2026-03-23T22:06:08"}, {"OBJECT_NAME": "CAPELLA-18 (ACADIA-8)", "NORAD_CAT_ID": 67385, "MEAN_MOTION": 14.87360263, "ECCENTRICITY": 0.0001807, "INCLINATION": 97.8025, "RA_OF_ASC_NODE": 82.966, "ARG_OF_PERICENTER": 95.2906, "MEAN_ANOMALY": 137.6911, "BSTAR": 0.0066283, "EPOCH": "2026-03-23T22:00:00"}, {"OBJECT_NAME": "BEIDOU-3 IGSO-2 (C39)", "NORAD_CAT_ID": 44337, "MEAN_MOTION": 1.00247364, "ECCENTRICITY": 0.0038734, "INCLINATION": 55.2338, "RA_OF_ASC_NODE": 159.8353, "ARG_OF_PERICENTER": 206.4752, "MEAN_ANOMALY": 153.7223, "BSTAR": 0.0, "EPOCH": "2026-02-26T16:31:51"}, {"OBJECT_NAME": "BEIDOU-3 IGSO-3 (C40)", "NORAD_CAT_ID": 44709, "MEAN_MOTION": 1.00264421, "ECCENTRICITY": 0.0040863, "INCLINATION": 54.953, "RA_OF_ASC_NODE": 281.8597, "ARG_OF_PERICENTER": 188.6782, "MEAN_ANOMALY": 171.2597, "BSTAR": 0.0, "EPOCH": "2026-03-19T23:01:31"}, {"OBJECT_NAME": "BEIDOU-2 M1", "NORAD_CAT_ID": 31115, "MEAN_MOTION": 1.77349143, "ECCENTRICITY": 0.0002047, "INCLINATION": 50.9625, "RA_OF_ASC_NODE": 222.1564, "ARG_OF_PERICENTER": 31.2719, "MEAN_ANOMALY": 328.696, "BSTAR": 0.0, "EPOCH": "2026-03-22T13:10:19"}, {"OBJECT_NAME": "BEIDOU-3 IGSO-1 (C38)", "NORAD_CAT_ID": 44204, "MEAN_MOTION": 1.0026447, "ECCENTRICITY": 0.0025163, "INCLINATION": 58.582, "RA_OF_ASC_NODE": 38.5272, "ARG_OF_PERICENTER": 236.6463, "MEAN_ANOMALY": 340.7025, "BSTAR": 0.0, "EPOCH": "2026-03-23T21:03:43"}, {"OBJECT_NAME": "BEIDOU-2 M4 (C12)", "NORAD_CAT_ID": 38251, "MEAN_MOTION": 1.86229613, "ECCENTRICITY": 0.0011831, "INCLINATION": 55.7315, "RA_OF_ASC_NODE": 306.9953, "ARG_OF_PERICENTER": 286.8996, "MEAN_ANOMALY": 73.0135, "BSTAR": 0.0, "EPOCH": "2026-03-23T04:59:02"}, {"OBJECT_NAME": "BEIDOU-2 M3 (C11)", "NORAD_CAT_ID": 38250, "MEAN_MOTION": 1.86230016, "ECCENTRICITY": 0.0020425, "INCLINATION": 55.8307, "RA_OF_ASC_NODE": 307.7721, "ARG_OF_PERICENTER": 279.7739, "MEAN_ANOMALY": 80.0389, "BSTAR": 0.0, "EPOCH": "2026-03-23T06:36:42"}, {"OBJECT_NAME": "CHINASAT 31 (BEIDOU-1 *)", "NORAD_CAT_ID": 26643, "MEAN_MOTION": 0.99252035, "ECCENTRICITY": 0.0076807, "INCLINATION": 12.4707, "RA_OF_ASC_NODE": 27.9518, "ARG_OF_PERICENTER": 85.31, "MEAN_ANOMALY": 282.0901, "BSTAR": 0.0, "EPOCH": "2026-03-23T18:07:58"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-1 (C06)", "NORAD_CAT_ID": 36828, "MEAN_MOTION": 1.00248068, "ECCENTRICITY": 0.0054955, "INCLINATION": 54.2916, "RA_OF_ASC_NODE": 163.7709, "ARG_OF_PERICENTER": 219.8858, "MEAN_ANOMALY": 238.0541, "BSTAR": 0.0, "EPOCH": "2026-03-19T22:41:52"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-4 (C09)", "NORAD_CAT_ID": 37763, "MEAN_MOTION": 1.00262119, "ECCENTRICITY": 0.0156205, "INCLINATION": 54.5875, "RA_OF_ASC_NODE": 166.457, "ARG_OF_PERICENTER": 231.2289, "MEAN_ANOMALY": 182.9636, "BSTAR": 0.0, "EPOCH": "2026-03-22T20:26:06"}, {"OBJECT_NAME": "BEIDOU-3 M25 (C47)", "NORAD_CAT_ID": 61186, "MEAN_MOTION": 1.86229796, "ECCENTRICITY": 0.0003866, "INCLINATION": 54.5489, "RA_OF_ASC_NODE": 305.2065, "ARG_OF_PERICENTER": 319.7829, "MEAN_ANOMALY": 271.3839, "BSTAR": 0.0, "EPOCH": "2026-03-23T21:22:34"}, {"OBJECT_NAME": "BEIDOU-3 M18 (C37)", "NORAD_CAT_ID": 43707, "MEAN_MOTION": 1.86226297, "ECCENTRICITY": 0.0006137, "INCLINATION": 54.17, "RA_OF_ASC_NODE": 185.2402, "ARG_OF_PERICENTER": 334.8391, "MEAN_ANOMALY": 25.1479, "BSTAR": 0.0, "EPOCH": "2026-03-22T04:27:45"}, {"OBJECT_NAME": "BEIDOU-3 M8 (C28)", "NORAD_CAT_ID": 43108, "MEAN_MOTION": 1.86231036, "ECCENTRICITY": 0.0002692, "INCLINATION": 54.3976, "RA_OF_ASC_NODE": 305.3279, "ARG_OF_PERICENTER": 294.6049, "MEAN_ANOMALY": 65.41, "BSTAR": 0.0, "EPOCH": "2026-03-22T07:25:10"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-5 (C10)", "NORAD_CAT_ID": 37948, "MEAN_MOTION": 1.0026733, "ECCENTRICITY": 0.0109241, "INCLINATION": 47.8347, "RA_OF_ASC_NODE": 272.5964, "ARG_OF_PERICENTER": 220.6724, "MEAN_ANOMALY": 93.5638, "BSTAR": 0.0, "EPOCH": "2026-03-23T20:42:32"}, {"OBJECT_NAME": "BEIDOU-3 M27 (C49)", "NORAD_CAT_ID": 61187, "MEAN_MOTION": 1.86229934, "ECCENTRICITY": 0.0002856, "INCLINATION": 54.5447, "RA_OF_ASC_NODE": 305.209, "ARG_OF_PERICENTER": 27.7133, "MEAN_ANOMALY": 329.452, "BSTAR": 0.0, "EPOCH": "2026-03-23T08:10:02"}, {"OBJECT_NAME": "BEIDOU-3 M4 (C22)", "NORAD_CAT_ID": 43207, "MEAN_MOTION": 1.86230782, "ECCENTRICITY": 0.0007324, "INCLINATION": 56.5782, "RA_OF_ASC_NODE": 65.9614, "ARG_OF_PERICENTER": 353.8516, "MEAN_ANOMALY": 6.1328, "BSTAR": 0.0, "EPOCH": "2026-03-23T19:48:29"}, {"OBJECT_NAME": "BEIDOU-2 G5 (C05)", "NORAD_CAT_ID": 38091, "MEAN_MOTION": 1.00271625, "ECCENTRICITY": 0.0013602, "INCLINATION": 3.3597, "RA_OF_ASC_NODE": 70.1793, "ARG_OF_PERICENTER": 257.0622, "MEAN_ANOMALY": 244.5781, "BSTAR": 0.0, "EPOCH": "2026-03-23T22:06:53"}, {"OBJECT_NAME": "BEIDOU-2 G8 (C01)", "NORAD_CAT_ID": 44231, "MEAN_MOTION": 1.00269782, "ECCENTRICITY": 0.0011425, "INCLINATION": 1.4405, "RA_OF_ASC_NODE": 74.265, "ARG_OF_PERICENTER": 274.4439, "MEAN_ANOMALY": 315.0432, "BSTAR": 0.0, "EPOCH": "2026-03-23T22:31:21"}, {"OBJECT_NAME": "BEIDOU-3 M3 (C21)", "NORAD_CAT_ID": 43208, "MEAN_MOTION": 1.86230975, "ECCENTRICITY": 0.0010098, "INCLINATION": 56.5746, "RA_OF_ASC_NODE": 66.0695, "ARG_OF_PERICENTER": 322.3953, "MEAN_ANOMALY": 37.533, "BSTAR": 0.0, "EPOCH": "2026-03-20T16:04:52"}, {"OBJECT_NAME": "BEIDOU-3 M9 (C29)", "NORAD_CAT_ID": 43245, "MEAN_MOTION": 1.86227975, "ECCENTRICITY": 9.22e-05, "INCLINATION": 54.2711, "RA_OF_ASC_NODE": 302.943, "ARG_OF_PERICENTER": 46.6737, "MEAN_ANOMALY": 313.3705, "BSTAR": 0.0, "EPOCH": "2026-03-23T13:43:55"}, {"OBJECT_NAME": "BEIDOU-3 M24 (C46)", "NORAD_CAT_ID": 44542, "MEAN_MOTION": 1.86229032, "ECCENTRICITY": 0.0008254, "INCLINATION": 54.3973, "RA_OF_ASC_NODE": 185.6054, "ARG_OF_PERICENTER": 16.7457, "MEAN_ANOMALY": 343.2983, "BSTAR": 0.0, "EPOCH": "2026-03-23T20:47:51"}, {"OBJECT_NAME": "COSMO-SKYMED 1", "NORAD_CAT_ID": 31598, "MEAN_MOTION": 14.96639429, "ECCENTRICITY": 0.0001093, "INCLINATION": 97.888, "RA_OF_ASC_NODE": 274.1209, "ARG_OF_PERICENTER": 87.9139, "MEAN_ANOMALY": 272.2208, "BSTAR": 0.00026118, "EPOCH": "2026-03-23T23:20:21"}, {"OBJECT_NAME": "COSMO-SKYMED 2", "NORAD_CAT_ID": 32376, "MEAN_MOTION": 14.8215995, "ECCENTRICITY": 0.0001271, "INCLINATION": 97.8874, "RA_OF_ASC_NODE": 267.7927, "ARG_OF_PERICENTER": 86.8376, "MEAN_ANOMALY": 273.2983, "BSTAR": 6.9612e-05, "EPOCH": "2026-03-23T23:06:21"}, {"OBJECT_NAME": "COSMO-SKYMED 4", "NORAD_CAT_ID": 37216, "MEAN_MOTION": 14.82155896, "ECCENTRICITY": 0.0001497, "INCLINATION": 97.8872, "RA_OF_ASC_NODE": 267.8056, "ARG_OF_PERICENTER": 80.9631, "MEAN_ANOMALY": 279.1752, "BSTAR": 5.1491e-05, "EPOCH": "2026-03-23T23:24:36"}, {"OBJECT_NAME": "COSMO-SKYMED 3", "NORAD_CAT_ID": 33412, "MEAN_MOTION": 15.06373703, "ECCENTRICITY": 0.0015467, "INCLINATION": 97.8399, "RA_OF_ASC_NODE": 300.0703, "ARG_OF_PERICENTER": 190.5997, "MEAN_ANOMALY": 169.4906, "BSTAR": 0.00049172, "EPOCH": "2026-03-23T22:27:03"}, {"OBJECT_NAME": "ISS (ZARYA)", "NORAD_CAT_ID": 25544, "MEAN_MOTION": 15.4850416, "ECCENTRICITY": 0.0006231, "INCLINATION": 51.6344, "RA_OF_ASC_NODE": 2.4037, "ARG_OF_PERICENTER": 225.1303, "MEAN_ANOMALY": 134.918, "BSTAR": 0.00029016, "EPOCH": "2026-03-23T20:10:56"}, {"OBJECT_NAME": "ISS (NAUKA)", "NORAD_CAT_ID": 49044, "MEAN_MOTION": 15.4850416, "ECCENTRICITY": 0.0006231, "INCLINATION": 51.6344, "RA_OF_ASC_NODE": 2.4037, "ARG_OF_PERICENTER": 225.1303, "MEAN_ANOMALY": 134.918, "BSTAR": 0.00029016, "EPOCH": "2026-03-23T20:10:56"}, {"OBJECT_NAME": "SWISSCUBE", "NORAD_CAT_ID": 35932, "MEAN_MOTION": 14.62296671, "ECCENTRICITY": 0.0007869, "INCLINATION": 98.4085, "RA_OF_ASC_NODE": 349.3798, "ARG_OF_PERICENTER": 112.7072, "MEAN_ANOMALY": 247.4962, "BSTAR": 0.0002546, "EPOCH": "2026-03-23T23:36:58"}, {"OBJECT_NAME": "AISSAT 1", "NORAD_CAT_ID": 36797, "MEAN_MOTION": 14.96977358, "ECCENTRICITY": 0.0008642, "INCLINATION": 98.101, "RA_OF_ASC_NODE": 339.017, "ARG_OF_PERICENTER": 352.649, "MEAN_ANOMALY": 7.4603, "BSTAR": 0.00030519, "EPOCH": "2026-03-23T19:07:38"}, {"OBJECT_NAME": "AISSAT 2", "NORAD_CAT_ID": 40075, "MEAN_MOTION": 14.8560182, "ECCENTRICITY": 0.000478, "INCLINATION": 98.3401, "RA_OF_ASC_NODE": 268.4723, "ARG_OF_PERICENTER": 335.0232, "MEAN_ANOMALY": 25.0749, "BSTAR": 0.00040707, "EPOCH": "2023-12-28T11:59:02"}, {"OBJECT_NAME": "ISS OBJECT XK", "NORAD_CAT_ID": 65731, "MEAN_MOTION": 16.39076546, "ECCENTRICITY": 0.0002464, "INCLINATION": 51.6052, "RA_OF_ASC_NODE": 44.2508, "ARG_OF_PERICENTER": 211.8886, "MEAN_ANOMALY": 148.1991, "BSTAR": 0.00075547, "EPOCH": "2026-03-09T22:15:28"}, {"OBJECT_NAME": "ISS (DESTINY)", "NORAD_CAT_ID": 26700, "MEAN_MOTION": 15.4850416, "ECCENTRICITY": 0.0006231, "INCLINATION": 51.6344, "RA_OF_ASC_NODE": 2.4037, "ARG_OF_PERICENTER": 225.1303, "MEAN_ANOMALY": 134.918, "BSTAR": 0.00029016, "EPOCH": "2026-03-23T20:10:56"}, {"OBJECT_NAME": "OUTPOST MISSION 2", "NORAD_CAT_ID": 58334, "MEAN_MOTION": 15.51742842, "ECCENTRICITY": 0.0006631, "INCLINATION": 97.3957, "RA_OF_ASC_NODE": 173.0636, "ARG_OF_PERICENTER": 81.0167, "MEAN_ANOMALY": 279.1836, "BSTAR": 0.00058729, "EPOCH": "2026-03-23T22:49:02"}, {"OBJECT_NAME": "ISS (UNITY)", "NORAD_CAT_ID": 25575, "MEAN_MOTION": 15.4850416, "ECCENTRICITY": 0.0006231, "INCLINATION": 51.6344, "RA_OF_ASC_NODE": 2.4037, "ARG_OF_PERICENTER": 225.1303, "MEAN_ANOMALY": 134.918, "BSTAR": 0.00029016, "EPOCH": "2026-03-23T20:10:56"}, {"OBJECT_NAME": "ISS (ZVEZDA)", "NORAD_CAT_ID": 26400, "MEAN_MOTION": 15.4850416, "ECCENTRICITY": 0.0006231, "INCLINATION": 51.6344, "RA_OF_ASC_NODE": 2.4037, "ARG_OF_PERICENTER": 225.1303, "MEAN_ANOMALY": 134.918, "BSTAR": 0.00029016, "EPOCH": "2026-03-23T20:10:56"}, {"OBJECT_NAME": "ISS OBJECT XU", "NORAD_CAT_ID": 66908, "MEAN_MOTION": 15.74515267, "ECCENTRICITY": 0.0005615, "INCLINATION": 51.6246, "RA_OF_ASC_NODE": 352.5981, "ARG_OF_PERICENTER": 171.79, "MEAN_ANOMALY": 188.3192, "BSTAR": 0.0013662, "EPOCH": "2026-03-23T22:47:26"}, {"OBJECT_NAME": "ISS OBJECT XW", "NORAD_CAT_ID": 66910, "MEAN_MOTION": 15.73130824, "ECCENTRICITY": 0.0005876, "INCLINATION": 51.6238, "RA_OF_ASC_NODE": 352.636, "ARG_OF_PERICENTER": 171.3926, "MEAN_ANOMALY": 188.7174, "BSTAR": 0.0012412, "EPOCH": "2026-03-23T22:56:18"}, {"OBJECT_NAME": "ISS OBJECT XX", "NORAD_CAT_ID": 66911, "MEAN_MOTION": 15.84301586, "ECCENTRICITY": 0.0007849, "INCLINATION": 51.6197, "RA_OF_ASC_NODE": 350.2546, "ARG_OF_PERICENTER": 180.3314, "MEAN_ANOMALY": 179.7686, "BSTAR": 0.0016589, "EPOCH": "2026-03-23T22:35:38"}, {"OBJECT_NAME": "ISS OBJECT XY", "NORAD_CAT_ID": 66912, "MEAN_MOTION": 15.66501049, "ECCENTRICITY": 0.000394, "INCLINATION": 51.6274, "RA_OF_ASC_NODE": 355.835, "ARG_OF_PERICENTER": 170.7309, "MEAN_ANOMALY": 189.376, "BSTAR": 0.0008260100000000001, "EPOCH": "2026-03-23T18:09:48"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 47853, "MEAN_MOTION": 16.41769315, "ECCENTRICITY": 0.0002766, "INCLINATION": 51.6054, "RA_OF_ASC_NODE": 359.3486, "ARG_OF_PERICENTER": 253.9051, "MEAN_ANOMALY": 215.3296, "BSTAR": 0.00025185999999999996, "EPOCH": "2024-03-08T00:53:30"}, {"OBJECT_NAME": "ISS DEB (SPX-26 IPA FSE)", "NORAD_CAT_ID": 55448, "MEAN_MOTION": 16.42868885, "ECCENTRICITY": 0.0004554, "INCLINATION": 51.6104, "RA_OF_ASC_NODE": 65.1026, "ARG_OF_PERICENTER": 338.3161, "MEAN_ANOMALY": 176.0212, "BSTAR": 0.00044128, "EPOCH": "2023-12-23T14:31:26"}, {"OBJECT_NAME": "OUTPOST MISSION 1", "NORAD_CAT_ID": 56226, "MEAN_MOTION": 15.72890214, "ECCENTRICITY": 0.0004432, "INCLINATION": 97.5991, "RA_OF_ASC_NODE": 237.3541, "ARG_OF_PERICENTER": 216.2652, "MEAN_ANOMALY": 143.8311, "BSTAR": 0.00075453, "EPOCH": "2026-03-23T22:00:42"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 56434, "MEAN_MOTION": 16.34589017, "ECCENTRICITY": 0.0007268, "INCLINATION": 51.614, "RA_OF_ASC_NODE": 110.8216, "ARG_OF_PERICENTER": 312.4976, "MEAN_ANOMALY": 47.5434, "BSTAR": 0.0007953900000000001, "EPOCH": "2023-12-17T13:53:44"}, {"OBJECT_NAME": "ISS DEB [SPX-28 IPA FSE]", "NORAD_CAT_ID": 57212, "MEAN_MOTION": 16.42826151, "ECCENTRICITY": 0.0005882, "INCLINATION": 51.6078, "RA_OF_ASC_NODE": 38.878, "ARG_OF_PERICENTER": 256.2665, "MEAN_ANOMALY": 278.8784, "BSTAR": 0.00035698999999999995, "EPOCH": "2024-05-22T10:35:37"}, {"OBJECT_NAME": "ISS DEB", "NORAD_CAT_ID": 58174, "MEAN_MOTION": 16.3176295, "ECCENTRICITY": 0.0009401, "INCLINATION": 51.6127, "RA_OF_ASC_NODE": 333.6363, "ARG_OF_PERICENTER": 246.0894, "MEAN_ANOMALY": 113.9145, "BSTAR": 0.001226, "EPOCH": "2024-03-28T18:25:22"}, {"OBJECT_NAME": "SHIJIAN-16 (SJ-16)", "NORAD_CAT_ID": 39358, "MEAN_MOTION": 14.92409667, "ECCENTRICITY": 0.0015181, "INCLINATION": 74.9727, "RA_OF_ASC_NODE": 158.0458, "ARG_OF_PERICENTER": 75.3179, "MEAN_ANOMALY": 284.969, "BSTAR": 0.00037484000000000004, "EPOCH": "2026-03-23T23:24:05"}, {"OBJECT_NAME": "SHIJIAN-6 01A (SJ-6 01A)", "NORAD_CAT_ID": 28413, "MEAN_MOTION": 15.16699228, "ECCENTRICITY": 0.0008013, "INCLINATION": 97.5958, "RA_OF_ASC_NODE": 109.8865, "ARG_OF_PERICENTER": 76.5614, "MEAN_ANOMALY": 283.6511, "BSTAR": 0.00045877, "EPOCH": "2026-03-23T23:08:50"}, {"OBJECT_NAME": "SHIJIAN-6 02A (SJ-6 02A)", "NORAD_CAT_ID": 29505, "MEAN_MOTION": 15.15908434, "ECCENTRICITY": 0.0004801, "INCLINATION": 97.647, "RA_OF_ASC_NODE": 123.4156, "ARG_OF_PERICENTER": 82.3537, "MEAN_ANOMALY": 277.8241, "BSTAR": 0.00023821, "EPOCH": "2026-03-23T23:17:09"}, {"OBJECT_NAME": "SHIJIAN-30A (SJ-30A)", "NORAD_CAT_ID": 66545, "MEAN_MOTION": 15.15581458, "ECCENTRICITY": 0.0011928, "INCLINATION": 51.797, "RA_OF_ASC_NODE": 219.1888, "ARG_OF_PERICENTER": 341.7646, "MEAN_ANOMALY": 18.2901, "BSTAR": 0.00051455, "EPOCH": "2026-03-23T20:11:54"}, {"OBJECT_NAME": "SHIJIAN-6 02B (SJ-6 02B)", "NORAD_CAT_ID": 29506, "MEAN_MOTION": 15.01070443, "ECCENTRICITY": 0.0013178, "INCLINATION": 97.7083, "RA_OF_ASC_NODE": 112.5855, "ARG_OF_PERICENTER": 358.2425, "MEAN_ANOMALY": 1.875, "BSTAR": 0.00021779000000000001, "EPOCH": "2026-03-23T23:07:44"}, {"OBJECT_NAME": "SHIJIAN-6 03A (SJ-6 03A)", "NORAD_CAT_ID": 33408, "MEAN_MOTION": 15.16257441, "ECCENTRICITY": 0.0010165, "INCLINATION": 97.8512, "RA_OF_ASC_NODE": 113.3165, "ARG_OF_PERICENTER": 311.1345, "MEAN_ANOMALY": 48.9008, "BSTAR": 0.00026581, "EPOCH": "2026-03-23T22:38:55"}, {"OBJECT_NAME": "SHIJIAN-6 03B (SJ-6 03B)", "NORAD_CAT_ID": 33409, "MEAN_MOTION": 15.03963882, "ECCENTRICITY": 0.0018735, "INCLINATION": 97.8637, "RA_OF_ASC_NODE": 101.4196, "ARG_OF_PERICENTER": 333.1255, "MEAN_ANOMALY": 26.8998, "BSTAR": 0.0002544, "EPOCH": "2026-03-23T22:47:52"}, {"OBJECT_NAME": "SHIJIAN-6 04A (SJ-6 04A)", "NORAD_CAT_ID": 37179, "MEAN_MOTION": 15.16857234, "ECCENTRICITY": 0.0019669, "INCLINATION": 97.8369, "RA_OF_ASC_NODE": 103.257, "ARG_OF_PERICENTER": 154.7013, "MEAN_ANOMALY": 205.5186, "BSTAR": 0.00030756, "EPOCH": "2026-03-23T23:03:37"}, {"OBJECT_NAME": "SHIJIAN-6 04B (SJ-6 04B)", "NORAD_CAT_ID": 37180, "MEAN_MOTION": 14.99137808, "ECCENTRICITY": 0.0010005, "INCLINATION": 97.8718, "RA_OF_ASC_NODE": 87.4605, "ARG_OF_PERICENTER": 239.0304, "MEAN_ANOMALY": 120.9936, "BSTAR": 8.427900000000001e-05, "EPOCH": "2026-03-23T22:14:35"}, {"OBJECT_NAME": "SHIJIAN-20 (SJ-20)", "NORAD_CAT_ID": 44910, "MEAN_MOTION": 1.00083072, "ECCENTRICITY": 0.000173, "INCLINATION": 4.6981, "RA_OF_ASC_NODE": 76.0963, "ARG_OF_PERICENTER": 18.8875, "MEAN_ANOMALY": 327.0576, "BSTAR": 0.0, "EPOCH": "2026-03-23T11:52:03"}, {"OBJECT_NAME": "SHIJIAN-17 (SJ-17)", "NORAD_CAT_ID": 41838, "MEAN_MOTION": 0.99867869, "ECCENTRICITY": 9.18e-05, "INCLINATION": 5.5343, "RA_OF_ASC_NODE": 73.6229, "ARG_OF_PERICENTER": 318.0889, "MEAN_ANOMALY": 248.5415, "BSTAR": 0.0, "EPOCH": "2026-03-23T06:26:49"}, {"OBJECT_NAME": "SHIJIAN-6 05A (SJ-6 05A)", "NORAD_CAT_ID": 49961, "MEAN_MOTION": 14.99135922, "ECCENTRICITY": 0.0001623, "INCLINATION": 97.4008, "RA_OF_ASC_NODE": 55.2645, "ARG_OF_PERICENTER": 44.1046, "MEAN_ANOMALY": 316.0307, "BSTAR": 0.00014823, "EPOCH": "2026-03-23T12:00:02"}, {"OBJECT_NAME": "SHIJIAN-6 05B (SJ-6 05B)", "NORAD_CAT_ID": 49962, "MEAN_MOTION": 15.11517098, "ECCENTRICITY": 0.0001799, "INCLINATION": 97.3742, "RA_OF_ASC_NODE": 66.2945, "ARG_OF_PERICENTER": 42.1784, "MEAN_ANOMALY": 317.9585, "BSTAR": 0.00012847, "EPOCH": "2026-03-23T17:24:10"}, {"OBJECT_NAME": "SHIJIAN-23 (SJ-23)", "NORAD_CAT_ID": 55131, "MEAN_MOTION": 1.00479735, "ECCENTRICITY": 0.0005778, "INCLINATION": 3.4976, "RA_OF_ASC_NODE": 79.3818, "ARG_OF_PERICENTER": 262.2916, "MEAN_ANOMALY": 266.0574, "BSTAR": 0.0, "EPOCH": "2026-03-23T18:48:39"}, {"OBJECT_NAME": "SHIJIAN-19 (SJ-19)", "NORAD_CAT_ID": 61444, "MEAN_MOTION": 15.77608314, "ECCENTRICITY": 0.0006944, "INCLINATION": 41.601, "RA_OF_ASC_NODE": 76.8644, "ARG_OF_PERICENTER": 34.8351, "MEAN_ANOMALY": 325.295, "BSTAR": 9.0833e-05, "EPOCH": "2024-10-10T17:00:10"}, {"OBJECT_NAME": "SHIJIAN-25 (SJ-25)", "NORAD_CAT_ID": 62485, "MEAN_MOTION": 1.00275314, "ECCENTRICITY": 0.0050052, "INCLINATION": 4.9114, "RA_OF_ASC_NODE": 61.6532, "ARG_OF_PERICENTER": 167.3694, "MEAN_ANOMALY": 359.6113, "BSTAR": 0.0, "EPOCH": "2026-03-23T18:40:14"}, {"OBJECT_NAME": "SHIJIAN-6 01B (SJ-6 01B)", "NORAD_CAT_ID": 28414, "MEAN_MOTION": 15.05717256, "ECCENTRICITY": 0.0006165, "INCLINATION": 97.6117, "RA_OF_ASC_NODE": 101.9105, "ARG_OF_PERICENTER": 76.9862, "MEAN_ANOMALY": 283.2053, "BSTAR": 0.00032, "EPOCH": "2026-03-23T22:32:48"}, {"OBJECT_NAME": "SHIJIAN-26 (SJ-26)", "NORAD_CAT_ID": 64199, "MEAN_MOTION": 15.22736808, "ECCENTRICITY": 0.0017738, "INCLINATION": 97.4583, "RA_OF_ASC_NODE": 162.0919, "ARG_OF_PERICENTER": 292.6885, "MEAN_ANOMALY": 67.2475, "BSTAR": 0.00026433, "EPOCH": "2026-03-23T23:49:39"}, {"OBJECT_NAME": "SHIJIAN-30B (SJ-30B)", "NORAD_CAT_ID": 66546, "MEAN_MOTION": 15.15533042, "ECCENTRICITY": 0.0009909, "INCLINATION": 51.7954, "RA_OF_ASC_NODE": 219.1773, "ARG_OF_PERICENTER": 344.3147, "MEAN_ANOMALY": 15.7521, "BSTAR": 0.00059473, "EPOCH": "2026-03-23T20:13:56"}, {"OBJECT_NAME": "SHIJIAN-30C (SJ-30C)", "NORAD_CAT_ID": 66547, "MEAN_MOTION": 15.15509191, "ECCENTRICITY": 0.0010193, "INCLINATION": 51.7963, "RA_OF_ASC_NODE": 219.1922, "ARG_OF_PERICENTER": 336.8046, "MEAN_ANOMALY": 23.2469, "BSTAR": 0.00058816, "EPOCH": "2026-03-23T20:14:51"}, {"OBJECT_NAME": "KONDOR-FKA NO. 1", "NORAD_CAT_ID": 56756, "MEAN_MOTION": 15.19762718, "ECCENTRICITY": 0.0001707, "INCLINATION": 97.4397, "RA_OF_ASC_NODE": 278.7441, "ARG_OF_PERICENTER": 86.4071, "MEAN_ANOMALY": 273.7359, "BSTAR": 0.00024218999999999998, "EPOCH": "2026-03-23T22:51:36"}, {"OBJECT_NAME": "KONDOR-FKA NO. 2", "NORAD_CAT_ID": 62138, "MEAN_MOTION": 15.1972016, "ECCENTRICITY": 0.0001693, "INCLINATION": 97.4341, "RA_OF_ASC_NODE": 287.5911, "ARG_OF_PERICENTER": 91.2839, "MEAN_ANOMALY": 268.859, "BSTAR": 0.00024024999999999999, "EPOCH": "2026-03-23T21:52:23"}, {"OBJECT_NAME": "CSO-3", "NORAD_CAT_ID": 63156, "MEAN_MOTION": 14.34022377, "ECCENTRICITY": 0.0001932, "INCLINATION": 98.606, "RA_OF_ASC_NODE": 14.3262, "ARG_OF_PERICENTER": 17.427, "MEAN_ANOMALY": 342.6981, "BSTAR": 0.00016071, "EPOCH": "2025-03-13T22:37:36"}, {"OBJECT_NAME": "GEOEYE 1", "NORAD_CAT_ID": 33331, "MEAN_MOTION": 14.64784167, "ECCENTRICITY": 0.0002966, "INCLINATION": 98.1183, "RA_OF_ASC_NODE": 157.965, "ARG_OF_PERICENTER": 344.0113, "MEAN_ANOMALY": 16.0996, "BSTAR": 0.00011465000000000001, "EPOCH": "2026-03-23T23:07:28"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-3 (C08)", "NORAD_CAT_ID": 37384, "MEAN_MOTION": 1.00288473, "ECCENTRICITY": 0.0034463, "INCLINATION": 62.2966, "RA_OF_ASC_NODE": 40.9675, "ARG_OF_PERICENTER": 189.7809, "MEAN_ANOMALY": 351.0224, "BSTAR": 0.0, "EPOCH": "2026-03-22T19:37:21"}, {"OBJECT_NAME": "BEIDOU-3S IGSO-1S (C31)", "NORAD_CAT_ID": 40549, "MEAN_MOTION": 1.00268843, "ECCENTRICITY": 0.0036261, "INCLINATION": 49.3559, "RA_OF_ASC_NODE": 296.4949, "ARG_OF_PERICENTER": 190.1236, "MEAN_ANOMALY": 255.118, "BSTAR": 0.0, "EPOCH": "2026-03-23T06:58:51"}, {"OBJECT_NAME": "BEIDOU-3S IGSO-2S (C56)", "NORAD_CAT_ID": 40938, "MEAN_MOTION": 1.00254296, "ECCENTRICITY": 0.0061515, "INCLINATION": 49.4373, "RA_OF_ASC_NODE": 260.4556, "ARG_OF_PERICENTER": 187.1638, "MEAN_ANOMALY": 17.4903, "BSTAR": 0.0, "EPOCH": "2026-01-27T16:14:57"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-6 (C13)", "NORAD_CAT_ID": 41434, "MEAN_MOTION": 1.00274108, "ECCENTRICITY": 0.0058883, "INCLINATION": 60.1143, "RA_OF_ASC_NODE": 38.8864, "ARG_OF_PERICENTER": 233.5094, "MEAN_ANOMALY": 327.1221, "BSTAR": 0.0, "EPOCH": "2026-03-23T21:29:38"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-7 (C16)", "NORAD_CAT_ID": 43539, "MEAN_MOTION": 1.00277421, "ECCENTRICITY": 0.0101497, "INCLINATION": 55.1843, "RA_OF_ASC_NODE": 163.8079, "ARG_OF_PERICENTER": 237.0388, "MEAN_ANOMALY": 126.0236, "BSTAR": 0.0, "EPOCH": "2026-03-23T15:31:21"}, {"OBJECT_NAME": "BEIDOU-2 IGSO-2 (C07)", "NORAD_CAT_ID": 37256, "MEAN_MOTION": 1.00254021, "ECCENTRICITY": 0.0047798, "INCLINATION": 47.7058, "RA_OF_ASC_NODE": 272.9336, "ARG_OF_PERICENTER": 212.7858, "MEAN_ANOMALY": 135.9365, "BSTAR": 0.0, "EPOCH": "2026-03-19T22:32:24"}, {"OBJECT_NAME": "SBIRS GEO-1 (USA 230)", "NORAD_CAT_ID": 37481, "MEAN_MOTION": 1.00272492, "ECCENTRICITY": 0.0002319, "INCLINATION": 4.3494, "RA_OF_ASC_NODE": 53.1046, "ARG_OF_PERICENTER": 309.5877, "MEAN_ANOMALY": 201.7295, "BSTAR": 0.0, "EPOCH": "2026-03-23T22:04:54"}, {"OBJECT_NAME": "SBIRS GEO-2 (USA 241)", "NORAD_CAT_ID": 39120, "MEAN_MOTION": 1.00271132, "ECCENTRICITY": 0.0002346, "INCLINATION": 4.3032, "RA_OF_ASC_NODE": 52.4044, "ARG_OF_PERICENTER": 308.8878, "MEAN_ANOMALY": 196.8605, "BSTAR": 0.0, "EPOCH": "2026-03-23T04:29:08"}, {"OBJECT_NAME": "SBIRS GEO-4 (USA 273)", "NORAD_CAT_ID": 41937, "MEAN_MOTION": 1.00271844, "ECCENTRICITY": 0.0002298, "INCLINATION": 2.0828, "RA_OF_ASC_NODE": 40.6763, "ARG_OF_PERICENTER": 313.3652, "MEAN_ANOMALY": 305.5989, "BSTAR": 0.0, "EPOCH": "2026-03-23T06:31:56"}, {"OBJECT_NAME": "SBIRS GEO-3 (USA 282)", "NORAD_CAT_ID": 43162, "MEAN_MOTION": 1.00271765, "ECCENTRICITY": 0.0002211, "INCLINATION": 2.1323, "RA_OF_ASC_NODE": 7.8827, "ARG_OF_PERICENTER": 345.1992, "MEAN_ANOMALY": 319.3096, "BSTAR": 0.0, "EPOCH": "2026-03-23T19:22:02"}, {"OBJECT_NAME": "SBIRS GEO-5 (USA 315)", "NORAD_CAT_ID": 48618, "MEAN_MOTION": 1.00271206, "ECCENTRICITY": 0.0001295, "INCLINATION": 5.2132, "RA_OF_ASC_NODE": 328.4733, "ARG_OF_PERICENTER": 17.0462, "MEAN_ANOMALY": 215.89, "BSTAR": 0.0, "EPOCH": "2026-03-23T19:05:24"}, {"OBJECT_NAME": "SBIRS GEO-6 (USA 336)", "NORAD_CAT_ID": 53355, "MEAN_MOTION": 1.00271105, "ECCENTRICITY": 0.0002144, "INCLINATION": 3.405, "RA_OF_ASC_NODE": 316.1189, "ARG_OF_PERICENTER": 33.8434, "MEAN_ANOMALY": 307.9026, "BSTAR": 0.0, "EPOCH": "2026-03-23T22:29:51"}, {"OBJECT_NAME": "GSAT0201 (GALILEO 5)", "NORAD_CAT_ID": 40128, "MEAN_MOTION": 1.85519959, "ECCENTRICITY": 0.1660183, "INCLINATION": 48.9731, "RA_OF_ASC_NODE": 277.1098, "ARG_OF_PERICENTER": 175.8997, "MEAN_ANOMALY": 185.6527, "BSTAR": 0.0, "EPOCH": "2026-03-19T09:45:05"}, {"OBJECT_NAME": "GSAT0202 (GALILEO 6)", "NORAD_CAT_ID": 40129, "MEAN_MOTION": 1.85520619, "ECCENTRICITY": 0.1661928, "INCLINATION": 48.9912, "RA_OF_ASC_NODE": 276.0296, "ARG_OF_PERICENTER": 176.7904, "MEAN_ANOMALY": 184.4267, "BSTAR": 0.0, "EPOCH": "2026-03-23T10:26:29"}, {"OBJECT_NAME": "GSAT0205 (GALILEO 9)", "NORAD_CAT_ID": 40889, "MEAN_MOTION": 1.674108, "ECCENTRICITY": 0.0359599, "INCLINATION": 53.6403, "RA_OF_ASC_NODE": 225.6057, "ARG_OF_PERICENTER": 59.9658, "MEAN_ANOMALY": 303.5577, "BSTAR": 0.0, "EPOCH": "2026-02-12T22:54:15"}, {"OBJECT_NAME": "GSAT0221 (GALILEO 25)", "NORAD_CAT_ID": 43564, "MEAN_MOTION": 1.70475348, "ECCENTRICITY": 0.0004608, "INCLINATION": 57.196, "RA_OF_ASC_NODE": 344.0989, "ARG_OF_PERICENTER": 314.7446, "MEAN_ANOMALY": 45.2586, "BSTAR": 0.0, "EPOCH": "2026-03-23T10:10:02"}, {"OBJECT_NAME": "GSAT0101 (GALILEO-PFM)", "NORAD_CAT_ID": 37846, "MEAN_MOTION": 1.70475565, "ECCENTRICITY": 0.0004066, "INCLINATION": 57.0214, "RA_OF_ASC_NODE": 344.1116, "ARG_OF_PERICENTER": 10.9905, "MEAN_ANOMALY": 349.0607, "BSTAR": 0.0, "EPOCH": "2026-03-22T14:53:28"}, {"OBJECT_NAME": "GSAT0103 (GALILEO-FM3)", "NORAD_CAT_ID": 38857, "MEAN_MOTION": 1.7047331, "ECCENTRICITY": 0.0005761, "INCLINATION": 55.7545, "RA_OF_ASC_NODE": 103.8861, "ARG_OF_PERICENTER": 288.83, "MEAN_ANOMALY": 71.1565, "BSTAR": 0.0, "EPOCH": "2026-03-23T20:01:00"}, {"OBJECT_NAME": "GSAT0219 (GALILEO 23)", "NORAD_CAT_ID": 43566, "MEAN_MOTION": 1.70475271, "ECCENTRICITY": 0.000465, "INCLINATION": 57.1997, "RA_OF_ASC_NODE": 344.1237, "ARG_OF_PERICENTER": 321.1452, "MEAN_ANOMALY": 38.8637, "BSTAR": 0.0, "EPOCH": "2026-03-22T16:35:00"}, {"OBJECT_NAME": "GSAT0220 (GALILEO 24)", "NORAD_CAT_ID": 43567, "MEAN_MOTION": 1.70475086, "ECCENTRICITY": 0.0004648, "INCLINATION": 57.1996, "RA_OF_ASC_NODE": 344.1659, "ARG_OF_PERICENTER": 316.9839, "MEAN_ANOMALY": 43.0239, "BSTAR": 0.0, "EPOCH": "2026-03-21T03:39:07"}, {"OBJECT_NAME": "GSAT0208 (GALILEO 11)", "NORAD_CAT_ID": 41175, "MEAN_MOTION": 1.70474159, "ECCENTRICITY": 0.0004625, "INCLINATION": 55.7697, "RA_OF_ASC_NODE": 103.6202, "ARG_OF_PERICENTER": 324.8735, "MEAN_ANOMALY": 35.1483, "BSTAR": 0.0, "EPOCH": "2026-03-23T00:44:50"}, {"OBJECT_NAME": "GSAT0212 (GALILEO 16)", "NORAD_CAT_ID": 41860, "MEAN_MOTION": 1.70474438, "ECCENTRICITY": 0.0004129, "INCLINATION": 55.4393, "RA_OF_ASC_NODE": 103.4982, "ARG_OF_PERICENTER": 335.5531, "MEAN_ANOMALY": 24.4876, "BSTAR": 0.0, "EPOCH": "2026-03-21T18:49:44"}, {"OBJECT_NAME": "GSAT0226 (GALILEO 31)", "NORAD_CAT_ID": 61183, "MEAN_MOTION": 1.70473983, "ECCENTRICITY": 0.00016, "INCLINATION": 55.2123, "RA_OF_ASC_NODE": 223.6784, "ARG_OF_PERICENTER": 149.9488, "MEAN_ANOMALY": 32.6488, "BSTAR": 0.0, "EPOCH": "2026-03-23T03:49:27"}, {"OBJECT_NAME": "GSAT0213 (GALILEO 17)", "NORAD_CAT_ID": 41861, "MEAN_MOTION": 1.70474684, "ECCENTRICITY": 0.0005658, "INCLINATION": 55.4417, "RA_OF_ASC_NODE": 103.4794, "ARG_OF_PERICENTER": 296.0774, "MEAN_ANOMALY": 63.918, "BSTAR": 0.0, "EPOCH": "2026-03-22T17:43:28"}, {"OBJECT_NAME": "GSAT0233 (GALILEO 33)", "NORAD_CAT_ID": 67160, "MEAN_MOTION": 1.70474578, "ECCENTRICITY": 0.0003011, "INCLINATION": 54.3935, "RA_OF_ASC_NODE": 105.2952, "ARG_OF_PERICENTER": 207.0582, "MEAN_ANOMALY": 359.4109, "BSTAR": 0.0, "EPOCH": "2026-02-13T12:00:00"}, {"OBJECT_NAME": "GSAT0234 (GALILEO 34)", "NORAD_CAT_ID": 67162, "MEAN_MOTION": 1.70474553, "ECCENTRICITY": 0.0002533, "INCLINATION": 54.2773, "RA_OF_ASC_NODE": 104.5763, "ARG_OF_PERICENTER": 232.1866, "MEAN_ANOMALY": 126.9487, "BSTAR": 0.0, "EPOCH": "2026-03-23T06:57:26"}, {"OBJECT_NAME": "GSAT0214 (GALILEO 18)", "NORAD_CAT_ID": 41862, "MEAN_MOTION": 1.70474567, "ECCENTRICITY": 0.0004661, "INCLINATION": 55.4398, "RA_OF_ASC_NODE": 103.5008, "ARG_OF_PERICENTER": 300.0353, "MEAN_ANOMALY": 59.9793, "BSTAR": 0.0, "EPOCH": "2026-03-21T17:04:29"}, {"OBJECT_NAME": "GSAT0102 (GALILEO-FM2)", "NORAD_CAT_ID": 37847, "MEAN_MOTION": 1.70475602, "ECCENTRICITY": 0.0005245, "INCLINATION": 57.0223, "RA_OF_ASC_NODE": 344.1222, "ARG_OF_PERICENTER": 7.3999, "MEAN_ANOMALY": 170.4606, "BSTAR": 0.0, "EPOCH": "2026-03-22T05:59:10"}, {"OBJECT_NAME": "GSAT0215 (GALILEO 19)", "NORAD_CAT_ID": 43055, "MEAN_MOTION": 1.70474468, "ECCENTRICITY": 4.38e-05, "INCLINATION": 55.1018, "RA_OF_ASC_NODE": 223.8654, "ARG_OF_PERICENTER": 309.2989, "MEAN_ANOMALY": 50.6521, "BSTAR": 0.0, "EPOCH": "2026-03-23T08:59:29"}, {"OBJECT_NAME": "GSAT0216 (GALILEO 20)", "NORAD_CAT_ID": 43056, "MEAN_MOTION": 1.70474604, "ECCENTRICITY": 0.0001803, "INCLINATION": 55.1032, "RA_OF_ASC_NODE": 223.9381, "ARG_OF_PERICENTER": 305.6583, "MEAN_ANOMALY": 54.2674, "BSTAR": 0.0, "EPOCH": "2026-03-20T17:35:24"}, {"OBJECT_NAME": "GALILEO104 [GAL]", "NORAD_CAT_ID": 38858, "MEAN_MOTION": 1.64592169, "ECCENTRICITY": 0.0001481, "INCLINATION": 55.4156, "RA_OF_ASC_NODE": 121.7581, "ARG_OF_PERICENTER": 313.3657, "MEAN_ANOMALY": 182.5612, "BSTAR": 0.0, "EPOCH": "2024-06-18T00:14:41"}, {"OBJECT_NAME": "GSAT0217 (GALILEO 21)", "NORAD_CAT_ID": 43057, "MEAN_MOTION": 1.70474514, "ECCENTRICITY": 0.0001863, "INCLINATION": 55.1014, "RA_OF_ASC_NODE": 223.8815, "ARG_OF_PERICENTER": 342.1923, "MEAN_ANOMALY": 17.7532, "BSTAR": 0.0, "EPOCH": "2026-03-22T17:07:21"}, {"OBJECT_NAME": "LUCH (OLYMP-K 1)", "NORAD_CAT_ID": 40258, "MEAN_MOTION": 0.99116915, "ECCENTRICITY": 0.0003121, "INCLINATION": 1.4978, "RA_OF_ASC_NODE": 84.5508, "ARG_OF_PERICENTER": 187.5711, "MEAN_ANOMALY": 175.9774, "BSTAR": 0.0, "EPOCH": "2026-01-14T19:27:12"}, {"OBJECT_NAME": "LUCH-5X (OLYMP-K 2)", "NORAD_CAT_ID": 55841, "MEAN_MOTION": 1.00274188, "ECCENTRICITY": 0.0001356, "INCLINATION": 0.0307, "RA_OF_ASC_NODE": 88.4804, "ARG_OF_PERICENTER": 89.6005, "MEAN_ANOMALY": 335.7043, "BSTAR": 0.0, "EPOCH": "2026-03-23T18:02:13"}, {"OBJECT_NAME": "LUCH (OLYMP-K 1) DEB", "NORAD_CAT_ID": 67745, "MEAN_MOTION": 0.97993456, "ECCENTRICITY": 0.0520769, "INCLINATION": 1.5795, "RA_OF_ASC_NODE": 85.6956, "ARG_OF_PERICENTER": 329.0115, "MEAN_ANOMALY": 28.4425, "BSTAR": 0.0, "EPOCH": "2026-02-26T11:11:16"}, {"OBJECT_NAME": "LUCH", "NORAD_CAT_ID": 23426, "MEAN_MOTION": 1.00178339, "ECCENTRICITY": 0.000498, "INCLINATION": 14.754, "RA_OF_ASC_NODE": 355.268, "ARG_OF_PERICENTER": 227.8824, "MEAN_ANOMALY": 132.0626, "BSTAR": 0.0, "EPOCH": "2026-03-22T16:13:05"}, {"OBJECT_NAME": "LUCH-1", "NORAD_CAT_ID": 23680, "MEAN_MOTION": 1.00266945, "ECCENTRICITY": 0.0004566, "INCLINATION": 15.0233, "RA_OF_ASC_NODE": 359.9227, "ARG_OF_PERICENTER": 143.0726, "MEAN_ANOMALY": 82.0987, "BSTAR": 0.0, "EPOCH": "2026-03-23T21:52:42"}, {"OBJECT_NAME": "LUCH 5A (SDCM/PRN 140)", "NORAD_CAT_ID": 37951, "MEAN_MOTION": 1.00268987, "ECCENTRICITY": 0.0003271, "INCLINATION": 8.5369, "RA_OF_ASC_NODE": 75.0506, "ARG_OF_PERICENTER": 264.3304, "MEAN_ANOMALY": 271.2766, "BSTAR": 0.0, "EPOCH": "2026-03-23T17:29:46"}, {"OBJECT_NAME": "LUCH 5B (SDCM/PRN 125)", "NORAD_CAT_ID": 38977, "MEAN_MOTION": 1.00270128, "ECCENTRICITY": 0.0003285, "INCLINATION": 10.3004, "RA_OF_ASC_NODE": 50.7067, "ARG_OF_PERICENTER": 223.8404, "MEAN_ANOMALY": 217.3707, "BSTAR": 0.0, "EPOCH": "2026-03-23T21:47:33"}, {"OBJECT_NAME": "LUCH 5V (SDCM/PRN 141)", "NORAD_CAT_ID": 39727, "MEAN_MOTION": 1.00273848, "ECCENTRICITY": 0.0002381, "INCLINATION": 4.9219, "RA_OF_ASC_NODE": 70.5767, "ARG_OF_PERICENTER": 297.8759, "MEAN_ANOMALY": 241.9596, "BSTAR": 0.0, "EPOCH": "2026-03-23T22:17:07"}, {"OBJECT_NAME": "LUCH DEB", "NORAD_CAT_ID": 44582, "MEAN_MOTION": 1.00696515, "ECCENTRICITY": 0.0017908, "INCLINATION": 14.6036, "RA_OF_ASC_NODE": 354.6964, "ARG_OF_PERICENTER": 88.2342, "MEAN_ANOMALY": 79.6774, "BSTAR": 0.0, "EPOCH": "2026-03-23T15:30:36"}, {"OBJECT_NAME": "PLANETUM1", "NORAD_CAT_ID": 52738, "MEAN_MOTION": 16.30442171, "ECCENTRICITY": 0.0009881, "INCLINATION": 97.5537, "RA_OF_ASC_NODE": 110.7236, "ARG_OF_PERICENTER": 292.3042, "MEAN_ANOMALY": 67.7204, "BSTAR": 0.0013242000000000002, "EPOCH": "2024-11-28T20:09:41"}, {"OBJECT_NAME": "PAZ", "NORAD_CAT_ID": 43215, "MEAN_MOTION": 15.19138617, "ECCENTRICITY": 0.0001723, "INCLINATION": 97.4458, "RA_OF_ASC_NODE": 91.5363, "ARG_OF_PERICENTER": 86.6125, "MEAN_ANOMALY": 273.5306, "BSTAR": 7.513e-05, "EPOCH": "2026-03-23T22:32:36"}, {"OBJECT_NAME": "SPOT 5", "NORAD_CAT_ID": 27421, "MEAN_MOTION": 14.54673966, "ECCENTRICITY": 0.0129757, "INCLINATION": 97.996, "RA_OF_ASC_NODE": 125.0272, "ARG_OF_PERICENTER": 182.7161, "MEAN_ANOMALY": 177.3348, "BSTAR": 9.466700000000001e-05, "EPOCH": "2026-03-23T09:35:04"}, {"OBJECT_NAME": "SPOT 6", "NORAD_CAT_ID": 38755, "MEAN_MOTION": 14.58552658, "ECCENTRICITY": 0.0001525, "INCLINATION": 98.2187, "RA_OF_ASC_NODE": 151.2043, "ARG_OF_PERICENTER": 88.0003, "MEAN_ANOMALY": 272.1371, "BSTAR": 9.0367e-05, "EPOCH": "2026-03-23T22:12:50"}, {"OBJECT_NAME": "SPOT 7", "NORAD_CAT_ID": 40053, "MEAN_MOTION": 14.60877471, "ECCENTRICITY": 0.0001565, "INCLINATION": 98.0709, "RA_OF_ASC_NODE": 147.571, "ARG_OF_PERICENTER": 96.2759, "MEAN_ANOMALY": 263.862, "BSTAR": 0.0001306, "EPOCH": "2026-03-23T21:51:06"}] \ No newline at end of file diff --git a/backend/data/tracked_names.json b/backend/data/tracked_names.json index ec9636e4..647f5538 100644 --- a/backend/data/tracked_names.json +++ b/backend/data/tracked_names.json @@ -212,6 +212,22 @@ "name": "Roman Abramovich", "category": "People" }, + { + "name": "Ron DeSantis", + "category": "Government" + }, + { + "name": "Oleg Deripaska", + "category": "People" + }, + { + "name": "Arkady Rotenberg", + "category": "People" + }, + { + "name": "Albert Avdolyan", + "category": "People" + }, { "name": "JR Motorsports", "category": "Sports" @@ -2409,7 +2425,7 @@ "category": "Business" }, { - "name": "Andr\u00e9 Esteves", + "name": "André Esteves", "category": "People" }, { @@ -2529,7 +2545,7 @@ "category": "People" }, { - "name": "Pap\u00e9 Machinery", + "name": "Papé Machinery", "category": "Business" }, { @@ -2833,7 +2849,7 @@ "category": "Business" }, { - "name": "Alberto Rodr\u00edguez Baldi (Baldi Hot Springs Hotel)", + "name": "Alberto Rodríguez Baldi (Baldi Hot Springs Hotel)", "category": "People" }, { @@ -2859,6 +2875,34 @@ { "name": "Jay Penske", "category": "People" + }, + { + "name": "Travis Scott", + "category": "Celebrity" + }, + { + "name": "Justin Bieber", + "category": "Celebrity" + }, + { + "name": "Saudi Royal Flight", + "category": "Government" + }, + { + "name": "UK Royal Family (RAF)", + "category": "Government" + }, + { + "name": "France Presidency (COTAM)", + "category": "Government" + }, + { + "name": "Germany Chancellery (Flugbereitschaft)", + "category": "Government" + }, + { + "name": "Japan Prime Minister", + "category": "Government" } ], "details": { @@ -2888,8 +2932,16 @@ "Elon Musk": { "category": "People", "registrations": [ - "N628TS" - ] + "N628TS", + "N8628", + "N8628T", + "N272BG", + "N502SX" + ], + "socials": { + "twitter": "elonmusk", + "instagram": "elonmusk" + } }, "Dallas Mavericks": { "category": "Sports", @@ -2901,7 +2953,11 @@ "category": "Celebrity", "registrations": [ "N313AR" - ] + ], + "socials": { + "twitter": "AROD", + "instagram": "arod" + } }, "State of Alabama": { "category": "State/Law", @@ -2967,8 +3023,16 @@ "registrations": [ "N897GV", "N887GV", - "N608GV" - ] + "N608GV", + "N887WM", + "N194WM", + "N754QS", + "N769QS" + ], + "socials": { + "twitter": "BillGates", + "instagram": "thisisbillgates" + } }, "Arizona Cardinals Football": { "category": "Sports", @@ -2981,7 +3045,11 @@ "category": "Celebrity", "registrations": [ "N703BG" - ] + ], + "socials": { + "twitter": "blakeshelton", + "instagram": "blakeshelton" + } }, "State of Alaska": { "category": "State/Law", @@ -3103,7 +3171,10 @@ "registrations": [ "N459WM", "N459BL" - ] + ], + "socials": { + "instagram": "melindafrenchgates" + } }, "Patriots": { "category": "Sports", @@ -3116,7 +3187,11 @@ "category": "Celebrity", "registrations": [ "N701DB" - ] + ], + "socials": { + "twitter": "DanBilzerian", + "instagram": "danbilzerian" + } }, "State of Arizona": { "category": "State/Law", @@ -3176,8 +3251,12 @@ "category": "People", "registrations": [ "N817GS", - "N417C" - ] + "N417C", + "N1LY" + ], + "socials": { + "twitter": "larryellison" + } }, "Colts": { "category": "Sports", @@ -3193,7 +3272,11 @@ "registrations": [ "N800JM", "N4DP" - ] + ], + "socials": { + "twitter": "DrPhil", + "instagram": "drphil" + } }, "State of Arkansas": { "category": "State/Law", @@ -3252,8 +3335,14 @@ "category": "People", "registrations": [ "N921MT", - "N718MC" - ] + "N718MC", + "N801DM", + "N123FT" + ], + "socials": { + "twitter": "mcuban", + "instagram": "markcuban" + } }, "Dallas Cowboys": { "category": "Sports", @@ -3264,8 +3353,13 @@ "Taylor Swift": { "category": "Celebrity", "registrations": [ - "N621MM" - ] + "N621MM", + "N898TS" + ], + "socials": { + "twitter": "taylorswift13", + "instagram": "taylorswift" + } }, "State of California": { "category": "State/Law", @@ -3367,6 +3461,37 @@ "P4-MES", "LX-LUX", "P4-BDL" + ], + "socials": { + "instagram": "romanabramovich" + } + }, + "Ron DeSantis": { + "category": "Government", + "registrations": [ + "N943FL" + ], + "socials": { + "twitter": "RonDeSantis", + "instagram": "rondesantis" + } + }, + "Oleg Deripaska": { + "category": "People", + "registrations": [ + "RA-09618" + ] + }, + "Arkady Rotenberg": { + "category": "People", + "registrations": [ + "RA-73555" + ] + }, + "Albert Avdolyan": { + "category": "People", + "registrations": [ + "T7-7AA" ] }, "JR Motorsports": { @@ -3383,7 +3508,11 @@ "registrations": [ "N767CJ", "N757CJ" - ] + ], + "socials": { + "twitter": "Drake", + "instagram": "champagnepapi" + } }, "State of Colorado": { "category": "State/Law", @@ -3521,7 +3650,11 @@ "category": "Celebrity", "registrations": [ "N305DG" - ] + ], + "socials": { + "twitter": "FloydMayweather", + "instagram": "floydmayweather" + } }, "State of Delaware": { "category": "State/Law", @@ -3622,7 +3755,11 @@ "category": "Celebrity", "registrations": [ "N810GT" - ] + ], + "socials": { + "twitter": "garthbrooks", + "instagram": "garthbrooks" + } }, "State of Florida": { "category": "State/Law", @@ -3703,7 +3840,10 @@ "category": "Celebrity", "registrations": [ "N518GS" - ] + ], + "socials": { + "instagram": "georgestrait" + } }, "State of Georgia": { "category": "State/Law", @@ -3787,7 +3927,11 @@ "registrations": [ "N247GC", "N880GC" - ] + ], + "socials": { + "twitter": "GrantCardone", + "instagram": "grantcardone" + } }, "Chase Elliot": { "category": "Sports", @@ -3799,7 +3943,10 @@ "category": "Celebrity", "registrations": [ "N6GU" - ] + ], + "socials": { + "instagram": "harrisonford" + } }, "DLR - German Aerospace Center": { "category": "Other", @@ -3855,7 +4002,10 @@ "N138EM", "N29UB", "N512XA" - ] + ], + "socials": { + "twitter": "rooky1" + } }, "Brad Keselowski": { "category": "Sports", @@ -3868,7 +4018,11 @@ "category": "Celebrity", "registrations": [ "N714JB" - ] + ], + "socials": { + "twitter": "JBALVIN", + "instagram": "jbalvin" + } }, "State of Idaho": { "category": "State/Law", @@ -3930,7 +4084,11 @@ "category": "Celebrity", "registrations": [ "N688JC" - ] + ], + "socials": { + "twitter": "EyeOfJackieChan", + "instagram": "jackiechan" + } }, "State of Illinois": { "category": "State/Law", @@ -4001,7 +4159,10 @@ "category": "People", "registrations": [ "VP-CZM" - ] + ], + "socials": { + "twitter": "jackma" + } }, "Dale Earnhardt Family": { "category": "Sports", @@ -4016,7 +4177,11 @@ "category": "Celebrity", "registrations": [ "N845JS" - ] + ], + "socials": { + "twitter": "JerrySeinfeld", + "instagram": "jerryseinfeld" + } }, "State of Indiana": { "category": "State/Law", @@ -4098,7 +4263,10 @@ "category": "Celebrity", "registrations": [ "N162JC" - ] + ], + "socials": { + "twitter": "JimCarrey" + } }, "State of Iowa": { "category": "State/Law", @@ -4153,7 +4321,11 @@ "category": "People", "registrations": [ "N831FR" - ] + ], + "socials": { + "twitter": "kimbal", + "instagram": "kimbalmusk" + } }, "Denny Hamlin Racing": { "category": "Sports", @@ -4227,7 +4399,10 @@ "category": "People", "registrations": [ "N113CS" - ] + ], + "socials": { + "twitter": "schwarzman" + } }, "Hendrick Motorsports": { "category": "Sports", @@ -4246,7 +4421,11 @@ "N500CE", "N905FJ", "N327JT" - ] + ], + "socials": { + "twitter": "johntravolta", + "instagram": "johntravolta" + } }, "State of Kentucky": { "category": "State/Law", @@ -4306,7 +4485,10 @@ "category": "People", "registrations": [ "N102WG" - ] + ], + "socials": { + "twitter": "SteveWitkoff" + } }, "Stewart-Haas Racing": { "category": "Sports", @@ -4319,7 +4501,10 @@ "category": "Celebrity", "registrations": [ "N768JJ" - ] + ], + "socials": { + "instagram": "julioiglesias" + } }, "State of Louisiana": { "category": "State/Law", @@ -4400,7 +4585,10 @@ "registrations": [ "N238MH", "N138GL" - ] + ], + "socials": { + "instagram": "georgehlucas" + } }, "Penske Racing": { "category": "Sports", @@ -4412,7 +4600,11 @@ "category": "Celebrity", "registrations": [ "N510CX" - ] + ], + "socials": { + "twitter": "KeithUrban", + "instagram": "keithurban" + } }, "State of Maine": { "category": "State/Law", @@ -4474,7 +4666,11 @@ "category": "Celebrity", "registrations": [ "N7KC" - ] + ], + "socials": { + "twitter": "kennychesney", + "instagram": "kennychesney" + } }, "State of Maryland": { "category": "State/Law", @@ -4541,7 +4737,11 @@ "category": "People", "registrations": [ "N32MJ" - ] + ], + "socials": { + "twitter": "MagicJohnson", + "instagram": "magicjohnson" + } }, "Victory Aviation (NASCAR)": { "category": "Sports", @@ -4562,7 +4762,11 @@ "category": "Celebrity", "registrations": [ "N71KR" - ] + ], + "socials": { + "twitter": "KidRock", + "instagram": "kidrock" + } }, "State of Massachusetts": { "category": "State/Law", @@ -4624,7 +4828,11 @@ "category": "Celebrity", "registrations": [ "N1980K" - ] + ], + "socials": { + "twitter": "KimKardashian", + "instagram": "kimkardashian" + } }, "State of Michigan": { "category": "State/Law", @@ -4687,7 +4895,11 @@ "category": "Celebrity", "registrations": [ "N810KJ" - ] + ], + "socials": { + "twitter": "KylieJenner", + "instagram": "kyliejenner" + } }, "State of Minnesota": { "category": "State/Law", @@ -4762,7 +4974,11 @@ "category": "Celebrity", "registrations": [ "N69FH" - ] + ], + "socials": { + "twitter": "LukeBryanOnline", + "instagram": "lukebryan" + } }, "State of Mississippi": { "category": "State/Law", @@ -4808,7 +5024,10 @@ "category": "People", "registrations": [ "N19HT" - ] + ], + "socials": { + "instagram": "philhknight" + } }, "Las Vegas Golden Knights": { "category": "Sports", @@ -4821,7 +5040,11 @@ "registrations": [ "N989DM", "N143MW" - ] + ], + "socials": { + "twitter": "markwahlberg", + "instagram": "markwahlberg" + } }, "State of Missouri": { "category": "State/Law", @@ -4878,7 +5101,11 @@ "category": "People", "registrations": [ "N1759C" - ] + ], + "socials": { + "twitter": "RickCaruso", + "instagram": "rickcaruso" + } }, "Sergio Garcia": { "category": "Sports", @@ -4890,7 +5117,11 @@ "category": "Celebrity", "registrations": [ "N236MJ" - ] + ], + "socials": { + "twitter": "Jumpman23", + "instagram": "jumpman23" + } }, "State of Montana": { "category": "State/Law", @@ -4929,7 +5160,11 @@ "registrations": [ "N88W", "N964PP" - ] + ], + "socials": { + "twitter": "realMeetKevin", + "instagram": "meetkevin" + } }, "Caterpillar Inc.": { "category": "Business", @@ -4955,7 +5190,10 @@ "N785QS", "N652WE", "N651WE" - ] + ], + "socials": { + "twitter": "ericschmidt" + } }, "Tennessee Titans": { "category": "Sports", @@ -4967,7 +5205,11 @@ "category": "Celebrity", "registrations": [ "N540W" - ] + ], + "socials": { + "twitter": "Oprah", + "instagram": "oprah" + } }, "State of Nebraska": { "category": "State/Law", @@ -5014,7 +5256,10 @@ "N256DV", "N257DV", "N258DV" - ] + ], + "socials": { + "twitter": "BetsyDeVos" + } }, "Drew Brees": { "category": "Sports", @@ -5026,7 +5271,11 @@ "category": "Celebrity", "registrations": [ "N600CK" - ] + ], + "socials": { + "twitter": "ozuna_pr", + "instagram": "ozuna" + } }, "State of Nevada": { "category": "State/Law", @@ -5070,7 +5319,11 @@ "category": "People", "registrations": [ "N555MZ" - ] + ], + "socials": { + "twitter": "yousuck2020", + "instagram": "ysknzw" + } }, "LA Chargers": { "category": "Sports", @@ -5083,7 +5336,11 @@ "registrations": [ "N305PB", "N365WW" - ] + ], + "socials": { + "twitter": "pitbull", + "instagram": "pitbull" + } }, "State of New Hampshire": { "category": "State/Law", @@ -5122,7 +5379,11 @@ "category": "People", "registrations": [ "M-GGAL" - ] + ], + "socials": { + "twitter": "richardbranson", + "instagram": "richardbranson" + } }, "Riley Herbst": { "category": "Sports", @@ -5174,7 +5435,11 @@ "category": "People", "registrations": [ "N950TR" - ] + ], + "socials": { + "twitter": "TonyRobbins", + "instagram": "tonyrobbins" + } }, "Robert Penske": { "category": "Sports", @@ -5225,7 +5490,11 @@ "category": "People", "registrations": [ "N517TW" - ] + ], + "socials": { + "twitter": "TigerWoods", + "instagram": "tigerwoods" + } }, "Jim Crane/Houston Astros": { "category": "Sports", @@ -5237,7 +5506,11 @@ "category": "Celebrity", "registrations": [ "N33055" - ] + ], + "socials": { + "twitter": "RickRoss", + "instagram": "richforever" + } }, "State of New York": { "category": "State/Law", @@ -5307,7 +5580,11 @@ "category": "Celebrity", "registrations": [ "N220F" - ] + ], + "socials": { + "twitter": "rihanna", + "instagram": "badgalriri" + } }, "State of North Carolina": { "category": "State/Law", @@ -5408,7 +5685,11 @@ "category": "Celebrity", "registrations": [ "N1013" - ] + ], + "socials": { + "twitter": "sammyhagar", + "instagram": "sammyhagar" + } }, "State of Ohio": { "category": "State/Law", @@ -5460,7 +5741,10 @@ "category": "People", "registrations": [ "N524EA" - ] + ], + "socials": { + "twitter": "DavidMRubenstein" + } }, "John Mara/NY Giants": { "category": "Sports", @@ -5473,7 +5757,11 @@ "category": "Celebrity", "registrations": [ "N877H" - ] + ], + "socials": { + "twitter": "SebastianComedy", + "instagram": "sebastiancomedy" + } }, "State of Oklahoma": { "category": "State/Law", @@ -5516,7 +5804,11 @@ "category": "Celebrity", "registrations": [ "N3250N" - ] + ], + "socials": { + "twitter": "SHAQ", + "instagram": "shaq" + } }, "State of Oregon": { "category": "State/Law", @@ -5552,7 +5844,11 @@ "registrations": [ "N3880", "N68885" - ] + ], + "socials": { + "twitter": "finkd", + "instagram": "zuck" + } }, "Ron Fowler/San Diego Padres": { "category": "Sports", @@ -5564,7 +5860,8 @@ "category": "Celebrity", "registrations": [ "N700KS", - "N800KS" + "N800KS", + "N900KS" ] }, "State of Pennsylvania": { @@ -5602,8 +5899,13 @@ "N10MV", "N6MV", "N47EG", - "N5MV" - ] + "N5MV", + "N11NY" + ], + "socials": { + "twitter": "MikeBloomberg", + "instagram": "mikebloomberg" + } }, "John Middleton/Phillies": { "category": "Sports", @@ -5641,13 +5943,21 @@ "category": "Sports", "registrations": [ "LX-GOL" - ] + ], + "socials": { + "twitter": "Cristiano", + "instagram": "cristiano" + } }, "Tom Cruise": { "category": "Celebrity", "registrations": [ "N77VA" - ] + ], + "socials": { + "twitter": "TomCruise", + "instagram": "tomcruise" + } }, "State of South Carolina": { "category": "State/Law", @@ -5753,7 +6063,10 @@ "category": "People", "registrations": [ "N221DG" - ] + ], + "socials": { + "instagram": "davidgeffen" + } }, "Jody Allen/Seahawks & Trail Blazers (Paul Allen Estate)": { "category": "Sports", @@ -5768,7 +6081,11 @@ "N378AP", "N387AP", "N378TP" - ] + ], + "socials": { + "twitter": "tylerperry", + "instagram": "tylerperry" + } }, "State of South Dakota": { "category": "State/Law", @@ -5812,7 +6129,11 @@ "category": "Celebrity", "registrations": [ "N446CJ" - ] + ], + "socials": { + "twitter": "DierksBentley", + "instagram": "dierksbentley" + } }, "State of Tennessee": { "category": "State/Law", @@ -5845,7 +6166,11 @@ "category": "People", "registrations": [ "N711RL" - ] + ], + "socials": { + "twitter": "RalphLauren", + "instagram": "ralphlauren" + } }, "John Elway": { "category": "Sports", @@ -5857,7 +6182,11 @@ "category": "Celebrity", "registrations": [ "N11PH" - ] + ], + "socials": { + "twitter": "ParisHilton", + "instagram": "parishilton" + } }, "State of Texas": { "category": "State/Law", @@ -5910,7 +6239,10 @@ "category": "People", "registrations": [ "N173JM" - ] + ], + "socials": { + "instagram": "adamweitsman" + } }, "Adam Scott": { "category": "Sports", @@ -5922,7 +6254,11 @@ "category": "Celebrity", "registrations": [ "N2320" - ] + ], + "socials": { + "twitter": "djkhaled", + "instagram": "djkhaled" + } }, "State of Utah": { "category": "State/Law", @@ -6072,7 +6408,10 @@ "category": "People", "registrations": [ "N74VW" - ] + ], + "socials": { + "twitter": "NickWoodman" + } }, "Craig Leipold/Minnesota Wild": { "category": "Sports", @@ -6200,13 +6539,21 @@ "N100GN", "N252TF", "N650FJ" - ] + ], + "socials": { + "twitter": "TilmanFertitta", + "instagram": "tilmanfertitta" + } }, "Lionel Messi": { "category": "Sports", "registrations": [ "LV-IRQ" - ] + ], + "socials": { + "twitter": "TeamMessi", + "instagram": "leomessi" + } }, "Discovery Land Company Real Estate": { "category": "Business", @@ -6228,7 +6575,10 @@ "N435FG", "N335FG", "N919FG" - ] + ], + "socials": { + "twitter": "shahidkhan" + } }, "Disney": { "category": "Business", @@ -6264,7 +6614,10 @@ "registrations": [ "VH-CPH", "VH-FMG" - ] + ], + "socials": { + "twitter": "TwiggyForrest" + } }, "Dollar General": { "category": "Business", @@ -6281,8 +6634,12 @@ "Steve Ballmer": { "category": "People", "registrations": [ - "N709DS" - ] + "N709DS", + "N711LS" + ], + "socials": { + "twitter": "Steven_Ballmer" + } }, "Dollar Tree": { "category": "Business", @@ -6303,7 +6660,11 @@ "registrations": [ "N62LV", "N611BF" - ] + ], + "socials": { + "twitter": "ArthurBlank", + "instagram": "arthurblank" + } }, "Domino's": { "category": "Business", @@ -6333,7 +6694,10 @@ "category": "People", "registrations": [ "N627JW" - ] + ], + "socials": { + "twitter": "johnwhenry" + } }, "Draft Kings": { "category": "Business", @@ -6355,7 +6719,10 @@ "category": "People", "registrations": [ "N312JC" - ] + ], + "socials": { + "twitter": "JimCrane" + } }, "DS Advisors (Equitec)": { "category": "Business", @@ -6377,7 +6744,11 @@ "category": "People", "registrations": [ "N480JJ" - ] + ], + "socials": { + "twitter": "JimmyJohnson", + "instagram": "jimmyjohnson" + } }, "Duke Energy": { "category": "Business", @@ -6444,7 +6815,10 @@ "category": "People", "registrations": [ "N814LL" - ] + ], + "socials": { + "twitter": "TedLeonsis" + } }, "Enterprise Car Rental": { "category": "Business", @@ -6547,8 +6921,15 @@ "N756LB", "N11AF", "N7WW", - "N2AF" - ] + "N2AF", + "N758PB", + "N271DV", + "N194PJ" + ], + "socials": { + "twitter": "JeffBezos", + "instagram": "jeffbezos" + } }, "Five Guys": { "category": "Business", @@ -6575,7 +6956,11 @@ "N122GA", "N721F", "N723GD" - ] + ], + "socials": { + "twitter": "TilmanFertitta", + "instagram": "tilmanfertitta" + } }, "Ford": { "category": "Business", @@ -6618,7 +7003,10 @@ "category": "People", "registrations": [ "N356ML" - ] + ], + "socials": { + "twitter": "realMikeLindell" + } }, "GAP Inc.": { "category": "Business", @@ -6643,7 +7031,10 @@ "category": "People", "registrations": [ "N8439E" - ] + ], + "socials": { + "twitter": "JoeManchin" + } }, "Globus Medical": { "category": "Business", @@ -6671,8 +7062,12 @@ "N89NC", "N988NC", "N900NC", - "N898NC" - ] + "N898NC", + "N4N" + ], + "socials": { + "twitter": "RupertMurdoch" + } }, "Goodyear Tire & Rubber Company": { "category": "Business", @@ -6721,7 +7116,11 @@ "category": "People", "registrations": [ "N808XX" - ] + ], + "socials": { + "twitter": "Benioff", + "instagram": "benioff" + } }, "Guggenheim Partners": { "category": "Business", @@ -6777,7 +7176,10 @@ "category": "People", "registrations": [ "N41127" - ] + ], + "socials": { + "twitter": "peterthiel" + } }, "Hess Corp": { "category": "Business", @@ -6815,7 +7217,10 @@ "registrations": [ "N631CD", "N631AB" - ] + ], + "socials": { + "twitter": "tyler" + } }, "Hobby Lobby": { "category": "Business", @@ -6863,8 +7268,14 @@ "Donald Trump": { "category": "People", "registrations": [ - "N757AF" - ] + "N757AF", + "N917XA", + "N725DT" + ], + "socials": { + "twitter": "realDonaldTrump", + "instagram": "realdonaldtrump" + } }, "HP": { "category": "Business", @@ -6909,7 +7320,10 @@ "category": "People", "registrations": [ "N155TM" - ] + ], + "socials": { + "twitter": "nikolatrevor" + } }, "Hy-Vee Grocery": { "category": "Business", @@ -6953,7 +7367,10 @@ "category": "People", "registrations": [ "N97DQ" - ] + ], + "socials": { + "twitter": "Druckenmiller" + } }, "IBM": { "category": "Business", @@ -6998,7 +7415,11 @@ "category": "People", "registrations": [ "N858JL" - ] + ], + "socials": { + "twitter": "thejoshaltman", + "instagram": "thejoshaltman" + } }, "Jacksons Food Stores": { "category": "Business", @@ -7092,7 +7513,10 @@ "category": "People", "registrations": [ "N900VL" - ] + ], + "socials": { + "twitter": "DavidSacks" + } }, "Johnson & Johnson": { "category": "Business", @@ -7116,7 +7540,11 @@ "category": "People", "registrations": [ "N818TH" - ] + ], + "socials": { + "twitter": "TommyHilfiger", + "instagram": "tommyhilfiger" + } }, "Johnson Development": { "category": "Business", @@ -7134,7 +7562,11 @@ "category": "People", "registrations": [ "N206SU" - ] + ], + "socials": { + "twitter": "EdwardNorton", + "instagram": "edwardnortonofficial" + } }, "Johnsonville Brats": { "category": "Business", @@ -7180,7 +7612,11 @@ "N992DC", "N945DC", "N1DC" - ] + ], + "socials": { + "twitter": "JerryJones", + "instagram": "jerryjones" + } }, "JRK Property Holdings": { "category": "Business", @@ -7224,7 +7660,10 @@ "registrations": [ "M-LDYS", "M-DANS" - ] + ], + "socials": { + "instagram": "danielsnyder_" + } }, "Khosla Ventures": { "category": "Business", @@ -7294,7 +7733,10 @@ "category": "People", "registrations": [ "N1089" - ] + ], + "socials": { + "twitter": "IOHK_Charles" + } }, "Koch Industries": { "category": "Business", @@ -7518,7 +7960,10 @@ "category": "People", "registrations": [ "N117AL" - ] + ], + "socials": { + "twitter": "sacca" + } }, "Maersk Shipping Company": { "category": "Business", @@ -7628,7 +8073,10 @@ "category": "People", "registrations": [ "VP-COR" - ] + ], + "socials": { + "twitter": "joetsai1969" + } }, "Menards": { "category": "Business", @@ -7694,7 +8142,10 @@ "category": "People", "registrations": [ "N108PB" - ] + ], + "socials": { + "twitter": "woodyjohnson4" + } }, "Miliken": { "category": "Business", @@ -7742,7 +8193,10 @@ "category": "People", "registrations": [ "N139MB" - ] + ], + "socials": { + "twitter": "MarkBurnettTV" + } }, "Morgan and Morgan Law Firm": { "category": "Business", @@ -7750,7 +8204,7 @@ "N946MM" ] }, - "Andr\u00e9 Esteves": { + "André Esteves": { "category": "People", "registrations": [ "PS-BTG" @@ -7821,7 +8275,11 @@ "category": "People", "registrations": [ "N1826K" - ] + ], + "socials": { + "twitter": "jaredkushner", + "instagram": "jaredckushner" + } }, "Netflix": { "category": "Business", @@ -8038,7 +8496,11 @@ "registrations": [ "N88W", "N964PP" - ] + ], + "socials": { + "twitter": "realMeetKevin", + "instagram": "meetkevin" + } }, "OtterBox": { "category": "Business", @@ -8090,7 +8552,7 @@ "N888ZF" ] }, - "Pap\u00e9 Machinery": { + "Papé Machinery": { "category": "Business", "registrations": [ "N101PG", @@ -8140,7 +8602,10 @@ "N583GD", "N582GD", "N962GA" - ] + ], + "socials": { + "twitter": "GovPritzker" + } }, "PayPal": { "category": "Business", @@ -8152,7 +8617,10 @@ "category": "People", "registrations": [ "N605DA" - ] + ], + "socials": { + "twitter": "AlikoDangote" + } }, "Penn National Gaming": { "category": "Business", @@ -8166,7 +8634,11 @@ "registrations": [ "N84PJ", "N83ML" - ] + ], + "socials": { + "twitter": "JohnSchnatter", + "instagram": "johnhschnatter" + } }, "Pepsi": { "category": "Business", @@ -8233,7 +8705,10 @@ "category": "People", "registrations": [ "N148L" - ] + ], + "socials": { + "twitter": "lefkofsky" + } }, "Precision Castparts": { "category": "Business", @@ -8273,7 +8748,11 @@ "category": "People", "registrations": [ "N3877" - ] + ], + "socials": { + "twitter": "saylor", + "instagram": "michael_saylor" + } }, "Proctor & Gamble": { "category": "Business", @@ -8390,7 +8869,11 @@ "category": "People", "registrations": [ "N188FJ" - ] + ], + "socials": { + "twitter": "KrisKrohn", + "instagram": "kriskrohn" + } }, "Quest Diagnostics (PC12)": { "category": "Business", @@ -8428,7 +8911,10 @@ "category": "People", "registrations": [ "N131EP" - ] + ], + "socials": { + "twitter": "realErikPrince" + } }, "RealPage Real Estate": { "category": "Business", @@ -8454,7 +8940,10 @@ "category": "People", "registrations": [ "N171EX" - ] + ], + "socials": { + "twitter": "pierre" + } }, "Resturant Brands Intl.": { "category": "Business", @@ -8555,7 +9044,11 @@ "category": "People", "registrations": [ "N616RH" - ] + ], + "socials": { + "twitter": "robertherjavec", + "instagram": "robertherjavec" + } }, "ScottsMiracle-Gro": { "category": "Business", @@ -8600,7 +9093,10 @@ "category": "People", "registrations": [ "N12MW" - ] + ], + "socials": { + "twitter": "SteveRattner" + } }, "Simon Property Group (Malls)": { "category": "Business", @@ -8621,7 +9117,7 @@ "N3E" ] }, - "Alberto Rodr\u00edguez Baldi (Baldi Hot Springs Hotel)": { + "Alberto Rodríguez Baldi (Baldi Hot Springs Hotel)": { "category": "People", "registrations": [ "N908BH" @@ -8673,6 +9169,57 @@ "registrations": [ "N7JP" ] + }, + "Travis Scott": { + "category": "Celebrity", + "registrations": [ + "N713TS" + ], + "socials": { + "twitter": "traborjamaicav", + "instagram": "travisscott" + } + }, + "Justin Bieber": { + "category": "Celebrity", + "registrations": [ + "N776RB" + ], + "socials": { + "twitter": "justinbieber", + "instagram": "justinbieber" + } + }, + "Saudi Royal Flight": { + "category": "Government", + "registrations": [ + "HZ-HM1", + "HZ-AS99" + ] + }, + "UK Royal Family (RAF)": { + "category": "Government", + "registrations": [ + "G-XWBG" + ] + }, + "France Presidency (COTAM)": { + "category": "Government", + "registrations": [ + "F-RARF" + ] + }, + "Germany Chancellery (Flugbereitschaft)": { + "category": "Government", + "registrations": [ + "10+01" + ] + }, + "Japan Prime Minister": { + "category": "Government", + "registrations": [ + "80-1111" + ] } } } \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 774072d1..8d8887c0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,10 +1,21 @@ import os +import sys import time import logging +import asyncio +import base64 +import hmac +import hmac as _hmac_mod +import secrets +import hashlib as _hashlib_mod +from dataclasses import dataclass, field +from typing import Any +from json import JSONDecodeError logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) _start_time = time.time() +_MESH_ONLY = os.environ.get("MESH_ONLY", "").strip().lower() in ("1", "true", "yes") # --------------------------------------------------------------------------- # Docker Swarm Secrets support @@ -19,6 +30,8 @@ "LTA_ACCOUNT_KEY", "CORS_ORIGINS", "ADMIN_KEY", + "SHODAN_API_KEY", + "FINNHUB_API_KEY", ] for _var in _SECRET_VARS: @@ -39,20 +52,57 @@ logger.error(f"Failed to read secret file {_file_path} for {_var}: {_e}") from fastapi import FastAPI, Request, Response, Query, Depends, HTTPException +from fastapi.responses import JSONResponse, StreamingResponse from fastapi.middleware.cors import CORSMiddleware +from starlette.background import BackgroundTask from contextlib import asynccontextmanager -from services.data_fetcher import start_scheduler, stop_scheduler, get_latest_data, source_timestamps +from services.data_fetcher import ( + start_scheduler, + stop_scheduler, + get_latest_data, +) from services.ais_stream import start_ais_stream, stop_ais_stream from services.carrier_tracker import start_carrier_tracker, stop_carrier_tracker from slowapi import Limiter, _rate_limit_exceeded_handler from slowapi.util import get_remote_address from slowapi.errors import RateLimitExceeded from services.schemas import HealthResponse, RefreshResponse +from services.config import get_settings import uvicorn import hashlib +import math import json as json_mod +import orjson import socket +from cachetools import TTLCache import threading +from services.mesh.mesh_crypto import ( + _derive_peer_key, + build_signature_payload, + derive_node_id, + normalize_peer_url, + verify_signature, + verify_node_binding, + parse_public_key_algo, +) +from services.mesh.mesh_protocol import ( + PROTOCOL_VERSION, + normalize_dm_message_payload_legacy, + normalize_payload, +) +from services.mesh.mesh_schema import validate_event_payload +from services.mesh.mesh_infonet_sync_support import ( + SyncWorkerState, + begin_sync, + eligible_sync_peers, + finish_sync, + should_run_sync, +) +from services.mesh.mesh_router import ( + authenticated_push_peer_urls, + configured_relay_peer_urls, + peer_transport_kind, +) limiter = Limiter(key_func=get_remote_address) @@ -61,17 +111,984 @@ # Set ADMIN_KEY in .env or Docker secrets. If unset, endpoints remain open # for local-dev convenience but will log a startup warning. # --------------------------------------------------------------------------- -_ADMIN_KEY = os.environ.get("ADMIN_KEY", "") -if not _ADMIN_KEY: - logger.warning("ADMIN_KEY is not set — sensitive endpoints are UNPROTECTED. " - "Set ADMIN_KEY in .env or Docker secrets for production.") +def _current_admin_key() -> str: + try: + return str(get_settings().ADMIN_KEY or "").strip() + except Exception: + return os.environ.get("ADMIN_KEY", "").strip() + + +def _allow_insecure_admin() -> bool: + try: + settings = get_settings() + return bool(getattr(settings, "ALLOW_INSECURE_ADMIN", False)) and bool( + getattr(settings, "MESH_DEBUG_MODE", False) + ) + except Exception: + return False + + +def _debug_mode_enabled() -> bool: + try: + return bool(getattr(get_settings(), "MESH_DEBUG_MODE", False)) + except Exception: + return False + + +def _admin_key_required_in_production() -> bool: + try: + settings = get_settings() + return not bool(getattr(settings, "MESH_DEBUG_MODE", False)) and not bool(_current_admin_key()) + except Exception: + return False + + +def _scoped_admin_tokens() -> dict[str, list[str]]: + raw = str(get_settings().MESH_SCOPED_TOKENS or "").strip() + if not raw: + return {} + try: + parsed = json_mod.loads(raw) + except Exception as exc: + logger.warning("failed to parse MESH_SCOPED_TOKENS: %s", exc) + return {} + if not isinstance(parsed, dict): + logger.warning("MESH_SCOPED_TOKENS must decode to an object mapping token -> scopes") + return {} + normalized: dict[str, list[str]] = {} + for token, scopes in parsed.items(): + token_key = str(token or "").strip() + if not token_key: + continue + values = scopes if isinstance(scopes, list) else [scopes] + normalized[token_key] = [str(scope or "").strip() for scope in values if str(scope or "").strip()] + return normalized + + +def _required_scope_for_request(request: Request) -> str: + path = str(request.url.path or "") + if path.startswith("/api/wormhole/gate/"): + return "gate" + if path.startswith("/api/wormhole/dm/"): + return "dm" + if path.startswith("/api/wormhole") or path in {"/api/settings/wormhole", "/api/settings/privacy-profile"}: + return "wormhole" + if path.startswith("/api/mesh/"): + return "mesh" + return "admin" + + +def _scope_allows(required_scope: str, allowed_scopes: list[str]) -> bool: + for scope in allowed_scopes: + normalized = str(scope or "").strip() + if not normalized: + continue + if normalized == "*" or required_scope == normalized: + return True + if required_scope.startswith(f"{normalized}.") or required_scope.startswith(f"{normalized}/"): + return True + return False + + +def _check_scoped_auth(request: Request, required_scope: str) -> tuple[bool, str]: + admin_key = _current_admin_key() + scoped_tokens = _scoped_admin_tokens() + presented = str(request.headers.get("X-Admin-Key", "") or "").strip() + host = (request.client.host or "").lower() if request.client else "" + if admin_key and hmac.compare_digest(presented.encode(), admin_key.encode()): + return True, "ok" + if presented: + presented_bytes = presented.encode() + for token_value, scopes in scoped_tokens.items(): + if hmac.compare_digest(presented_bytes, str(token_value or "").encode()): + if _scope_allows(required_scope, scopes): + return True, "ok" + return False, "insufficient scope" + if not admin_key and not scoped_tokens: + if _allow_insecure_admin() or (_debug_mode_enabled() and host == "test"): + return True, "ok" + return False, "Forbidden — admin key not configured" + return False, "Forbidden — invalid or missing admin key" + + +def _public_mesh_log_entry(entry: dict[str, Any]) -> dict[str, Any] | None: + tier_str = str((entry or {}).get("trust_tier", "public_degraded") or "public_degraded").strip().lower() + if tier_str.startswith("private_"): + return None + return { + "sender": str((entry or {}).get("sender", "") or ""), + "destination": str((entry or {}).get("destination", "") or ""), + "routed_via": str((entry or {}).get("routed_via", "") or ""), + "priority": str((entry or {}).get("priority", "") or ""), + "route_reason": str((entry or {}).get("route_reason", "") or ""), + "timestamp": float((entry or {}).get("timestamp", 0) or 0), + } + + +def _public_mesh_log_size(entries: list[dict[str, Any]]) -> int: + return sum(1 for item in entries if _public_mesh_log_entry(item) is not None) + + +_WORMHOLE_PUBLIC_SETTINGS_FIELDS = {"enabled", "transport", "anonymous_mode"} +_WORMHOLE_PUBLIC_PROFILE_FIELDS = {"profile", "wormhole_enabled"} +_PRIVATE_LANE_CONTROL_FIELDS = {"private_lane_tier", "private_lane_policy"} +_PUBLIC_RNS_STATUS_FIELDS = {"enabled", "ready", "configured_peers", "active_peers"} +_NODE_RUNTIME_LOCK = threading.RLock() +_NODE_SYNC_STOP = threading.Event() +_NODE_SYNC_STATE = SyncWorkerState() +_NODE_BOOTSTRAP_STATE: dict[str, Any] = { + "node_mode": "participant", + "manifest_loaded": False, + "manifest_signer_id": "", + "manifest_valid_until": 0, + "bootstrap_peer_count": 0, + "sync_peer_count": 0, + "push_peer_count": 0, + "operator_peer_count": 0, + "last_bootstrap_error": "", +} +_NODE_PUSH_STATE: dict[str, Any] = { + "last_event_id": "", + "last_push_ok_at": 0, + "last_push_error": "", + "last_results": [], +} +_NODE_PUBLIC_EVENT_HOOK_REGISTERED = False + + +def _current_node_mode() -> str: + mode = str(get_settings().MESH_NODE_MODE or "participant").strip().lower() + if mode not in {"participant", "relay", "perimeter"}: + return "participant" + return mode + + +def _node_runtime_supported() -> bool: + return _current_node_mode() in {"participant", "relay"} + + +def _node_activation_enabled() -> bool: + from services.node_settings import read_node_settings + + try: + settings = read_node_settings() + except Exception: + return False + return bool(settings.get("enabled", False)) + + +def _participant_node_enabled() -> bool: + return _node_runtime_supported() and _node_activation_enabled() + + +def _node_runtime_snapshot() -> dict[str, Any]: + with _NODE_RUNTIME_LOCK: + return { + "node_mode": _NODE_BOOTSTRAP_STATE.get("node_mode", "participant"), + "node_enabled": _participant_node_enabled(), + "bootstrap": dict(_NODE_BOOTSTRAP_STATE), + "sync_runtime": _NODE_SYNC_STATE.to_dict(), + "push_runtime": dict(_NODE_PUSH_STATE), + } + + +def _set_node_sync_disabled_state(*, current_head: str = "") -> SyncWorkerState: + return SyncWorkerState( + current_head=str(current_head or ""), + last_outcome="disabled", + ) + + +def _set_participant_node_enabled(enabled: bool) -> dict[str, Any]: + from services.mesh.mesh_hashchain import infonet + from services.node_settings import write_node_settings + + settings = write_node_settings(enabled=bool(enabled)) + current_head = str(infonet.head_hash or "") + with _NODE_RUNTIME_LOCK: + _NODE_BOOTSTRAP_STATE["node_mode"] = _current_node_mode() + globals()["_NODE_SYNC_STATE"] = ( + SyncWorkerState(current_head=current_head) + if bool(enabled) and _node_runtime_supported() + else _set_node_sync_disabled_state(current_head=current_head) + ) + return { + **settings, + "node_mode": _current_node_mode(), + "node_enabled": _participant_node_enabled(), + } + + +def _refresh_node_peer_store(*, now: float | None = None) -> dict[str, Any]: + from services.mesh.mesh_bootstrap_manifest import load_bootstrap_manifest_from_settings + from services.mesh.mesh_peer_store import ( + DEFAULT_PEER_STORE_PATH, + PeerStore, + make_bootstrap_peer_record, + make_push_peer_record, + make_sync_peer_record, + ) + + timestamp = int(now if now is not None else time.time()) + mode = _current_node_mode() + store = PeerStore(DEFAULT_PEER_STORE_PATH) + try: + store.load() + except Exception: + store = PeerStore(DEFAULT_PEER_STORE_PATH) + + operator_peers = configured_relay_peer_urls() + for peer_url in operator_peers: + transport = peer_transport_kind(peer_url) + if not transport: + continue + store.upsert( + make_sync_peer_record( + peer_url=peer_url, + transport=transport, + role="relay", + source="operator", + now=timestamp, + ) + ) + store.upsert( + make_push_peer_record( + peer_url=peer_url, + transport=transport, + role="relay", + source="operator", + now=timestamp, + ) + ) + + manifest = None + bootstrap_error = "" + try: + manifest = load_bootstrap_manifest_from_settings(now=timestamp) + except Exception as exc: + bootstrap_error = str(exc or "").strip() + + if manifest is not None: + for peer in manifest.peers: + store.upsert( + make_bootstrap_peer_record( + peer_url=peer.peer_url, + transport=peer.transport, + role=peer.role, + label=peer.label, + signer_id=manifest.signer_id, + now=timestamp, + ) + ) + store.upsert( + make_sync_peer_record( + peer_url=peer.peer_url, + transport=peer.transport, + role=peer.role, + source="bootstrap_promoted", + label=peer.label, + signer_id=manifest.signer_id, + now=timestamp, + ) + ) + + store.save() + snapshot = { + "node_mode": mode, + "manifest_loaded": manifest is not None, + "manifest_signer_id": manifest.signer_id if manifest is not None else "", + "manifest_valid_until": int(manifest.valid_until or 0) if manifest is not None else 0, + "bootstrap_peer_count": len(store.records_for_bucket("bootstrap")), + "sync_peer_count": len(store.records_for_bucket("sync")), + "push_peer_count": len(store.records_for_bucket("push")), + "operator_peer_count": len(operator_peers), + "last_bootstrap_error": bootstrap_error, + } + with _NODE_RUNTIME_LOCK: + _NODE_BOOTSTRAP_STATE.update(snapshot) + return snapshot + + +def _materialize_local_infonet_state() -> None: + from services.mesh.mesh_hashchain import infonet + + infonet.ensure_materialized() + + +def _peer_sync_response(peer_url: str, body: dict[str, Any]) -> dict[str, Any]: + import requests as _requests + from services.wormhole_supervisor import _check_arti_ready + + normalized = normalize_peer_url(peer_url) + if not normalized: + raise ValueError("invalid peer URL") + + timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10) + kwargs: dict[str, Any] = { + "json": body, + "timeout": timeout, + "headers": {"Content-Type": "application/json"}, + } + if peer_transport_kind(normalized) == "onion": + if not bool(get_settings().MESH_ARTI_ENABLED): + raise RuntimeError("onion sync requires Arti to be enabled") + if not _check_arti_ready(): + raise RuntimeError("onion sync requires a ready Arti transport") + socks_port = int(get_settings().MESH_ARTI_SOCKS_PORT or 9050) + proxy = f"socks5h://127.0.0.1:{socks_port}" + kwargs["proxies"] = {"http": proxy, "https": proxy} + response = _requests.post(f"{normalized}/api/mesh/infonet/sync", **kwargs) + try: + payload = response.json() + except Exception as exc: + raise ValueError(f"peer sync returned non-JSON response ({response.status_code})") from exc + if response.status_code != 200: + detail = str(payload.get("detail", "") or f"HTTP {response.status_code}").strip() + raise ValueError(detail or f"HTTP {response.status_code}") + if not isinstance(payload, dict): + raise ValueError("peer sync returned malformed payload") + return payload + + +def _hydrate_gate_store_from_chain(events: list[dict]) -> int: + """Copy any gate_message chain events into the local gate_store for read/decrypt.""" + import copy + + from services.mesh.mesh_hashchain import gate_store + + count = 0 + for evt in events: + if evt.get("event_type") != "gate_message": + continue + payload = evt.get("payload") or {} + gate_id = str(payload.get("gate", "") or "").strip() + if not gate_id: + continue + try: + # Deep copy so gate_store mutations (e.g. adding gate_envelope) + # don't corrupt the chain event's payload hash. + gate_store.append(gate_id, copy.deepcopy(evt)) + count += 1 + except Exception: + pass + return count + + +def _sync_from_peer(peer_url: str, *, page_limit: int = 100, max_rounds: int = 5) -> tuple[bool, str, bool]: + from services.mesh.mesh_hashchain import infonet + + rounds = 0 + while rounds < max_rounds: + body = { + "protocol_version": PROTOCOL_VERSION, + "locator": infonet.get_locator(), + "limit": page_limit, + } + payload = _peer_sync_response(peer_url, body) + if bool(payload.get("forked")): + # Auto-recover small local forks: if the local chain is tiny + # (< 20 events) and the remote has a longer chain, reset local + # state and re-sync from genesis instead of failing forever. + remote_count = int(payload.get("count", 0) or 0) + local_count = len(infonet.events) + if local_count < 20: + logger.warning( + "Fork detected with small local chain (%d events). " + "Resetting to re-sync from peer (remote has %d events).", + local_count, + remote_count, + ) + infonet.reset_chain() + continue # retry sync with clean genesis locator + return False, "fork detected", True + events = payload.get("events", []) + if not isinstance(events, list): + return False, "peer sync events must be a list", False + if not events: + return True, "", False + result = infonet.ingest_events(events) + _hydrate_gate_store_from_chain(events) + rejected = list(result.get("rejected", []) or []) + if rejected: + return False, f"sync ingest rejected {len(rejected)} event(s)", False + if int(result.get("accepted", 0) or 0) == 0 and int(result.get("duplicates", 0) or 0) >= len(events): + return True, "", False + if len(events) < page_limit: + return True, "", False + rounds += 1 + return True, "", False + + +def _run_public_sync_cycle() -> SyncWorkerState: + from services.mesh.mesh_hashchain import infonet + from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore + + if not _participant_node_enabled(): + updated = _set_node_sync_disabled_state(current_head=infonet.head_hash) + with _NODE_RUNTIME_LOCK: + globals()["_NODE_SYNC_STATE"] = updated + return updated + + store = PeerStore(DEFAULT_PEER_STORE_PATH) + try: + store.load() + except Exception: + store = PeerStore(DEFAULT_PEER_STORE_PATH) + + peers = eligible_sync_peers(store.records(), now=time.time()) + current_state = _NODE_SYNC_STATE + if not peers: + updated = finish_sync( + current_state, + ok=False, + error="no active sync peers", + now=time.time(), + current_head=infonet.head_hash, + failure_backoff_s=int(get_settings().MESH_SYNC_FAILURE_BACKOFF_S or 60), + ) + with _NODE_RUNTIME_LOCK: + globals()["_NODE_SYNC_STATE"] = updated + return updated + + last_error = "sync failed" + for record in peers: + started = begin_sync( + current_state, + peer_url=record.peer_url, + current_head=infonet.head_hash, + now=time.time(), + ) + with _NODE_RUNTIME_LOCK: + globals()["_NODE_SYNC_STATE"] = started + try: + ok, error, forked = _sync_from_peer(record.peer_url) + except Exception as exc: + ok = False + error = str(exc or type(exc).__name__) + forked = False + if ok: + store.mark_seen(record.peer_url, "sync", now=time.time()) + store.mark_sync_success(record.peer_url, now=time.time()) + store.save() + updated = finish_sync( + started, + ok=True, + peer_url=record.peer_url, + current_head=infonet.head_hash, + now=time.time(), + interval_s=int(get_settings().MESH_SYNC_INTERVAL_S or 300), + ) + with _NODE_RUNTIME_LOCK: + globals()["_NODE_SYNC_STATE"] = updated + return updated + + last_error = error + store.mark_failure( + record.peer_url, + "sync", + error=error, + cooldown_s=int(get_settings().MESH_RELAY_FAILURE_COOLDOWN_S or 120), + now=time.time(), + ) + store.save() + updated = finish_sync( + started, + ok=False, + peer_url=record.peer_url, + current_head=infonet.head_hash, + error=error, + fork_detected=forked, + now=time.time(), + interval_s=int(get_settings().MESH_SYNC_INTERVAL_S or 300), + failure_backoff_s=int(get_settings().MESH_SYNC_FAILURE_BACKOFF_S or 60), + ) + with _NODE_RUNTIME_LOCK: + globals()["_NODE_SYNC_STATE"] = updated + if forked: + return updated + current_state = updated + + return updated if peers else finish_sync( + current_state, + ok=False, + error=last_error, + now=time.time(), + current_head=infonet.head_hash, + failure_backoff_s=int(get_settings().MESH_SYNC_FAILURE_BACKOFF_S or 60), + ) + + +def _public_infonet_sync_loop() -> None: + from services.mesh.mesh_hashchain import infonet + + while not _NODE_SYNC_STOP.is_set(): + try: + if not _node_runtime_supported(): + _NODE_SYNC_STOP.wait(5.0) + continue + if not _participant_node_enabled(): + disabled = _set_node_sync_disabled_state(current_head=infonet.head_hash) + with _NODE_RUNTIME_LOCK: + globals()["_NODE_SYNC_STATE"] = disabled + _NODE_SYNC_STOP.wait(5.0) + continue + state = _NODE_SYNC_STATE + if should_run_sync(state, now=time.time()): + _run_public_sync_cycle() + except Exception: + logger.exception("public infonet sync loop failed") + _NODE_SYNC_STOP.wait(5.0) + + +def _record_public_push_result(event_id: str, *, ok: bool, error: str = "", results: list[dict[str, Any]] | None = None) -> None: + snapshot = { + "last_event_id": str(event_id or ""), + "last_push_ok_at": int(time.time()) if ok else int(_NODE_PUSH_STATE.get("last_push_ok_at", 0) or 0), + "last_push_error": "" if ok else str(error or "").strip(), + "last_results": list(results or []), + } + with _NODE_RUNTIME_LOCK: + _NODE_PUSH_STATE.update(snapshot) + + +def _propagate_public_event_to_peers(event_dict: dict[str, Any]) -> None: + from services.mesh.mesh_router import MeshEnvelope, mesh_router + + if not _participant_node_enabled(): + return + if not authenticated_push_peer_urls(): + return + + envelope = MeshEnvelope( + sender_id=str(event_dict.get("node_id", "") or ""), + destination="broadcast", + payload=json_mod.dumps(event_dict, sort_keys=True, separators=(",", ":"), ensure_ascii=False), + trust_tier="public_degraded", + ) + results = [] + for transport in (mesh_router.internet, mesh_router.tor_arti): + try: + if transport.can_reach(envelope): + result = transport.send(envelope, {}) + results.append(result.to_dict()) + except Exception as exc: + results.append({"ok": False, "transport": getattr(transport, "NAME", "unknown"), "detail": type(exc).__name__}) + ok = any(bool(result.get("ok")) for result in results) + _record_public_push_result( + str(event_dict.get("event_id", "") or ""), + ok=ok, + error="" if ok else "all push peers failed", + results=results, + ) + + +def _schedule_public_event_propagation(event_dict: dict[str, Any]) -> None: + threading.Thread( + target=_propagate_public_event_to_peers, + args=(dict(event_dict),), + daemon=True, + ).start() + + +# ─── Background HTTP Peer Push Worker ──────────────────────────────────── +# Runs alongside the sync loop. Every PUSH_INTERVAL seconds, batches new +# Infonet events and sends them via HMAC-authenticated POST to push peers. + +_PEER_PUSH_INTERVAL_S = 30 +_PEER_PUSH_BATCH_SIZE = 50 +_peer_push_last_index: dict[str, int] = {} # peer_url → last pushed event index + + +def _http_peer_push_loop() -> None: + """Background thread: push new Infonet events to HTTP peers.""" + import requests as _requests + from services.mesh.mesh_hashchain import infonet + from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore + + while not _NODE_SYNC_STOP.is_set(): + try: + if not _participant_node_enabled(): + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + continue + + secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip() + if not secret: + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + continue + + peers = authenticated_push_peer_urls() + if not peers: + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + continue + + all_events = infonet.events + total = len(all_events) + + for peer_url in peers: + normalized = normalize_peer_url(peer_url) + if not normalized: + continue + last_idx = _peer_push_last_index.get(normalized, 0) + if last_idx >= total: + continue # nothing new + + batch = all_events[last_idx : last_idx + _PEER_PUSH_BATCH_SIZE] + if not batch: + continue + + try: + body_bytes = json_mod.dumps( + {"events": batch}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + peer_key = _derive_peer_key(secret, normalized) + if not peer_key: + continue + import hmac as _hmac_mod2 + import hashlib as _hashlib_mod2 + hmac_hex = _hmac_mod2.new(peer_key, body_bytes, _hashlib_mod2.sha256).hexdigest() + + timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10) + resp = _requests.post( + f"{normalized}/api/mesh/infonet/peer-push", + data=body_bytes, + headers={ + "Content-Type": "application/json", + "X-Peer-HMAC": hmac_hex, + }, + timeout=timeout, + ) + if resp.status_code == 200: + _peer_push_last_index[normalized] = last_idx + len(batch) + logger.info( + f"Pushed {len(batch)} event(s) to {normalized[:40]} " + f"(idx {last_idx}→{last_idx + len(batch)})" + ) + else: + logger.warning(f"Peer push to {normalized[:40]} returned {resp.status_code}") + except Exception as exc: + logger.warning(f"Peer push to {normalized[:40]} failed: {exc}") + + except Exception: + logger.exception("HTTP peer push loop error") + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + + +# ─── Background Gate Message Push Worker ───────────────────────────────── + +_gate_push_last_count: dict[str, dict[str, int]] = {} # peer → {gate_id → count} + + +def _http_gate_push_loop() -> None: + """Background thread: push new gate messages to HTTP peers.""" + import requests as _requests + from services.mesh.mesh_hashchain import gate_store + + while not _NODE_SYNC_STOP.is_set(): + try: + if not _participant_node_enabled(): + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + continue + + secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip() + if not secret: + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + continue + + peers = authenticated_push_peer_urls() + if not peers: + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + continue + + with gate_store._lock: + gate_ids = list(gate_store._gates.keys()) + + for peer_url in peers: + normalized = normalize_peer_url(peer_url) + if not normalized: + continue + + peer_key = _derive_peer_key(secret, normalized) + if not peer_key: + continue + + peer_counts = _gate_push_last_count.setdefault(normalized, {}) + + for gate_id in gate_ids: + with gate_store._lock: + all_events = list(gate_store._gates.get(gate_id, [])) + total = len(all_events) + last = peer_counts.get(gate_id, 0) + if last >= total: + continue + + batch = all_events[last : last + _PEER_PUSH_BATCH_SIZE] + if not batch: + continue + + try: + body_bytes = json_mod.dumps( + {"events": batch}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + import hmac as _hmac_mod3 + import hashlib as _hashlib_mod3 + hmac_hex = _hmac_mod3.new(peer_key, body_bytes, _hashlib_mod3.sha256).hexdigest() + + timeout = int(get_settings().MESH_RELAY_PUSH_TIMEOUT_S or 10) + resp = _requests.post( + f"{normalized}/api/mesh/gate/peer-push", + data=body_bytes, + headers={ + "Content-Type": "application/json", + "X-Peer-HMAC": hmac_hex, + }, + timeout=timeout, + ) + if resp.status_code == 200: + peer_counts[gate_id] = last + len(batch) + logger.info( + f"Gate push: {len(batch)} event(s) for {gate_id[:12]} " + f"to {normalized[:40]}" + ) + else: + logger.warning( + f"Gate push to {normalized[:40]} returned {resp.status_code}" + ) + except Exception as exc: + logger.warning(f"Gate push to {normalized[:40]} failed: {exc}") + + except Exception: + logger.exception("HTTP gate push loop error") + _NODE_SYNC_STOP.wait(_PEER_PUSH_INTERVAL_S) + + +def _scoped_view_authenticated(request: Request, scope: str) -> bool: + ok, _detail = _check_scoped_auth(request, scope) + if ok: + return True + return _is_debug_test_request(request) + + +def _redacted_gate_timestamp(event: dict[str, Any]) -> float: + raw_ts = float((event or {}).get("timestamp", 0) or 0.0) + if raw_ts <= 0: + return 0.0 + try: + jitter_window = max(0, int(get_settings().MESH_GATE_TIMESTAMP_JITTER_S or 0)) + except Exception: + jitter_window = 0 + if jitter_window <= 0: + return raw_ts + event_id = str((event or {}).get("event_id", "") or "") + seed = _hashlib_mod.sha256(f"{event_id}|{int(raw_ts)}".encode("utf-8")).digest() + fraction = int.from_bytes(seed[:8], "big") / float(2**64 - 1) + return max(0.0, raw_ts - (fraction * float(jitter_window))) + + +def _redact_wormhole_settings(settings: dict[str, Any], authenticated: bool) -> dict[str, Any]: + if authenticated: + return dict(settings) + return { + key: settings.get(key) + for key in _WORMHOLE_PUBLIC_SETTINGS_FIELDS + if key in settings + } + + +def _redact_privacy_profile_settings( + settings: dict[str, Any], + authenticated: bool, +) -> dict[str, Any]: + profile = { + "profile": settings.get("privacy_profile", "default"), + "wormhole_enabled": bool(settings.get("enabled")), + "transport": settings.get("transport", "direct"), + "anonymous_mode": bool(settings.get("anonymous_mode")), + } + if authenticated: + return profile + return { + key: profile.get(key) + for key in _WORMHOLE_PUBLIC_PROFILE_FIELDS + } + + +def _redact_private_lane_control_fields( + payload: dict[str, Any], + authenticated: bool, +) -> dict[str, Any]: + redacted = dict(payload) + if authenticated: + return redacted + for field in _PRIVATE_LANE_CONTROL_FIELDS: + redacted.pop(field, None) + return redacted + + +def _redact_public_rns_status( + payload: dict[str, Any], + authenticated: bool, +) -> dict[str, Any]: + redacted = _redact_private_lane_control_fields(payload, authenticated=authenticated) + if authenticated: + return redacted + return { + key: redacted.get(key) + for key in _PUBLIC_RNS_STATUS_FIELDS + if key in redacted + } + + +def _redact_public_mesh_status( + payload: dict[str, Any], + authenticated: bool, +) -> dict[str, Any]: + if authenticated: + return dict(payload) + return { + "message_log_size": int(payload.get("message_log_size", 0) or 0), + } + + +def _redact_public_oracle_profile( + payload: dict[str, Any], + authenticated: bool, +) -> dict[str, Any]: + redacted = dict(payload) + if authenticated: + return redacted + redacted["active_stakes"] = [] + redacted["prediction_history"] = [] + return redacted + + +def _redact_public_oracle_predictions( + predictions: list[dict[str, Any]], + authenticated: bool, +) -> dict[str, Any]: + if authenticated: + return {"predictions": list(predictions)} + return { + "predictions": [], + "count": len(predictions), + } + + +def _redact_public_oracle_stakes( + payload: dict[str, Any], + authenticated: bool, +) -> dict[str, Any]: + redacted = dict(payload) + if authenticated: + return redacted + redacted["truth_stakers"] = [] + redacted["false_stakers"] = [] + return redacted + + +def _redact_public_node_history( + events: list[dict[str, Any]], + authenticated: bool, +) -> list[dict[str, Any]]: + if authenticated: + return [dict(event) for event in events] + return [ + { + "event_id": str(event.get("event_id", "") or ""), + "event_type": str(event.get("event_type", "") or ""), + "timestamp": float(event.get("timestamp", 0) or 0), + } + for event in events + ] + + +def _redact_composed_gate_message(payload: dict[str, Any]) -> dict[str, Any]: + safe = { + "ok": bool(payload.get("ok")), + "gate_id": str(payload.get("gate_id", "") or ""), + "identity_scope": str(payload.get("identity_scope", "") or ""), + "ciphertext": str(payload.get("ciphertext", "") or ""), + "nonce": str(payload.get("nonce", "") or ""), + "sender_ref": str(payload.get("sender_ref", "") or ""), + "format": str(payload.get("format", "mls1") or "mls1"), + "timestamp": float(payload.get("timestamp", 0) or 0), + } + epoch = payload.get("epoch", 0) + if epoch: + safe["epoch"] = int(epoch or 0) + if payload.get("detail"): + safe["detail"] = str(payload.get("detail", "") or "") + if payload.get("key_commitment"): + safe["key_commitment"] = str(payload.get("key_commitment", "") or "") + return safe + + +def _validate_admin_startup() -> None: + admin_key = _current_admin_key() + debug_mode = False + try: + debug_mode = bool(getattr(get_settings(), "MESH_DEBUG_MODE", False)) + except Exception: + debug_mode = False + + if not debug_mode and not admin_key: + logger.critical( + "ADMIN_KEY must be set when MESH_DEBUG_MODE is False. " + "Set ADMIN_KEY or enable MESH_DEBUG_MODE for development. Refusing to start." + ) + sys.exit(1) + + if admin_key: + if len(admin_key) < 16: + message = ( + f"ADMIN_KEY is too short ({len(admin_key)} chars, minimum 16). " + "Use a strong key." + ) + if debug_mode: + logger.warning("%s Debug mode allows startup.", message) + else: + logger.critical("%s Refusing to start.", message) + sys.exit(1) + elif len(admin_key) < 32: + logger.warning( + "ADMIN_KEY is short (%s chars). Consider using at least 32 characters for production.", + len(admin_key), + ) + elif debug_mode: + logger.warning( + "ADMIN_KEY is not set — debug mode allows startup without admin auth hardening. " + "Set ADMIN_KEY for production." + ) + def require_admin(request: Request): """FastAPI dependency that rejects requests without a valid X-Admin-Key header.""" - if not _ADMIN_KEY: - return # No key configured — allow all (local dev) - if request.headers.get("X-Admin-Key") != _ADMIN_KEY: - raise HTTPException(status_code=403, detail="Forbidden — invalid or missing admin key") + required_scope = _required_scope_for_request(request) + ok, detail = _check_scoped_auth(request, required_scope) + if ok: + return + if detail == "insufficient scope": + raise HTTPException(status_code=403, detail="Forbidden — insufficient scope") + raise HTTPException(status_code=403, detail=detail) + + +def require_local_operator(request: Request): + """Allow local tooling on loopback, or a valid admin key from elsewhere.""" + host = (request.client.host or "").lower() if request.client else "" + if host in {"127.0.0.1", "::1", "localhost"} or (_debug_mode_enabled() and host == "test"): + return + admin_key = _current_admin_key() + presented = str(request.headers.get("X-Admin-Key", "") or "").strip() + if admin_key and hmac.compare_digest(presented.encode(), admin_key.encode()): + return + raise HTTPException(status_code=403, detail="Forbidden — local operator access only") def _build_cors_origins(): @@ -99,47 +1116,162 @@ def _build_cors_origins(): origins.extend([o.strip() for o in extra.split(",") if o.strip()]) return list(set(origins)) # deduplicate + +def _safe_int(val, default=0): + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _safe_float(val, default=0.0): + try: + parsed = float(val) + if not math.isfinite(parsed): + return default + return parsed + except (TypeError, ValueError): + return default + + @asynccontextmanager async def lifespan(app: FastAPI): + _validate_admin_startup() + # Validate environment variables before starting anything from services.env_check import validate_env - validate_env(strict=True) - # Start AIS stream first — it loads the disk cache (instant ships) then - # begins accumulating live vessel data via WebSocket in the background. - start_ais_stream() + validate_env(strict=not _MESH_ONLY) + + if _MESH_ONLY: + logger.info("MESH_ONLY enabled — skipping global data fetchers/schedulers.") + else: + # Start AIS stream first — it loads the disk cache (instant ships) then + # begins accumulating live vessel data via WebSocket in the background. + start_ais_stream() + + # Carrier tracker runs its own initial update_carrier_positions() internally + # in _scheduler_loop, so we do NOT call it again in the preload thread. + start_carrier_tracker() + + # Start SIGINT grid eagerly — APRS-IS TCP + Meshtastic MQTT connections + # take a few seconds to handshake and start receiving packets. By starting + # now, the bridges are already accumulating signals by the time the first + # fetch_sigint() reads them during the preload cycle. + from services.sigint_bridge import sigint_grid + + sigint_grid.start() + + # Start Reticulum bridge (optional) + try: + from services.mesh.mesh_rns import rns_bridge + + rns_bridge.start() + except Exception as e: + logger.warning(f"RNS bridge failed to start: {e}") + + # Start periodic Infonet verifier + def _verify_loop(): + from services.mesh.mesh_hashchain import infonet - # Carrier tracker runs its own initial update_carrier_positions() internally - # in _scheduler_loop, so we do NOT call it again in the preload thread. - start_carrier_tracker() + while True: + try: + interval = int(get_settings().MESH_VERIFY_INTERVAL_S or 0) + if interval <= 0: + time.sleep(30) + continue + verify_signatures = bool(get_settings().MESH_VERIFY_SIGNATURES) + valid, reason = infonet.validate_chain_incremental(verify_signatures=verify_signatures) + if not valid: + logger.error(f"Infonet validation failed: {reason}") + try: + from services.mesh.mesh_metrics import increment as metrics_inc - # Start the recurring scheduler (fast=60s, slow=30min). - start_scheduler() + metrics_inc("infonet_validate_failed") + except Exception: + pass + time.sleep(max(5, interval)) + except Exception: + time.sleep(30) - # Kick off the full data preload in a background thread so the server - # is listening on port 8000 instantly. The frontend's adaptive polling - # (retries every 3s) will pick up data piecemeal as each fetcher finishes. - def _background_preload(): - logger.info("=== PRELOADING DATA (background — server already accepting requests) ===") + threading.Thread(target=_verify_loop, daemon=True).start() + + # Only the primary backend supervises Wormhole. The Wormhole process itself + # runs this same app in MESH_ONLY mode and must not recurse into spawning. + if not _MESH_ONLY: try: - update_all_data() - logger.info("=== PRELOAD COMPLETE ===") + from services.wormhole_supervisor import sync_wormhole_with_settings + + sync_wormhole_with_settings() + except Exception as e: + logger.warning(f"Wormhole supervisor failed to sync: {e}") + try: + from services.mesh.mesh_hashchain import register_public_event_append_hook + + _materialize_local_infonet_state() + _refresh_node_peer_store() + if _node_runtime_supported(): + if not _participant_node_enabled(): + globals()["_NODE_SYNC_STATE"] = _set_node_sync_disabled_state() + _NODE_SYNC_STOP.clear() + threading.Thread(target=_public_infonet_sync_loop, daemon=True).start() + threading.Thread(target=_http_peer_push_loop, daemon=True).start() + threading.Thread(target=_http_gate_push_loop, daemon=True).start() + global _NODE_PUBLIC_EVENT_HOOK_REGISTERED + if not _NODE_PUBLIC_EVENT_HOOK_REGISTERED: + register_public_event_append_hook(_schedule_public_event_propagation) + _NODE_PUBLIC_EVENT_HOOK_REGISTERED = True except Exception as e: - logger.error(f"Data preload failed (non-fatal): {e}") + logger.warning(f"Node bootstrap runtime failed to initialize: {e}") + + if not _MESH_ONLY: + # Start the recurring scheduler (fast=60s, slow=30min). + start_scheduler() + + # Kick off the full data preload in a background thread so the server + # is listening on port 8000 instantly. The frontend's adaptive polling + # (retries every 3s) will pick up data piecemeal as each fetcher finishes. + def _background_preload(): + logger.info("=== PRELOADING DATA (background — server already accepting requests) ===") + try: + update_all_data(startup_mode=True) + logger.info("=== PRELOAD COMPLETE ===") + except Exception as e: + logger.error(f"Data preload failed (non-fatal): {e}") - threading.Thread(target=_background_preload, daemon=True).start() + threading.Thread(target=_background_preload, daemon=True).start() yield - # Shutdown: Stop all background services - stop_ais_stream() - stop_scheduler() - stop_carrier_tracker() + if not _MESH_ONLY: + # Shutdown: Stop all background services + _NODE_SYNC_STOP.set() + stop_ais_stream() + stop_scheduler() + stop_carrier_tracker() + try: + sigint_grid.stop() + except Exception: + pass + if not _MESH_ONLY: + try: + from services.wormhole_supervisor import shutdown_wormhole_supervisor + + shutdown_wormhole_supervisor() + except Exception: + pass + app = FastAPI(title="Live Risk Dashboard API", lifespan=lifespan) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + +@app.exception_handler(JSONDecodeError) +async def json_decode_error_handler(_request: Request, _exc: JSONDecodeError): + return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"}) + from fastapi.middleware.gzip import GZipMiddleware + app.add_middleware(GZipMiddleware, minimum_size=1000) app.add_middleware( CORSMiddleware, @@ -149,33 +1281,739 @@ def _background_preload(): allow_headers=["*"], ) -from services.data_fetcher import update_all_data +_NO_STORE_HEADERS = { + "Cache-Control": "no-store, max-age=0", + "Pragma": "no-cache", +} +_SECURITY_HEADERS_PROD = { + "Content-Security-Policy": ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline' blob:; " + "style-src 'self' 'unsafe-inline'; " + "img-src 'self' data: blob: https:; " + "connect-src 'self' ws: wss: https:; " + "font-src 'self' data:; " + "object-src 'none'; " + "frame-ancestors 'none'; " + "base-uri 'self'" + ), + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", +} +_SECURITY_HEADERS_DEBUG = { + **_SECURITY_HEADERS_PROD, + "Content-Security-Policy": ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline' 'unsafe-eval' blob:; " + "style-src 'self' 'unsafe-inline'; " + "img-src 'self' data: blob: https:; " + "connect-src 'self' ws: wss: http://127.0.0.1:8000 http://127.0.0.1:8787 https:; " + "font-src 'self' data:; " + "object-src 'none'; " + "frame-ancestors 'none'; " + "base-uri 'self'" + ), +} -_refresh_lock = threading.Lock() -@app.get("/api/refresh", response_model=RefreshResponse) -@limiter.limit("2/minute") -async def force_refresh(request: Request): - if not _refresh_lock.acquire(blocking=False): - return {"status": "refresh already in progress"} - def _do_refresh(): - try: - update_all_data() +def _security_headers() -> dict[str, str]: + return _SECURITY_HEADERS_DEBUG if _debug_mode_enabled() else _SECURITY_HEADERS_PROD + + +@app.middleware("http") +async def mesh_security_headers(request: Request, call_next): + response = await call_next(request) + for header, value in _security_headers().items(): + response.headers.setdefault(header, value) + return response + + +@app.middleware("http") +async def mesh_no_store_headers(request: Request, call_next): + response = await call_next(request) + if request.url.path.startswith("/api/mesh/"): + response.headers["Cache-Control"] = "no-store, max-age=0" + response.headers["Pragma"] = "no-cache" + return response + + +def _is_anonymous_mesh_write_path(path: str, method: str) -> bool: + if method.upper() not in {"POST", "PUT", "DELETE"}: + return False + if path == "/api/mesh/send": + return True + if path in { + "/api/mesh/vote", + "/api/mesh/report", + "/api/mesh/trust/vouch", + "/api/mesh/gate/create", + "/api/mesh/oracle/predict", + "/api/mesh/oracle/resolve", + "/api/mesh/oracle/stake", + "/api/mesh/oracle/resolve-stakes", + }: + return True + if path.startswith("/api/mesh/gate/") and path.endswith("/message"): + return True + return False + + +def _is_anonymous_dm_action_path(path: str, method: str) -> bool: + method_name = method.upper() + if method_name == "POST" and path in { + "/api/mesh/dm/register", + "/api/mesh/dm/send", + "/api/mesh/dm/poll", + "/api/mesh/dm/count", + "/api/mesh/dm/block", + "/api/mesh/dm/witness", + }: + return True + if method_name == "GET" and path in { + "/api/mesh/dm/pubkey", + "/api/mesh/dm/prekey-bundle", + }: + return True + return False + + +def _is_anonymous_wormhole_gate_admin_path(path: str, method: str) -> bool: + if method.upper() != "POST": + return False + return path in { + "/api/wormhole/gate/enter", + "/api/wormhole/gate/persona/create", + "/api/wormhole/gate/persona/activate", + "/api/wormhole/gate/persona/retire", + } + + +def _is_private_infonet_write_path(path: str, method: str) -> bool: + if method.upper() != "POST": + return False + if path in { + "/api/mesh/gate/create", + "/api/mesh/vote", + }: + return True + return path.startswith("/api/mesh/gate/") and path.endswith("/message") + + +def _validate_gate_vote_context(voter_id: str, gate_id: str) -> tuple[bool, str]: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return True, "" + try: + from services.mesh.mesh_reputation import gate_manager + except Exception as exc: + return False, f"Gate validation unavailable: {exc}" + + gate = gate_manager.get_gate(gate_key) + if not gate: + return False, f"Gate '{gate_key}' does not exist" + + can_enter, reason = gate_manager.can_enter(voter_id, gate_key) + if not can_enter: + return False, f"Gate vote denied: {reason}" + + try: + from services.mesh.mesh_hashchain import gate_store + + if not gate_store.get_messages(gate_key, limit=1): + return False, f"Gate '{gate_key}' has no activity" + except Exception: + pass + + return True, gate_key + + +def _anonymous_mode_state() -> dict[str, Any]: + try: + from services.wormhole_settings import read_wormhole_settings + from services.wormhole_status import read_wormhole_status + + settings = read_wormhole_settings() + status = read_wormhole_status() + enabled = bool(settings.get("enabled")) + anonymous_mode = bool(settings.get("anonymous_mode")) + transport_configured = str(settings.get("transport", "direct") or "direct").lower() + transport_active = str(status.get("transport_active", "") or "").lower() + effective_transport = transport_active or transport_configured + ready = bool(status.get("running")) and bool(status.get("ready")) + hidden_transport_ready = enabled and ready and effective_transport in { + "tor", + "tor_arti", + "i2p", + "mixnet", + } + return { + "enabled": anonymous_mode, + "wormhole_enabled": enabled, + "ready": hidden_transport_ready, + "effective_transport": effective_transport or "direct", + } + except Exception: + return { + "enabled": False, + "wormhole_enabled": False, + "ready": False, + "effective_transport": "direct", + } + + +def _is_sensitive_no_store_path(path: str) -> bool: + if not path.startswith("/api/"): + return False + if path.startswith("/api/wormhole/"): + return True + if path.startswith("/api/settings/"): + return True + if path.startswith("/api/mesh/dm/"): + return True + if path in { + "/api/refresh", + "/api/debug-latest", + "/api/system/update", + "/api/mesh/infonet/ingest", + }: + return True + return False + + +def _private_infonet_required_tier(path: str, method: str) -> str: + method_name = method.upper() + if path in { + "/api/mesh/dm/register", + "/api/mesh/dm/send", + "/api/mesh/dm/poll", + "/api/mesh/dm/count", + "/api/mesh/dm/block", + "/api/mesh/dm/witness", + } and method_name in {"GET", "POST"}: + return "strong" + if not _is_private_infonet_write_path(path, method): + return "" + if method_name != "POST": + return "" + # Current release policy: non-DM private gate actions are allowed in + # PRIVATE / TRANSITIONAL once Wormhole is ready. Strong-mode-only actions + # should be added here explicitly instead of being implied elsewhere. + return "transitional" + + +_TRANSPORT_TIER_ORDER = { + "public_degraded": 0, + "private_transitional": 1, + "private_strong": 2, +} + + +def _current_private_lane_tier(wormhole: dict | None) -> str: + from services.wormhole_supervisor import transport_tier_from_state + + return transport_tier_from_state(wormhole) + + +def _transport_tier_is_sufficient(current_tier: str, required_tier: str) -> bool: + return _TRANSPORT_TIER_ORDER.get(current_tier, 0) >= _TRANSPORT_TIER_ORDER.get(required_tier, 0) + + +_GATE_REDACT_FIELDS = ("sender_ref", "epoch", "nonce") +_KEY_ROTATE_REDACT_FIELDS = { + "old_node_id", + "old_public_key", + "old_public_key_algo", + "old_signature", +} + + +def _redact_gate_metadata(event: dict) -> dict: + """Strip MLS-internal fields from gate_message events in public sync responses.""" + if not isinstance(event, dict): + return event + event_type = str(event.get("event_type", "") or "") + if event_type != "gate_message": + return event + redacted = dict(event) + for field in ("node_id", "sequence"): + redacted.pop(field, None) + if isinstance(redacted.get("payload"), dict): + payload = dict(redacted.get("payload") or {}) + for field in _GATE_REDACT_FIELDS: + payload.pop(field, None) + redacted["payload"] = payload + return redacted + for field in _GATE_REDACT_FIELDS: + redacted.pop(field, None) + return redacted + + +def _redact_key_rotate_payload(event: dict) -> dict: + """Strip identity-linking fields from key_rotate events in public responses.""" + if not isinstance(event, dict): + return event + if str(event.get("event_type", "") or "") != "key_rotate": + return event + redacted = dict(event) + payload = redacted.get("payload") + if isinstance(payload, dict): + payload = dict(payload) + for field in _KEY_ROTATE_REDACT_FIELDS: + payload.pop(field, None) + redacted["payload"] = payload + return redacted + + +def _redact_vote_gate(event: dict) -> dict: + """Strip gate label from vote events in public responses.""" + if not isinstance(event, dict): + return event + if str(event.get("event_type", "") or "") != "vote": + return event + redacted = dict(event) + payload = redacted.get("payload") + if isinstance(payload, dict): + payload = dict(payload) + payload.pop("gate", None) + redacted["payload"] = payload + return redacted + + +def _redact_public_event(event: dict) -> dict: + """Apply all public-response redactions for public chain endpoints.""" + return _redact_vote_gate(_redact_key_rotate_payload(_redact_gate_metadata(event))) + + +def _is_debug_test_request(request: Request) -> bool: + if not _debug_mode_enabled(): + return False + client_host = (request.client.host or "").lower() if request.client else "" + url_host = (request.url.hostname or "").lower() if request.url else "" + return client_host == "test" or url_host == "test" + + +def _strip_gate_identity(event: dict) -> dict: + """Return the private-plane gate event shape exposed to API consumers.""" + if not isinstance(event, dict): + event = {} + payload = event.get("payload") + if not isinstance(payload, dict): + payload = {} + node_id = str(event.get("node_id", "") or "") + public_key = str(event.get("public_key", "") or "") + public_key_algo = str(event.get("public_key_algo", "") or "") + # If the event doesn't carry a public_key but has a node_id, resolve it + # from the local persona/session store so the frontend can display it. + if node_id and not public_key: + gate_id = str(payload.get("gate", "") or "") + if gate_id: + try: + binding = _lookup_gate_member_binding(gate_id, node_id) + if binding: + public_key, public_key_algo = binding + except Exception: + pass + return { + "event_id": str(event.get("event_id", "") or ""), + "event_type": "gate_message", + "timestamp": _redacted_gate_timestamp(event), + "node_id": node_id, + "sequence": int(event.get("sequence", 0) or 0), + "signature": str(event.get("signature", "") or ""), + "public_key": public_key, + "public_key_algo": public_key_algo, + "protocol_version": str(event.get("protocol_version", "") or ""), + "payload": { + "gate": str(payload.get("gate", "") or ""), + "ciphertext": str(payload.get("ciphertext", "") or ""), + "format": str(payload.get("format", "") or ""), + "nonce": str(payload.get("nonce", "") or ""), + "sender_ref": str(payload.get("sender_ref", "") or ""), + "gate_envelope": str(payload.get("gate_envelope", "") or ""), + "reply_to": str(payload.get("reply_to", "") or ""), + }, + } +def _lookup_gate_member_binding(gate_id: str, node_id: str) -> tuple[str, str] | None: + gate_key = str(gate_id or "").strip().lower() + candidate = str(node_id or "").strip() + if not gate_key or not candidate: + return None + try: + from services.mesh.mesh_wormhole_persona import ( + bootstrap_wormhole_persona_state, + read_wormhole_persona_state, + ) + + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + except Exception: + return None + for persona in list(state.get("gate_personas", {}).get(gate_key) or []): + if str(persona.get("node_id", "") or "").strip() != candidate: + continue + public_key = str(persona.get("public_key", "") or "").strip() + public_key_algo = str(persona.get("public_key_algo", "Ed25519") or "Ed25519").strip() + if public_key and public_key_algo: + return public_key, public_key_algo + session = dict(state.get("gate_sessions", {}).get(gate_key) or {}) + if str(session.get("node_id", "") or "").strip() == candidate: + public_key = str(session.get("public_key", "") or "").strip() + public_key_algo = str(session.get("public_key_algo", "Ed25519") or "Ed25519").strip() + if public_key and public_key_algo: + return public_key, public_key_algo + return None + + +def _resolve_gate_proof_identity(gate_id: str) -> dict[str, Any] | None: + from services.mesh.mesh_wormhole_persona import ( + bootstrap_wormhole_persona_state, + read_wormhole_persona_state, + ) + + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return None + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + active_persona_id = str(state.get("active_gate_personas", {}).get(gate_key, "") or "") + for persona in list(state.get("gate_personas", {}).get(gate_key) or []): + if str(persona.get("persona_id", "") or "") == active_persona_id: + return dict(persona or {}) + for persona in list(state.get("gate_personas", {}).get(gate_key) or []): + if persona.get("private_key"): + return dict(persona or {}) + session_identity = dict(state.get("gate_sessions", {}).get(gate_key) or {}) + if session_identity.get("private_key"): + return session_identity + return None + + +def _sign_gate_access_proof(gate_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + identity = _resolve_gate_proof_identity(gate_key) + if not identity: + return {"ok": False, "detail": "gate_access_proof_unavailable"} + private_key_b64 = str(identity.get("private_key", "") or "").strip() + node_id = str(identity.get("node_id", "") or "").strip() + public_key = str(identity.get("public_key", "") or "").strip() + public_key_algo = str(identity.get("public_key_algo", "Ed25519") or "Ed25519").strip() + if not (private_key_b64 and node_id and public_key and public_key_algo): + return {"ok": False, "detail": "gate_access_proof_unavailable"} + try: + from cryptography.hazmat.primitives.asymmetric import ec, ed25519 + + ts = int(time.time()) + challenge = f"{gate_key}:{ts}" + key_bytes = base64.b64decode(private_key_b64) + algo = parse_public_key_algo(public_key_algo) + if algo == "Ed25519": + signing_key = ed25519.Ed25519PrivateKey.from_private_bytes(key_bytes) + signature = signing_key.sign(challenge.encode("utf-8")) + elif algo == "ECDSA_P256": + from cryptography.hazmat.primitives import hashes + + signing_key = ec.derive_private_key(int.from_bytes(key_bytes, "big"), ec.SECP256R1()) + signature = signing_key.sign(challenge.encode("utf-8"), ec.ECDSA(hashes.SHA256())) + else: + return {"ok": False, "detail": "gate_access_proof_unsupported_algo"} + except Exception as exc: + logger.warning("Gate access proof signing failed: %s", type(exc).__name__) + return {"ok": False, "detail": "gate_access_proof_failed"} + return { + "ok": True, + "gate_id": gate_key, + "node_id": node_id, + "ts": ts, + "proof": base64.b64encode(signature).decode("ascii"), + } + + +def _verify_gate_access(request: Request, gate_id: str) -> bool: + """Verify the requester has access to a gate's private message feed.""" + ok, _detail = _check_scoped_auth(request, "gate") + if ok: + return True + + gate_key = str(gate_id or "").strip().lower() + node_id = str(request.headers.get("x-wormhole-node-id", "") or "").strip() + proof_b64 = str(request.headers.get("x-wormhole-gate-proof", "") or "").strip() + ts_str = str(request.headers.get("x-wormhole-gate-ts", "") or "").strip() + if not gate_key or not node_id or not proof_b64 or not ts_str: + return False + try: + ts = int(ts_str) + except (TypeError, ValueError): + return False + if abs(int(time.time()) - ts) > 60: + return False + binding = _lookup_gate_member_binding(gate_key, node_id) + if not binding: + return False + public_key, public_key_algo = binding + if not verify_node_binding(node_id, public_key): + return False + try: + signature_hex = base64.b64decode(proof_b64, validate=True).hex() + except Exception: + return False + challenge = f"{gate_key}:{ts_str}" + return verify_signature( + public_key_b64=public_key, + public_key_algo=public_key_algo, + signature_hex=signature_hex, + payload=challenge, + ) + + +def _peer_hmac_url_from_request(request: Request) -> str: + header_url = normalize_peer_url(str(request.headers.get("x-peer-url", "") or "")) + if header_url: + return header_url + if not request.url: + return "" + base_url = f"{request.url.scheme}://{request.url.netloc}".rstrip("/") + return normalize_peer_url(base_url) + + +def _verify_peer_push_hmac(request: Request, body_bytes: bytes) -> bool: + """Verify HMAC-SHA256 peer authentication on push requests.""" + secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip() + if not secret: + return False + + provided = str(request.headers.get("x-peer-hmac", "") or "").strip() + if not provided: + return False + + peer_url = _peer_hmac_url_from_request(request) + allowed_peers = set(authenticated_push_peer_urls()) + if not peer_url or peer_url not in allowed_peers: + return False + peer_key = _derive_peer_key(secret, peer_url) + if not peer_key: + return False + + expected = _hmac_mod.new( + peer_key, + body_bytes, + _hashlib_mod.sha256, + ).hexdigest() + return _hmac_mod.compare_digest(provided.lower(), expected.lower()) + + +def _minimum_transport_tier(path: str, method: str) -> str: + method_name = method.upper() + private_infonet = _private_infonet_required_tier(path, method) + if private_infonet == "transitional": + return "private_transitional" + if private_infonet == "strong": + return "private_strong" + + if method_name == "GET" and path in { + "/api/mesh/dm/prekey-bundle", + }: + return "private_transitional" + + if method_name == "POST" and path in { + "/api/wormhole/dm/compose", + "/api/mesh/report", + "/api/mesh/trust/vouch", + "/api/mesh/oracle/predict", + "/api/mesh/oracle/resolve", + "/api/mesh/oracle/stake", + "/api/mesh/oracle/resolve-stakes", + "/api/wormhole/gate/enter", + "/api/wormhole/gate/leave", + "/api/wormhole/gate/persona/create", + "/api/wormhole/gate/persona/activate", + "/api/wormhole/gate/persona/clear", + "/api/wormhole/gate/persona/retire", + "/api/wormhole/gate/key/grant", + "/api/wormhole/gate/key/rotate", + "/api/wormhole/gate/message/compose", + "/api/wormhole/gate/message/decrypt", + "/api/wormhole/gate/messages/decrypt", + "/api/wormhole/dm/decrypt", + }: + return "private_transitional" + + if method_name == "POST" and path in { + "/api/wormhole/dm/register-key", + "/api/wormhole/dm/prekey/register", + "/api/wormhole/dm/bootstrap-encrypt", + "/api/wormhole/dm/bootstrap-decrypt", + "/api/wormhole/dm/sender-token", + "/api/wormhole/dm/open-seal", + "/api/wormhole/dm/build-seal", + "/api/wormhole/dm/dead-drop-token", + "/api/wormhole/dm/pairwise-alias", + "/api/wormhole/dm/pairwise-alias/rotate", + "/api/wormhole/dm/dead-drop-tokens", + "/api/wormhole/dm/sas", + "/api/wormhole/dm/encrypt", + "/api/wormhole/dm/decrypt", + "/api/wormhole/dm/reset", + }: + return "private_strong" + + return "" + + +def _transport_tier_precondition(required_tier: str, current_tier: str) -> JSONResponse: + return JSONResponse( + status_code=428, + content={ + "ok": False, + "detail": "transport tier insufficient", + "required": required_tier, + "current": current_tier, + }, + ) + + +def _private_infonet_policy_snapshot() -> dict[str, Any]: + return { + "gate_actions": { + "post_message": "private_transitional", + "vote": "private_transitional", + "create_gate": "private_transitional", + }, + "gate_chat": { + "trust_tier": "private_transitional", + "wormhole_required": True, + "content_private": True, + "storage_model": "private_gate_store_encrypted_envelope", + "notes": [ + "Gate messages stay off the public hashchain and live on the private gate plane.", + "Anonymous gate sessions use rotating gate-scoped public keys and can participate on the private gate lane.", + "Use the DM/Dead Drop lane for the strongest transport posture currently available.", + ], + }, + "dm_lane": { + "trust_tier_when_wormhole_ready": "private_transitional", + "trust_tier_when_rns_ready": "private_strong", + "reticulum_preferred": True, + "relay_fallback": True, + "public_transports_excluded": True, + "notes": [ + "Private DMs stay off the public hashchain.", + "Public perimeter transports are excluded from secure DM carriage.", + ], + }, + "reserved_for_private_strong": [], + "notes": [ + "Non-DM gate chat and gate lifecycle actions are currently allowed in PRIVATE / TRANSITIONAL once Wormhole is ready.", + "DM policy remains stricter and is intentionally managed separately from gate-chat policy.", + ], + } + + +@app.middleware("http") +async def enforce_high_privacy_mesh(request: Request, call_next): + path = request.url.path + if path.startswith("/api/mesh") or path.startswith("/api/wormhole/gate/") or path.startswith("/api/wormhole/dm/"): + current_tier = "public_degraded" + required_tier = _minimum_transport_tier(path, request.method) + if required_tier: + try: + from services.wormhole_supervisor import get_wormhole_state + + wormhole = get_wormhole_state() + except Exception: + wormhole = {"configured": False, "ready": False, "rns_ready": False} + current_tier = _current_private_lane_tier(wormhole) + if not _transport_tier_is_sufficient(current_tier, required_tier): + return _transport_tier_precondition(required_tier, current_tier) + try: + from services.wormhole_settings import read_wormhole_settings + + data = read_wormhole_settings() + if ( + path.startswith("/api/mesh") + and str(data.get("privacy_profile", "default")).lower() == "high" + and not bool(data.get("enabled")) + ): + return JSONResponse( + status_code=428, + content={ + "ok": False, + "detail": "High privacy requires Wormhole to be enabled.", + }, + ) + except Exception: + pass + state = _anonymous_mode_state() + if state["enabled"] and ( + _is_anonymous_mesh_write_path(path, request.method) + or _is_anonymous_dm_action_path(path, request.method) + or _is_anonymous_wormhole_gate_admin_path(path, request.method) + ): + if not state["wormhole_enabled"]: + return JSONResponse( + status_code=428, + content={ + "ok": False, + "detail": "Anonymous mode requires Wormhole to be enabled.", + }, + ) + if not state["ready"]: + return JSONResponse( + status_code=428, + content={ + "ok": False, + "detail": ( + "Anonymous mode requires a hidden Wormhole transport " + "(Tor/I2P/Mixnet) to be ready before public posting, " + "gate persona changes, or private DM activity." + ), + }, + ) + return await call_next(request) + + +@app.middleware("http") +async def apply_no_store_to_sensitive_paths(request: Request, call_next): + response = await call_next(request) + if _is_sensitive_no_store_path(request.url.path): + for key, value in _NO_STORE_HEADERS.items(): + response.headers[key] = value + return response + +from services.data_fetcher import update_all_data + +_refresh_lock = threading.Lock() + + +@app.get("/api/refresh", response_model=RefreshResponse, dependencies=[Depends(require_admin)]) +@limiter.limit("2/minute") +async def force_refresh(request: Request): + if not _refresh_lock.acquire(blocking=False): + return {"status": "refresh already in progress"} + + def _do_refresh(): + try: + update_all_data() finally: _refresh_lock.release() + t = threading.Thread(target=_do_refresh) t.start() return {"status": "refreshing in background"} + @app.post("/api/ais/feed") @limiter.limit("60/minute") async def ais_feed(request: Request): """Accept AIS-catcher HTTP JSON feed (POST decoded AIS messages).""" from services.ais_stream import ingest_ais_catcher + try: body = await request.json() except Exception: - return Response(content='{"error":"invalid JSON"}', status_code=400, media_type="application/json") + return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"}) msgs = body.get("msgs", []) if not msgs: @@ -184,47 +2022,229 @@ async def ais_feed(request: Request): count = ingest_ais_catcher(msgs) return {"status": "ok", "ingested": count} + from pydantic import BaseModel + + class ViewportUpdate(BaseModel): s: float w: float n: float e: float + +_LAST_VIEWPORT_UPDATE: tuple[float, float, float, float] | None = None +_LAST_VIEWPORT_UPDATE_TS = 0.0 +_VIEWPORT_UPDATE_LOCK = threading.Lock() +_VIEWPORT_DEDUPE_EPSILON = 1.0 +_VIEWPORT_MIN_UPDATE_S = 10.0 + + +def _normalize_longitude(value: float) -> float: + normalized = ((value + 180.0) % 360.0 + 360.0) % 360.0 - 180.0 + if normalized == -180.0 and value > 0: + return 180.0 + return normalized + + +def _normalize_viewport_bounds(s: float, w: float, n: float, e: float) -> tuple[float, float, float, float]: + south = max(-90.0, min(90.0, s)) + north = max(-90.0, min(90.0, n)) + raw_width = abs(e - w) + if not math.isfinite(raw_width) or raw_width >= 360.0: + return south, -180.0, north, 180.0 + west = _normalize_longitude(w) + east = _normalize_longitude(e) + if east < west: + return south, -180.0, north, 180.0 + return south, west, north, east + + +def _viewport_changed_enough(bounds: tuple[float, float, float, float]) -> bool: + global _LAST_VIEWPORT_UPDATE, _LAST_VIEWPORT_UPDATE_TS + now = time.monotonic() + with _VIEWPORT_UPDATE_LOCK: + if _LAST_VIEWPORT_UPDATE is None: + _LAST_VIEWPORT_UPDATE = bounds + _LAST_VIEWPORT_UPDATE_TS = now + return True + changed = any( + abs(current - previous) > _VIEWPORT_DEDUPE_EPSILON + for current, previous in zip(bounds, _LAST_VIEWPORT_UPDATE) + ) + if not changed and (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S: + return False + if (now - _LAST_VIEWPORT_UPDATE_TS) < _VIEWPORT_MIN_UPDATE_S: + return False + _LAST_VIEWPORT_UPDATE = bounds + _LAST_VIEWPORT_UPDATE_TS = now + return True + + +def _queue_viirs_change_refresh() -> None: + from services.fetchers.earth_observation import fetch_viirs_change_nodes + + threading.Thread(target=fetch_viirs_change_nodes, daemon=True).start() + + @app.post("/api/viewport") @limiter.limit("60/minute") async def update_viewport(vp: ViewportUpdate, request: Request): """Receive frontend map bounds to dynamically choke the AIS stream.""" from services.ais_stream import update_ais_bbox + + south, west, north, east = _normalize_viewport_bounds(vp.s, vp.w, vp.n, vp.e) + normalized_bounds = (south, west, north, east) + + if not _viewport_changed_enough(normalized_bounds): + return {"status": "ok", "deduped": True} + # Add a gentle 10% padding so ships don't pop-in right at the edge - pad_lat = (vp.n - vp.s) * 0.1 + pad_lat = (north - south) * 0.1 # handle antimeridian bounding box padding later if needed, simple for now: - pad_lng = (vp.e - vp.w) * 0.1 if vp.e > vp.w else 0 - + pad_lng = (east - west) * 0.1 if east > west else 0 + update_ais_bbox( - south=max(-90, vp.s - pad_lat), - west=max(-180, vp.w - pad_lng) if pad_lng else vp.w, - north=min(90, vp.n + pad_lat), - east=min(180, vp.e + pad_lng) if pad_lng else vp.e + south=max(-90, south - pad_lat), + west=max(-180, west - pad_lng) if pad_lng else west, + north=min(90, north + pad_lat), + east=min(180, east + pad_lng) if pad_lng else east, + ) + return {"status": "ok"} + + +class LayerUpdate(BaseModel): + layers: dict[str, bool] + + +@app.post("/api/layers") +@limiter.limit("30/minute") +async def update_layers(update: LayerUpdate, request: Request): + """Receive frontend layer toggle state. Starts/stops streams accordingly.""" + from services.fetchers._store import active_layers, bump_active_layers_version, is_any_active + + # Snapshot old stream states before applying changes + old_ships = is_any_active( + "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts" + ) + old_mesh = is_any_active("sigint_meshtastic") + old_aprs = is_any_active("sigint_aprs") + old_viirs = is_any_active("viirs_nightlights") + + # Update only known keys + changed = False + for key, value in update.layers.items(): + if key in active_layers: + if active_layers[key] != value: + changed = True + active_layers[key] = value + + if changed: + bump_active_layers_version() + + new_ships = is_any_active( + "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts" ) + new_mesh = is_any_active("sigint_meshtastic") + new_aprs = is_any_active("sigint_aprs") + new_viirs = is_any_active("viirs_nightlights") + + # Start/stop AIS stream on transition + if old_ships and not new_ships: + from services.ais_stream import stop_ais_stream + + stop_ais_stream() + logger.info("AIS stream stopped (all ship layers disabled)") + elif not old_ships and new_ships: + from services.ais_stream import start_ais_stream + + start_ais_stream() + logger.info("AIS stream started (ship layer enabled)") + + # Start/stop SIGINT bridges on transition + from services.sigint_bridge import sigint_grid + + if old_mesh and not new_mesh: + sigint_grid.mesh.stop() + logger.info("Meshtastic MQTT bridge stopped (layer disabled)") + elif not old_mesh and new_mesh: + sigint_grid.mesh.start() + logger.info("Meshtastic MQTT bridge started (layer enabled)") + + if old_aprs and not new_aprs: + sigint_grid.aprs.stop() + logger.info("APRS bridge stopped (layer disabled)") + elif not old_aprs and new_aprs: + sigint_grid.aprs.start() + logger.info("APRS bridge started (layer enabled)") + + if not old_viirs and new_viirs: + _queue_viirs_change_refresh() + logger.info("VIIRS change refresh queued (layer enabled)") + return {"status": "ok"} + @app.get("/api/live-data") @limiter.limit("120/minute") async def live_data(request: Request): return get_latest_data() + def _etag_response(request: Request, payload: dict, prefix: str = "", default=None): - """Serialize once, hash the bytes for ETag, return 304 or full response.""" - content = json_mod.dumps(payload, default=default) - etag = hashlib.md5(f"{prefix}{content}".encode()).hexdigest()[:16] + """Serialize once, use data version for ETag, return 304 or full response. + + Uses a monotonic version counter instead of MD5-hashing the full payload. + The 304 fast path avoids serialization entirely. + """ + etag = _current_etag(prefix) if request.headers.get("if-none-match") == etag: return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"}) - return Response(content=content, media_type="application/json", - headers={"ETag": etag, "Cache-Control": "no-cache"}) + content = json_mod.dumps(_json_safe(payload), default=default, allow_nan=False) + return Response( + content=content, + media_type="application/json", + headers={"ETag": etag, "Cache-Control": "no-cache"}, + ) -def _bbox_filter(items: list, s: float, w: float, n: float, e: float, - lat_key: str = "lat", lng_key: str = "lng") -> list: + +def _current_etag(prefix: str = "") -> str: + from services.fetchers._store import get_active_layers_version, get_data_version + + return f"{prefix}v{get_data_version()}-l{get_active_layers_version()}" + + +def _json_safe(value): + """Recursively replace non-finite floats with None so responses stay valid JSON.""" + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + # Snapshot mutable mappings first so background fetcher updates do not + # invalidate iteration while we serialize a response. + return {k: _json_safe(v) for k, v in list(value.items())} + if isinstance(value, list): + return [_json_safe(v) for v in list(value)] + if isinstance(value, tuple): + return [_json_safe(v) for v in list(value)] + return value + + +def _sanitize_payload(value): + """Thread-safe snapshot with NaN→None. Cheaper than _json_safe: only deep- + copies dicts (for thread safety) and replaces non-finite floats. Lists are + shallow-copied — orjson handles the leaf serialisation natively.""" + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {k: _sanitize_payload(v) for k, v in list(value.items())} + if isinstance(value, (list, tuple)): + return list(value) + return value + + +def _bbox_filter( + items: list, s: float, w: float, n: float, e: float, lat_key: str = "lat", lng_key: str = "lng" +) -> list: """Filter a list of dicts to those within the bounding box (with 20% padding). Handles antimeridian crossing (e.g. w=170, e=-170).""" pad_lat = (n - s) * 0.2 @@ -249,246 +2269,5845 @@ def _bbox_filter(items: list, s: float, w: float, n: float, e: float, out.append(item) return out + +def _bbox_filter_geojson_points(items: list, s: float, w: float, n: float, e: float) -> list: + """Filter GeoJSON Point features to a padded bounding box.""" + pad_lat = (n - s) * 0.2 + pad_lng = (e - w) * 0.2 if e > w else ((e + 360 - w) * 0.2) + s2, n2 = s - pad_lat, n + pad_lat + w2, e2 = w - pad_lng, e + pad_lng + crosses_antimeridian = w2 > e2 + out = [] + for item in items: + geometry = item.get("geometry") if isinstance(item, dict) else None + coords = geometry.get("coordinates") if isinstance(geometry, dict) else None + if not isinstance(coords, (list, tuple)) or len(coords) < 2: + out.append(item) + continue + lng, lat = coords[0], coords[1] + if lat is None or lng is None: + out.append(item) + continue + if not (s2 <= lat <= n2): + continue + if crosses_antimeridian: + if lng >= w2 or lng <= e2: + out.append(item) + else: + if w2 <= lng <= e2: + out.append(item) + return out + + +def _bbox_spans(s: float | None, w: float | None, n: float | None, e: float | None) -> tuple[float, float]: + if None in (s, w, n, e): + return 180.0, 360.0 + lat_span = max(0.0, float(n) - float(s)) + lng_span = float(e) - float(w) + if lng_span < 0: + lng_span += 360.0 + if lng_span == 0 and w == -180 and e == 180: + lng_span = 360.0 + return lat_span, max(0.0, lng_span) + + +def _downsample_points(items: list, max_items: int) -> list: + if max_items <= 0 or len(items) <= max_items: + return items + step = len(items) / float(max_items) + return [items[min(len(items) - 1, int(i * step))] for i in range(max_items)] + + +def _world_and_continental_scale( + has_bbox: bool, s: float | None, w: float | None, n: float | None, e: float | None +) -> tuple[bool, bool]: + lat_span, lng_span = _bbox_spans(s, w, n, e) + world_scale = (not has_bbox) or lng_span >= 300 or lat_span >= 120 + continental_scale = has_bbox and not world_scale and (lng_span >= 120 or lat_span >= 55) + return world_scale, continental_scale + + +def _filter_sigint_by_layers(items: list, active_layers: dict[str, bool]) -> list: + allow_aprs = bool(active_layers.get("sigint_aprs", True)) + allow_mesh = bool(active_layers.get("sigint_meshtastic", True)) + if allow_aprs and allow_mesh: + return items + + allowed_sources: set[str] = {"js8call"} + if allow_aprs: + allowed_sources.add("aprs") + if allow_mesh: + allowed_sources.update({"meshtastic", "meshtastic-map"}) + return [item for item in items if str(item.get("source") or "").lower() in allowed_sources] + + +def _sigint_totals_for_items(items: list) -> dict[str, int]: + totals = { + "total": len(items), + "meshtastic": 0, + "meshtastic_live": 0, + "meshtastic_map": 0, + "aprs": 0, + "js8call": 0, + } + for item in items: + source = str(item.get("source") or "").lower() + if source == "meshtastic": + totals["meshtastic"] += 1 + if bool(item.get("from_api")): + totals["meshtastic_map"] += 1 + else: + totals["meshtastic_live"] += 1 + elif source == "aprs": + totals["aprs"] += 1 + elif source == "js8call": + totals["js8call"] += 1 + return totals + + @app.get("/api/live-data/fast") @limiter.limit("120/minute") -async def live_data_fast(request: Request, - s: float = Query(None, description="South bound"), - w: float = Query(None, description="West bound"), - n: float = Query(None, description="North bound"), - e: float = Query(None, description="East bound")): - d = get_latest_data() - has_bbox = all(v is not None for v in (s, w, n, e)) - def _f(items, lat_key="lat", lng_key="lng"): - return _bbox_filter(items, s, w, n, e, lat_key, lng_key) if has_bbox else items +async def live_data_fast( + request: Request, + # bbox params accepted for backward compat but no longer used for filtering — + # all cached data is returned and the frontend culls off-screen entities via MapLibre. + s: float = Query(None, description="South bound (ignored)", ge=-90, le=90), + w: float = Query(None, description="West bound (ignored)", ge=-180, le=180), + n: float = Query(None, description="North bound (ignored)", ge=-90, le=90), + e: float = Query(None, description="East bound (ignored)", ge=-180, le=180), +): + etag = _current_etag(prefix="fast|full|") + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"}) + + from services.fetchers._store import ( + active_layers, + get_latest_data_subset, + get_source_timestamps_snapshot, + ) + + d = get_latest_data_subset( + "last_updated", + "commercial_flights", + "military_flights", + "private_flights", + "private_jets", + "tracked_flights", + "ships", + "cctv", + "uavs", + "liveuamap", + "gps_jamming", + "satellites", + "satellite_source", + "sigint", + "sigint_totals", + "trains", + ) + freshness = get_source_timestamps_snapshot() + + ships_enabled = any( + active_layers.get(key, True) + for key in ( + "ships_military", + "ships_cargo", + "ships_civilian", + "ships_passenger", + "ships_tracked_yachts", + ) + ) + cctv_total = len(d.get("cctv") or []) + sigint_items = _filter_sigint_by_layers(d.get("sigint") or [], active_layers) + sigint_totals = _sigint_totals_for_items(sigint_items) + payload = { - "commercial_flights": _f(d.get("commercial_flights", [])), - "military_flights": _f(d.get("military_flights", [])), - "private_flights": _f(d.get("private_flights", [])), - "private_jets": _f(d.get("private_jets", [])), - "tracked_flights": d.get("tracked_flights", []), # Always send tracked (small set) - "ships": _f(d.get("ships", [])), - "cctv": _f(d.get("cctv", []), lat_key="lat", lng_key="lon"), - "uavs": _f(d.get("uavs", [])), - "liveuamap": _f(d.get("liveuamap", [])), - "gps_jamming": _f(d.get("gps_jamming", [])), - "satellites": _f(d.get("satellites", [])), + "commercial_flights": (d.get("commercial_flights") or []) if active_layers.get("flights", True) else [], + "military_flights": (d.get("military_flights") or []) if active_layers.get("military", True) else [], + "private_flights": (d.get("private_flights") or []) if active_layers.get("private", True) else [], + "private_jets": (d.get("private_jets") or []) if active_layers.get("jets", True) else [], + "tracked_flights": (d.get("tracked_flights") or []) if active_layers.get("tracked", True) else [], + "ships": (d.get("ships") or []) if ships_enabled else [], + "cctv": (d.get("cctv") or []) if active_layers.get("cctv", True) else [], + "uavs": (d.get("uavs") or []) if active_layers.get("military", True) else [], + "liveuamap": (d.get("liveuamap") or []) if active_layers.get("global_incidents", True) else [], + "gps_jamming": (d.get("gps_jamming") or []) if active_layers.get("gps_jamming", True) else [], + "satellites": (d.get("satellites") or []) if active_layers.get("satellites", True) else [], "satellite_source": d.get("satellite_source", "none"), - "freshness": dict(source_timestamps), + "sigint": sigint_items + if (active_layers.get("sigint_meshtastic", True) or active_layers.get("sigint_aprs", True)) + else [], + "sigint_totals": sigint_totals, + "cctv_total": cctv_total, + "trains": (d.get("trains") or []) if active_layers.get("trains", True) else [], + "freshness": freshness, } - bbox_tag = f"{s},{w},{n},{e}" if has_bbox else "full" - return _etag_response(request, payload, prefix=f"fast|{bbox_tag}|") + return Response( + content=orjson.dumps(_sanitize_payload(payload)), + media_type="application/json", + headers={"ETag": etag, "Cache-Control": "no-cache"}, + ) + @app.get("/api/live-data/slow") @limiter.limit("60/minute") -async def live_data_slow(request: Request, - s: float = Query(None, description="South bound"), - w: float = Query(None, description="West bound"), - n: float = Query(None, description="North bound"), - e: float = Query(None, description="East bound")): - d = get_latest_data() - has_bbox = all(v is not None for v in (s, w, n, e)) - def _f(items, lat_key="lat", lng_key="lng"): - return _bbox_filter(items, s, w, n, e, lat_key, lng_key) if has_bbox else items +async def live_data_slow( + request: Request, + # bbox params accepted for backward compat but no longer used for filtering. + s: float = Query(None, description="South bound (ignored)", ge=-90, le=90), + w: float = Query(None, description="West bound (ignored)", ge=-180, le=180), + n: float = Query(None, description="North bound (ignored)", ge=-90, le=90), + e: float = Query(None, description="East bound (ignored)", ge=-180, le=180), +): + etag = _current_etag(prefix="slow|full|") + if request.headers.get("if-none-match") == etag: + return Response(status_code=304, headers={"ETag": etag, "Cache-Control": "no-cache"}) + + from services.fetchers._store import ( + active_layers, + get_latest_data_subset, + get_source_timestamps_snapshot, + ) + + d = get_latest_data_subset( + "last_updated", + "news", + "stocks", + "financial_source", + "oil", + "weather", + "traffic", + "earthquakes", + "frontlines", + "gdelt", + "airports", + "kiwisdr", + "satnogs_stations", + "satnogs_observations", + "tinygs_satellites", + "space_weather", + "internet_outages", + "firms_fires", + "datacenters", + "military_bases", + "power_plants", + "viirs_change_nodes", + "scanners", + "weather_alerts", + "ukraine_alerts", + "air_quality", + "volcanoes", + "fishing_activity", + "psk_reporter", + "correlations", + "threat_level", + "trending_markets", + ) + freshness = get_source_timestamps_snapshot() + payload = { "last_updated": d.get("last_updated"), - "news": d.get("news", []), # News has coords but we always send it (small set, important) + "threat_level": d.get("threat_level"), + "trending_markets": d.get("trending_markets", []), + "news": d.get("news", []), "stocks": d.get("stocks", {}), + "financial_source": d.get("financial_source", ""), "oil": d.get("oil", {}), "weather": d.get("weather"), "traffic": d.get("traffic", []), - "earthquakes": _f(d.get("earthquakes", [])), - "frontlines": d.get("frontlines"), # Always send (GeoJSON polygon, not point-filterable) - "gdelt": d.get("gdelt", []), # GeoJSON features — filtered client-side - "airports": d.get("airports", []), # Always send (reference data) - "kiwisdr": _f(d.get("kiwisdr", []), lat_key="lat", lng_key="lon"), + "earthquakes": (d.get("earthquakes") or []) if active_layers.get("earthquakes", True) else [], + "frontlines": d.get("frontlines") if active_layers.get("ukraine_frontline", True) else None, + "gdelt": (d.get("gdelt") or []) if active_layers.get("global_incidents", True) else [], + "airports": d.get("airports") or [], + "kiwisdr": (d.get("kiwisdr") or []) if active_layers.get("kiwisdr", True) else [], + "satnogs_stations": (d.get("satnogs_stations") or []) if active_layers.get("satnogs", True) else [], + "satnogs_total": len(d.get("satnogs_stations") or []), + "satnogs_observations": (d.get("satnogs_observations") or []) if active_layers.get("satnogs", True) else [], + "tinygs_satellites": (d.get("tinygs_satellites") or []) if active_layers.get("tinygs", True) else [], + "tinygs_total": len(d.get("tinygs_satellites") or []), + "psk_reporter": (d.get("psk_reporter") or []) if active_layers.get("psk_reporter", True) else [], "space_weather": d.get("space_weather"), - "internet_outages": _f(d.get("internet_outages", [])), - "firms_fires": _f(d.get("firms_fires", [])), - "datacenters": _f(d.get("datacenters", [])), - "military_bases": _f(d.get("military_bases", [])), - "power_plants": _f(d.get("power_plants", [])), - "freshness": dict(source_timestamps), + "internet_outages": (d.get("internet_outages") or []) if active_layers.get("internet_outages", True) else [], + "firms_fires": (d.get("firms_fires") or []) if active_layers.get("firms", True) else [], + "datacenters": (d.get("datacenters") or []) if active_layers.get("datacenters", True) else [], + "military_bases": (d.get("military_bases") or []) if active_layers.get("military_bases", True) else [], + "power_plants": (d.get("power_plants") or []) if active_layers.get("power_plants", True) else [], + "viirs_change_nodes": (d.get("viirs_change_nodes") or []) if active_layers.get("viirs_nightlights", True) else [], + "scanners": (d.get("scanners") or []) if active_layers.get("scanners", True) else [], + "weather_alerts": d.get("weather_alerts", []) if active_layers.get("weather_alerts", True) else [], + "ukraine_alerts": d.get("ukraine_alerts", []) if active_layers.get("ukraine_alerts", True) else [], + "air_quality": (d.get("air_quality") or []) if active_layers.get("air_quality", True) else [], + "volcanoes": (d.get("volcanoes") or []) if active_layers.get("volcanoes", True) else [], + "fishing_activity": (d.get("fishing_activity") or []) if active_layers.get("fishing_activity", True) else [], + "correlations": (d.get("correlations") or []) if active_layers.get("correlations", True) else [], + "freshness": freshness, } - bbox_tag = f"{s},{w},{n},{e}" if has_bbox else "full" - return _etag_response(request, payload, prefix=f"slow|{bbox_tag}|", default=str) - -@app.get("/api/debug-latest") -@limiter.limit("30/minute") -async def debug_latest_data(request: Request): - return list(get_latest_data().keys()) + return Response( + content=orjson.dumps( + _sanitize_payload(payload), + default=str, + option=orjson.OPT_NON_STR_KEYS, + ), + media_type="application/json", + headers={"ETag": etag, "Cache-Control": "no-cache"}, + ) -@app.get("/api/health", response_model=HealthResponse) +@app.get("/api/oracle/region-intel") @limiter.limit("30/minute") -async def health_check(request: Request): +async def oracle_region_intel( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), +): + """Get oracle intelligence summary for a geographic region.""" + from services.oracle_service import get_region_oracle_intel + + news_items = get_latest_data().get("news", []) + return get_region_oracle_intel(lat, lng, news_items) + + +@app.get("/api/thermal/verify") +@limiter.limit("10/minute") +async def thermal_verify( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), + radius_km: float = Query(10, ge=1, le=100), +): + """On-demand thermal anomaly verification using Sentinel-2 SWIR bands.""" + from services.thermal_sentinel import search_thermal_anomaly + + result = search_thermal_anomaly(lat, lng, radius_km) + return result + + +@app.post("/api/sigint/transmit") +@limiter.limit("5/minute") +async def sigint_transmit(request: Request): + """Send an APRS-IS message to a specific callsign. Requires ham radio credentials.""" + from services.wormhole_supervisor import get_transport_tier + + tier = get_transport_tier() + if str(tier or "").startswith("private_"): + return {"ok": False, "detail": "APRS transmit blocked in private transport mode"} + body = await request.json() + callsign = body.get("callsign", "") + passcode = body.get("passcode", "") + target = body.get("target", "") + message = body.get("message", "") + if not all([callsign, passcode, target, message]): + return { + "ok": False, + "detail": "Missing required fields: callsign, passcode, target, message", + } + from services.sigint_bridge import send_aprs_message + + return send_aprs_message(callsign, passcode, target, message) + + +@app.get("/api/sigint/nearest-sdr") +@limiter.limit("30/minute") +async def nearest_sdr( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), +): + """Find the nearest KiwiSDR receivers to a given coordinate.""" + from services.sigint_bridge import find_nearest_kiwisdr + + kiwisdr_data = get_latest_data().get("kiwisdr", []) + return find_nearest_kiwisdr(lat, lng, kiwisdr_data) + + +# ─── Per-Identity Throttle State ────────────────────────────────────────── +# In-memory: {node_id: {"last_send": timestamp, "daily_count": int, "daily_reset": timestamp}} +# Bounded to 10000 entries with 24hr TTL to prevent unbounded memory growth +_node_throttle: TTLCache = TTLCache(maxsize=10000, ttl=86400) +_gate_post_cooldown: TTLCache = TTLCache(maxsize=20000, ttl=86400) + +# Byte limits per payload type +_BYTE_LIMITS = {"text": 200, "pin": 300, "emergency": 200, "command": 200} + + +def _check_throttle( + node_id: str, priority_str: str, transport_lock: str = "" +) -> tuple[bool, str]: + """Per-identity rate limiting based on node age and reputation. + + Tiers: + New (rep < 3, age < 24h): 1 msg / 5 min, 10/day + Established (rep >= 3 OR > 24h): 1 msg / 2 min, 50/day + Trusted (rep >= 10): 1 msg / 30 sec, 200/day + Emergency: no throttle + + Meshtastic public mesh is intentionally looser in testnet mode: + Any public mesh sender: 2 msgs / min, tier caps unchanged + """ + if priority_str == "emergency": + return True, "" + + now = time.time() + state = _node_throttle.get(node_id) + if not state: + _node_throttle[node_id] = { + "last_send": 0, + "daily_count": 0, + "daily_reset": now, + "first_seen": now, + } + state = _node_throttle[node_id] + + # Reset daily counter at midnight + if now - state["daily_reset"] > 86400: + state["daily_count"] = 0 + state["daily_reset"] = now + + # Determine tier (reputation integration will come with Feature 2) + age_hours = (now - state.get("first_seen", now)) / 3600 + rep_score = 0 + try: + from services.mesh.mesh_reputation import reputation_ledger + + rep_score = reputation_ledger.get_reputation(node_id).get("overall", 0) + age_hours = max(age_hours, reputation_ledger.get_node_age_days(node_id) * 24) + except Exception: + rep_score = 0 + + if rep_score >= 20 or age_hours >= 168: + interval, daily_cap, tier = 30, 200, "trusted" + elif rep_score >= 5 or age_hours >= 48: + interval, daily_cap, tier = 120, 75, "established" + else: + interval, daily_cap, tier = 300, 15, "new" + + if str(transport_lock or "").lower() == "meshtastic": + interval = min(interval, 30) + + # Check daily cap + if state["daily_count"] >= daily_cap: + return ( + False, + f"Daily message limit reached ({daily_cap} messages for {tier} nodes). Resets in {int(86400 - (now - state['daily_reset']))}s.", + ) + + # Check interval + elapsed = now - state["last_send"] + if elapsed < interval: + remaining = int(interval - elapsed) + return False, f"Rate limit: 1 message per {interval}s for {tier} nodes. Wait {remaining}s." + + # Allowed + state["last_send"] = now + state["daily_count"] += 1 + return True, "" + + +def _check_gate_post_cooldown(sender_id: str, gate_id: str) -> tuple[bool, str]: + """Check cooldown — does NOT record it. Call _record_gate_post_cooldown() after success.""" + gate_key = str(gate_id or "").strip().lower() + sender_key = str(sender_id or "").strip() + if not gate_key or not sender_key: + return True, "" + now = time.time() + cooldown_key = f"{sender_key}:{gate_key}" + last_post = float(_gate_post_cooldown.get(cooldown_key, 0) or 0) + if last_post > 0: + elapsed = now - last_post + if elapsed < 30: + remaining = max(1, math.ceil(30 - elapsed)) + return False, f"Gate post cooldown: wait {remaining}s before posting again." + return True, "" + + +def _record_gate_post_cooldown(sender_id: str, gate_id: str) -> None: + """Stamp the cooldown AFTER a successful gate post.""" + gate_key = str(gate_id or "").strip().lower() + sender_key = str(sender_id or "").strip() + if gate_key and sender_key: + _gate_post_cooldown[f"{sender_key}:{gate_key}"] = time.time() + + +def _verify_signed_event( + *, + event_type: str, + node_id: str, + sequence: int, + public_key: str, + public_key_algo: str, + signature: str, + payload: dict, + protocol_version: str, +) -> tuple[bool, str]: + from services.mesh.mesh_metrics import increment as metrics_inc + + if not protocol_version: + metrics_inc("signature_missing_protocol") + return False, "Missing protocol_version" + + if protocol_version != PROTOCOL_VERSION: + metrics_inc("signature_protocol_mismatch") + return False, f"Unsupported protocol_version: {protocol_version}" + + if not signature or not public_key or not public_key_algo: + metrics_inc("signature_missing_fields") + return False, "Missing signature or public key" + + if sequence <= 0: + metrics_inc("signature_invalid_sequence") + return False, "Missing or invalid sequence" + + if not verify_node_binding(node_id, public_key): + metrics_inc("signature_node_mismatch") + return False, "node_id does not match public key" + + algo = parse_public_key_algo(public_key_algo) + if not algo: + metrics_inc("signature_bad_algo") + return False, "Unsupported public_key_algo" + + normalized = normalize_payload(event_type, payload) + sig_payload = build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=normalized, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=algo, + signature_hex=signature, + payload=sig_payload, + ): + if event_type == "dm_message": + legacy_sig_payload = build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=normalize_dm_message_payload_legacy(payload), + ) + if verify_signature( + public_key_b64=public_key, + public_key_algo=algo, + signature_hex=signature, + payload=legacy_sig_payload, + ): + return True, "ok" + metrics_inc("signature_invalid") + return False, "Invalid signature" + + return True, "ok" + + +def _preflight_signed_event_integrity( + *, + event_type: str, + node_id: str, + sequence: int, + public_key: str, + public_key_algo: str, + signature: str, + protocol_version: str, +) -> tuple[bool, str]: + if not protocol_version or not signature or not public_key or not public_key_algo: + return False, "Missing signature or public key" + + if sequence <= 0: + return False, "Missing or invalid sequence" + + try: + from services.mesh.mesh_hashchain import infonet + except Exception as exc: + logger.error("Signed event integrity preflight unavailable: %s", exc) + return False, "Signed event integrity preflight unavailable" + + if infonet.check_replay(node_id, sequence): + last = infonet.node_sequences.get(node_id, 0) + return False, f"Replay detected: sequence {sequence} <= last {last}" + + existing = infonet.public_key_bindings.get(public_key) + if existing and existing != node_id: + return False, f"public key already bound to {existing}" + + revoked, _info = infonet._revocation_status(public_key) + if revoked and event_type != "key_revoke": + return False, "public key is revoked" + + return True, "ok" + + +@app.post("/api/mesh/send") +@limiter.limit("10/minute") +async def mesh_send(request: Request): + """Unified mesh message endpoint — auto-routes via optimal transport. + + Body: { destination, message, priority?, channel?, node_id?, credentials? } + The router picks APRS, Meshtastic, or Internet based on gate logic. + Enforces byte limits and per-identity rate limiting. + """ + body = await request.json() + destination = body.get("destination", "") + message = body.get("message", "") + if not destination or not message: + return {"ok": False, "detail": "Missing required fields: destination, message"} + + # ─── Byte limit enforcement ─────────────────────────────────── + payload_bytes = len(message.encode("utf-8")) + payload_type = body.get("payload_type", "text") + max_bytes = _BYTE_LIMITS.get(payload_type, 200) + if payload_bytes > max_bytes: + return { + "ok": False, + "detail": f"Message too long ({payload_bytes} bytes). Maximum: {max_bytes} bytes for {payload_type} messages.", + } + + # ─── Signature verification & node registration ────────────── + node_id = body.get("node_id", body.get("sender_id", "anonymous")) + public_key = body.get("public_key", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("signature", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + signed_payload = { + "message": message, + "destination": destination, + "channel": body.get("channel", "LongFast"), + "priority": body.get("priority", "normal").lower(), + "ephemeral": bool(body.get("ephemeral", False)), + } + if body.get("transport_lock"): + signed_payload["transport_lock"] = str(body.get("transport_lock")) + sig_ok, sig_reason = _verify_signed_event( + event_type="message", + node_id=node_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=signed_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="message", + node_id=node_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + # Register node in reputation ledger (auto-creates if new) + if node_id != "anonymous": + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(node_id, public_key, public_key_algo) + except Exception: + pass # Non-critical — don't block sends if reputation module fails + + # ─── Per-identity throttle ──────────────────────────────────── + priority_str = signed_payload["priority"] + transport_lock = str(body.get("transport_lock", "") or "").lower() + throttle_ok, throttle_reason = _check_throttle(node_id, priority_str, transport_lock) + if not throttle_ok: + return {"ok": False, "detail": throttle_reason} + + from services.mesh.mesh_router import ( + MeshEnvelope, + MeshtasticTransport, + Priority, + TransportResult, + mesh_router, + ) + + priority_map = { + "emergency": Priority.EMERGENCY, + "high": Priority.HIGH, + "normal": Priority.NORMAL, + "low": Priority.LOW, + } + priority = priority_map.get(priority_str, Priority.NORMAL) + + # ─── C-1 fix: compute trust_tier from Wormhole state ─────── + from services.wormhole_supervisor import get_transport_tier + + computed_tier = get_transport_tier() + + envelope = MeshEnvelope( + sender_id=node_id, + destination=destination, + channel=body.get("channel", "LongFast"), + priority=priority, + payload=message, + ephemeral=body.get("ephemeral", False), + trust_tier=computed_tier, + ) + + credentials = body.get("credentials", {}) + # ─── C-2 fix: enforce tier before transport_lock dispatch ── + private_tier = str(envelope.trust_tier or "").startswith("private_") + if transport_lock == "meshtastic": + if private_tier: + results = [TransportResult( + False, "meshtastic", + "Private-tier content cannot be sent over Meshtastic" + )] + elif not mesh_router.meshtastic.can_reach(envelope): + results = [TransportResult(False, "meshtastic", "Message exceeds Meshtastic payload limit")] + else: + cb_ok, cb_reason = mesh_router.breakers["meshtastic"].check_and_record(envelope.priority) + if not cb_ok: + results = [TransportResult(False, "meshtastic", cb_reason)] + else: + envelope.route_reason = ( + "Transport locked to Meshtastic public path" + if MeshtasticTransport._parse_node_id(destination) is None + else "Transport locked to Meshtastic public node-targeted path" + ) + result = mesh_router.meshtastic.send(envelope, credentials) + if result.ok: + envelope.routed_via = mesh_router.meshtastic.NAME + results = [result] + elif transport_lock == "aprs": + if private_tier: + results = [TransportResult( + False, "aprs", + "Private-tier content cannot be sent over APRS" + )] + else: + results = mesh_router.route(envelope, credentials) + else: + results = mesh_router.route(envelope, credentials) + any_ok = any(r.ok for r in results) + + # ─── Mirror to Meshtastic bridge feed ──────────────────────── + # The MQTT broker won't echo our own publishes back to our subscriber, + # so inject successfully-sent messages into the bridge's deque directly. + if any_ok and envelope.routed_via == "meshtastic": + try: + from services.sigint_bridge import sigint_grid + + bridge = sigint_grid.mesh + if bridge: + from datetime import datetime + + bridge.messages.appendleft( + { + "from": MeshtasticTransport.mesh_address_for_sender(node_id), + "to": destination if MeshtasticTransport._parse_node_id(destination) is not None else "broadcast", + "text": message, + "region": credentials.get("mesh_region", "US"), + "channel": body.get("channel", "LongFast"), + "timestamp": datetime.utcnow().isoformat() + "Z", + } + ) + except Exception: + pass # Non-critical + + return { + "ok": any_ok, + "message_id": envelope.message_id, + "event_id": "", + "routed_via": envelope.routed_via, + "route_reason": envelope.route_reason, + "results": [r.to_dict() for r in results], + } + + +@app.get("/api/mesh/log") +@limiter.limit("30/minute") +async def mesh_log(request: Request): + """Get recent mesh message routing log (audit trail).""" + from services.mesh.mesh_router import mesh_router + + mesh_router.prune_message_log() + entries = list(mesh_router.message_log) + ok, _detail = _check_scoped_auth(request, "mesh.audit") + if ok: + return {"log": entries} + public_entries = [entry for entry in (_public_mesh_log_entry(item) for item in entries) if entry] + return {"log": public_entries} + + +@app.get("/api/mesh/status") +@limiter.limit("30/minute") +async def mesh_status(request: Request): + """Get mesh system status including circuit breaker state.""" + from services.env_check import get_security_posture_warnings + from services.mesh.mesh_router import mesh_router + from services.sigint_bridge import sigint_grid + + mesh_router.prune_message_log() + entries = list(mesh_router.message_log) + sigs = sigint_grid.get_all_signals() + aprs = sum(1 for s in sigs if s.get("source") == "aprs") + mesh = sum(1 for s in sigs if s.get("source") == "meshtastic") + js8 = sum(1 for s in sigs if s.get("source") == "js8call") + ok, _detail = _check_scoped_auth(request, "mesh.audit") + authenticated = _scoped_view_authenticated(request, "mesh.audit") + response = { + "circuit_breakers": { + name: breaker.get_status() for name, breaker in mesh_router.breakers.items() + }, + "message_log_size": len(entries) if ok else _public_mesh_log_size(entries), + "signal_counts": { + "aprs": aprs, + "meshtastic": mesh, + "js8call": js8, + "total": aprs + mesh + js8, + }, + } + if ok: + response["public_message_log_size"] = _public_mesh_log_size(entries) + response["private_log_retention_seconds"] = int( + getattr(get_settings(), "MESH_PRIVATE_LOG_TTL_S", 900) or 0 + ) + response["security_warnings"] = get_security_posture_warnings(get_settings()) + + return _redact_public_mesh_status(response, authenticated=authenticated) + + +@app.get("/api/mesh/signals") +@limiter.limit("30/minute") +async def mesh_signals( + request: Request, + source: str = "", + region: str = "", + root: str = "", + limit: int = 50, +): + """Get SIGINT signals with optional source/region/root filters.""" + from services.fetchers.sigint import build_sigint_snapshot + + sigs, _channel_stats, totals = build_sigint_snapshot() + if source: + sigs = [s for s in sigs if s.get("source") == source.lower()] + if region: + region_filter = region.upper() + sigs = [ + s + for s in sigs + if s.get("region", "").upper() == region_filter + or s.get("root", "").upper() == region_filter + ] + if root: + root_filter = root.upper() + sigs = [s for s in sigs if s.get("root", "").upper() == root_filter] + return { + "signals": sigs[: min(limit, 500)], + "total": len(sigs), + "source_totals": totals, + } + + +@app.get("/api/mesh/messages") +@limiter.limit("30/minute") +async def mesh_messages( + request: Request, + region: str = "", + root: str = "", + channel: str = "", + limit: int = 30, +): + """Get recent Meshtastic text messages from the MQTT bridge.""" + from services.sigint_bridge import sigint_grid + + bridge = sigint_grid.mesh + if not bridge: + return [] + msgs = list(bridge.messages) + if region: + region_filter = region.upper() + msgs = [ + m + for m in msgs + if m.get("region", "").upper() == region_filter + or m.get("root", "").upper() == region_filter + ] + if root: + root_filter = root.upper() + msgs = [m for m in msgs if m.get("root", "").upper() == root_filter] + if channel: + msgs = [m for m in msgs if m.get("channel", "").lower() == channel.lower()] + return msgs[: min(limit, 100)] + + +@app.get("/api/mesh/channels") +@limiter.limit("30/minute") +async def mesh_channels(request: Request): + """Get Meshtastic channel population stats — nodes per region/channel.""" + stats = get_latest_data().get("mesh_channel_stats", {}) + return stats + + +# ─── Reputation Endpoints ───────────────────────────────────────────────── + +# Cached root node_id — avoids 5 encrypted disk reads per vote. +_root_node_id_cache: dict[str, object] = {"value": None, "ts": 0.0} +_ROOT_NODE_ID_TTL = 30.0 # seconds + + +def _cached_root_node_id() -> str: + import time as _time + + now = _time.time() + if _root_node_id_cache["value"] is not None and (now - float(_root_node_id_cache["ts"])) < _ROOT_NODE_ID_TTL: + return str(_root_node_id_cache["value"]) + try: + from services.mesh.mesh_wormhole_persona import read_wormhole_persona_state + + ps = read_wormhole_persona_state() + nid = str(ps.get("root_identity", {}).get("node_id", "") or "").strip() + _root_node_id_cache["value"] = nid + _root_node_id_cache["ts"] = now + return nid + except Exception: + return "" + + +@app.post("/api/mesh/vote") +@limiter.limit("30/minute") +async def mesh_vote(request: Request): + """Cast a reputation vote on a node. + + Body: {voter_id, voter_pubkey?, voter_sig?, target_id, vote: 1|-1, gate?: string} + """ + from services.mesh.mesh_reputation import reputation_ledger + + body = await request.json() + voter_id = body.get("voter_id", "") + target_id = body.get("target_id", "") + vote = body.get("vote", 0) + gate = body.get("gate", "") + public_key = body.get("voter_pubkey", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("voter_sig", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + + if not voter_id or not target_id: + return {"ok": False, "detail": "Missing voter_id or target_id"} + if vote not in (1, -1): + return {"ok": False, "detail": "Vote must be 1 or -1"} + + gate_ok, gate_detail = _validate_gate_vote_context(voter_id, gate) + if not gate_ok: + return {"ok": False, "detail": gate_detail} + gate = gate_detail or "" + + vote_payload = {"target_id": target_id, "vote": vote, "gate": gate} + sig_ok, sig_reason = _verify_signed_event( + event_type="vote", + node_id=voter_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=vote_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="vote", + node_id=voter_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + # Resolve stable local operator ID for duplicate-vote prevention. + # Personas generate unique keypairs, so voter_id alone is insufficient — + # use the root identity's node_id as a stable anchor so switching personas + # doesn't let the same operator vote multiple times on the same post. + stable_voter_id = voter_id + try: + root_nid = _cached_root_node_id() + if root_nid: + stable_voter_id = root_nid + except Exception: + pass + + # Register node if not known + reputation_ledger.register_node(voter_id, public_key, public_key_algo) + + ok, reason, vote_weight = reputation_ledger.cast_vote(stable_voter_id, target_id, vote, gate) + + # Record on Infonet + if ok: + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("vote", vote_payload) + infonet.append( + event_type="vote", + node_id=voter_id, + payload=normalized_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + pass + + return {"ok": ok, "detail": reason, "weight": round(vote_weight, 2)} + + +@app.post("/api/mesh/report") +@limiter.limit("10/minute") +async def mesh_report(request: Request): + """Report abusive or fraudulent behavior (signed, public, non-anonymous).""" + body = await request.json() + reporter_id = body.get("reporter_id", "") + target_id = body.get("target_id", "") + reason = body.get("reason", "") + gate = body.get("gate", "") + evidence = body.get("evidence", "") + public_key = body.get("public_key", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("signature", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + + if not reporter_id or not target_id or not reason: + return {"ok": False, "detail": "Missing reporter_id, target_id, or reason"} + + report_payload = {"target_id": target_id, "reason": reason, "gate": gate, "evidence": evidence} + sig_ok, sig_reason = _verify_signed_event( + event_type="abuse_report", + node_id=reporter_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=report_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="abuse_report", + node_id=reporter_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(reporter_id, public_key, public_key_algo) + except Exception: + pass + + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("abuse_report", report_payload) + infonet.append( + event_type="abuse_report", + node_id=reporter_id, + payload=normalized_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + logger.exception("failed to record abuse report on infonet") + return {"ok": False, "detail": "report_record_failed"} + + return {"ok": True, "detail": "Report recorded"} + + +@app.get("/api/mesh/reputation") +@limiter.limit("60/minute") +async def mesh_reputation(request: Request, node_id: str = ""): + """Get reputation for a single node. + + Public callers receive a summary-only view; authenticated audit callers may + access the richer breakdown. + """ + from services.mesh.mesh_reputation import reputation_ledger + + if not node_id: + return {"ok": False, "detail": "Provide ?node_id=xxx"} + return reputation_ledger.get_reputation_log( + node_id, + detailed=_scoped_view_authenticated(request, "mesh.audit"), + ) + + +@app.get("/api/mesh/reputation/batch") +@limiter.limit("60/minute") +async def mesh_reputation_batch(request: Request, node_id: list[str] = Query(default=[])): + """Get overall public reputation for multiple public node IDs.""" + from services.mesh.mesh_reputation import reputation_ledger + + normalized: list[str] = [] + seen: set[str] = set() + for raw in list(node_id or []): + candidate = str(raw or "").strip() + if not candidate or candidate in seen: + continue + seen.add(candidate) + normalized.append(candidate) + if len(normalized) >= 100: + break + if not normalized: + return {"ok": False, "detail": "Provide at least one node_id", "reputations": {}} + return { + "ok": True, + "reputations": { + candidate: reputation_ledger.get_reputation(candidate).get("overall", 0) or 0 + for candidate in normalized + }, + } + + +@app.get("/api/mesh/reputation/all", dependencies=[Depends(require_admin)]) +@limiter.limit("30/minute") +async def mesh_reputation_all(request: Request): + """Get all known node reputations.""" + from services.mesh.mesh_reputation import reputation_ledger + + return {"reputations": reputation_ledger.get_all_reputations()} + + +@app.post("/api/mesh/identity/rotate") +@limiter.limit("5/minute") +async def mesh_identity_rotate(request: Request): + """Link a new node_id to an old one via dual-signature rotation.""" + body = await request.json() + old_node_id = body.get("old_node_id", "").strip() + old_public_key = body.get("old_public_key", "").strip() + old_public_key_algo = body.get("old_public_key_algo", "").strip() + old_signature = body.get("old_signature", "").strip() + new_node_id = body.get("new_node_id", "").strip() + new_public_key = body.get("new_public_key", "").strip() + new_public_key_algo = body.get("new_public_key_algo", "").strip() + new_signature = body.get("new_signature", "").strip() + timestamp = _safe_int(body.get("timestamp", 0) or 0) + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + + if not ( + old_node_id + and old_public_key + and old_public_key_algo + and old_signature + and new_node_id + and new_public_key + and new_public_key_algo + and new_signature + and timestamp + ): + return {"ok": False, "detail": "Missing rotation fields"} + if old_node_id == new_node_id: + return {"ok": False, "detail": "old_node_id must differ from new_node_id"} + if abs(timestamp - int(time.time())) > 7 * 86400: + return {"ok": False, "detail": "Rotation timestamp is too far from current time"} + + rotation_payload = { + "old_node_id": old_node_id, + "old_public_key": old_public_key, + "old_public_key_algo": old_public_key_algo, + "new_public_key": new_public_key, + "new_public_key_algo": new_public_key_algo, + "timestamp": timestamp, + "old_signature": old_signature, + } + sig_ok, sig_reason = _verify_signed_event( + event_type="key_rotate", + node_id=new_node_id, + sequence=sequence, + public_key=new_public_key, + public_key_algo=new_public_key_algo, + signature=new_signature, + payload=rotation_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="key_rotate", + node_id=new_node_id, + sequence=sequence, + public_key=new_public_key, + public_key_algo=new_public_key_algo, + signature=new_signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + from services.mesh.mesh_crypto import ( + build_signature_payload, + parse_public_key_algo, + verify_signature, + verify_node_binding, + ) + + if not verify_node_binding(old_node_id, old_public_key): + return {"ok": False, "detail": "old_node_id does not match old public key"} + + old_algo = parse_public_key_algo(old_public_key_algo) + if not old_algo: + return {"ok": False, "detail": "Unsupported old_public_key_algo"} + + claim_payload = { + "old_node_id": old_node_id, + "old_public_key": old_public_key, + "old_public_key_algo": old_public_key_algo, + "new_public_key": new_public_key, + "new_public_key_algo": new_public_key_algo, + "timestamp": timestamp, + } + old_sig_payload = build_signature_payload( + event_type="key_rotate", + node_id=old_node_id, + sequence=0, + payload=claim_payload, + ) + if not verify_signature( + public_key_b64=old_public_key, + public_key_algo=old_algo, + signature_hex=old_signature, + payload=old_sig_payload, + ): + return {"ok": False, "detail": "Invalid old_signature"} + + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(new_node_id, new_public_key, new_public_key_algo) + ok, reason = reputation_ledger.link_identities(old_node_id, new_node_id) + if not ok: + return {"ok": False, "detail": reason} + + # Record on Infonet + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("key_rotate", rotation_payload) + infonet.append( + event_type="key_rotate", + node_id=new_node_id, + payload=normalized_payload, + signature=new_signature, + sequence=sequence, + public_key=new_public_key, + public_key_algo=new_public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + pass + + return {"ok": True, "detail": "Identity linked"} + + +@app.post("/api/mesh/identity/revoke") +@limiter.limit("5/minute") +async def mesh_identity_revoke(request: Request): + """Revoke a node's key with a grace window.""" + body = await request.json() + node_id = body.get("node_id", "").strip() + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + revoked_at = _safe_int(body.get("revoked_at", 0) or 0) + grace_until = _safe_int(body.get("grace_until", 0) or 0) + reason = body.get("reason", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + + if not (node_id and public_key and public_key_algo and signature and revoked_at and grace_until): + return {"ok": False, "detail": "Missing revocation fields"} + + now = int(time.time()) + max_grace = 7 * 86400 + if grace_until < revoked_at: + return {"ok": False, "detail": "grace_until must be >= revoked_at"} + if grace_until - revoked_at > max_grace: + return {"ok": False, "detail": "Grace window too large (max 7 days)"} + if abs(revoked_at - now) > max_grace: + return {"ok": False, "detail": "revoked_at is too far from current time"} + + payload = { + "revoked_public_key": public_key, + "revoked_public_key_algo": public_key_algo, + "revoked_at": revoked_at, + "grace_until": grace_until, + "reason": reason, + } + sig_ok, sig_reason = _verify_signed_event( + event_type="key_revoke", + node_id=node_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + if payload["revoked_public_key"] != public_key: + return {"ok": False, "detail": "revoked_public_key must match public_key"} + if payload["revoked_public_key_algo"] != public_key_algo: + return {"ok": False, "detail": "revoked_public_key_algo must match public_key_algo"} + + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("key_revoke", payload) + infonet.append( + event_type="key_revoke", + node_id=node_id, + payload=normalized_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + logger.exception("failed to record key revocation on infonet") + return {"ok": False, "detail": "revocation_record_failed"} + + return {"ok": True, "detail": "Identity revoked"} + + +# ─── Gate Endpoints ─────────────────────────────────────────────────────── + + +@app.post("/api/mesh/gate/create") +@limiter.limit("5/hour") +async def gate_create(request: Request): + """Create a new reputation-gated community. + + Body: {creator_id, creator_pubkey?, creator_sig?, gate_id, display_name, rules?: {min_overall_rep, min_gate_rep}} + """ + from services.mesh.mesh_reputation import ( + ALLOW_DYNAMIC_GATES, + reputation_ledger, + gate_manager, + ) + + if not ALLOW_DYNAMIC_GATES: + return {"ok": False, "detail": "Gate creation is disabled for the fixed private launch catalog"} + + body = await request.json() + creator_id = body.get("creator_id", "") + gate_id = body.get("gate_id", "") + display_name = body.get("display_name", gate_id) + rules = body.get("rules", {}) + public_key = body.get("creator_pubkey", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("creator_sig", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + + if not creator_id or not gate_id: + return {"ok": False, "detail": "Missing creator_id or gate_id"} + + gate_payload = {"gate_id": gate_id, "display_name": display_name, "rules": rules} + sig_ok, sig_reason = _verify_signed_event( + event_type="gate_create", + node_id=creator_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=gate_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="gate_create", + node_id=creator_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + reputation_ledger.register_node(creator_id, public_key, public_key_algo) + + ok, reason = gate_manager.create_gate( + creator_id, + gate_id, + display_name, + min_overall_rep=rules.get("min_overall_rep", 0), + min_gate_rep=rules.get("min_gate_rep"), + ) + + # Record on Infonet + if ok: + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("gate_create", gate_payload) + infonet.append( + event_type="gate_create", + node_id=creator_id, + payload=normalized_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + pass + + return {"ok": ok, "detail": reason} + + +@app.get("/api/mesh/gate/list") +@limiter.limit("30/minute") +async def gate_list(request: Request): + """List all known gates.""" + from services.mesh.mesh_reputation import gate_manager + + return {"gates": gate_manager.list_gates()} + + +@app.get("/api/mesh/gate/{gate_id}") +@limiter.limit("30/minute") +async def gate_detail(request: Request, gate_id: str): + """Get gate details including ratification status.""" + from services.mesh.mesh_reputation import gate_manager + + gate = gate_manager.get_gate(gate_id) + if not gate: + return {"ok": False, "detail": f"Gate '{gate_id}' not found"} + gate["ratification"] = gate_manager.get_ratification_status(gate_id) + return gate + + +@app.post("/api/mesh/gate/{gate_id}/message") +@limiter.limit("10/minute") +async def gate_message(request: Request, gate_id: str): + """Post a message to a gate. Checks entry rules against sender's reputation. + + Body: {sender_id, ciphertext, nonce, sender_ref, signature?} + """ + body = await request.json() + return _submit_gate_message_envelope(request, gate_id, body) + + +def _submit_gate_message_envelope(request: Request, gate_id: str, body: dict[str, Any]) -> dict[str, Any]: + """Validate and record an encrypted gate envelope on the private plane.""" + from services.mesh.mesh_reputation import reputation_ledger, gate_manager + sender_id = body.get("sender_id", "") + epoch = _safe_int(body.get("epoch", 0) or 0) + ciphertext = str(body.get("ciphertext", "")) + nonce = str(body.get("nonce", body.get("iv", ""))) + sender_ref = str(body.get("sender_ref", "")) + payload_format = str(body.get("format", "mls1") or "mls1") + public_key = body.get("public_key", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("signature", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + + if not sender_id: + return {"ok": False, "detail": "Missing sender_id"} + if "message" in body and str(body.get("message", "")).strip(): + return { + "ok": False, + "detail": "Plaintext gate messages are no longer accepted. Submit an encrypted gate envelope.", + } + + gate_envelope = str(body.get("gate_envelope", "") or "").strip() + reply_to = str(body.get("reply_to", "") or "").strip() + + gate_payload_input = { + "gate": gate_id, + "ciphertext": ciphertext, + "nonce": nonce, + "sender_ref": sender_ref, + "format": payload_format, + } + if epoch > 0: + gate_payload_input["epoch"] = epoch + gate_payload = normalize_payload("gate_message", gate_payload_input) + # Validate BEFORE adding gate_envelope (which is not a normalized field). + payload_ok, payload_reason = validate_event_payload("gate_message", gate_payload) + if not payload_ok: + return {"ok": False, "detail": payload_reason} + # gate_envelope and reply_to are NOT part of the signed payload — add after validation. + if gate_envelope: + gate_payload["gate_envelope"] = gate_envelope + if reply_to: + gate_payload["reply_to"] = reply_to + # Signature verification payload must exclude epoch, gate_envelope, and reply_to + # because compose_encrypted_gate_message signs without them. + signature_gate_payload = normalize_payload( + "gate_message", + { + "gate": gate_id, + "ciphertext": ciphertext, + "nonce": nonce, + "sender_ref": sender_ref, + "format": payload_format, + }, + ) + + sig_ok, sig_reason = _verify_signed_event( + event_type="gate_message", + node_id=sender_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=signature_gate_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="gate_message", + node_id=sender_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + reputation_ledger.register_node(sender_id, public_key, public_key_algo) + + # Check gate access + can_enter, reason = gate_manager.can_enter(sender_id, gate_id) + if not can_enter: + return {"ok": False, "detail": f"Gate access denied: {reason}"} + + cooldown_ok, cooldown_reason = _check_gate_post_cooldown(sender_id, gate_id) + if not cooldown_ok: + return {"ok": False, "detail": cooldown_reason} + + # Record on hashchain (encrypted — only gate members can decrypt). + # NOTE: infonet.append() validates and advances the sequence counter + # internally, so we must NOT call validate_and_set_sequence() beforehand + # — doing so would pre-advance the counter and cause append() to reject + # the event as a replay, silently dropping the message. + # + # The chain payload must match the signed payload exactly. The message + # was signed WITHOUT the `epoch` field (compose_encrypted_gate_message + # excludes it from the signing payload), so we must strip it here too — + # otherwise infonet.append() re-verifies the signature against a payload + # that includes epoch and gets a mismatch → "invalid signature". + chain_payload = {k: v for k, v in gate_payload.items() if k != "epoch"} + chain_event_id = "" + try: + from services.mesh.mesh_hashchain import infonet, gate_store + + chain_result = infonet.append( + event_type="gate_message", + node_id=sender_id, + payload=chain_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version or PROTOCOL_VERSION, + ) + chain_event_id = str(chain_result.get("event_id", "") or "") + except ValueError as exc: + # Sequence replay, signature failure, payload validation, etc. + return {"ok": False, "detail": str(exc)} + except Exception: + logger.exception("Failed to record gate message on chain") + return {"ok": False, "detail": "Failed to record gate message"} + + gate_manager.record_message(gate_id) + _record_gate_post_cooldown(sender_id, gate_id) + logger.info("Encrypted gate message accepted on obfuscated gate plane") + + # Store in gate_store for fast local read/decrypt (separate try so a + # gate_store hiccup doesn't discard the already-committed chain event). + try: + from services.mesh.mesh_hashchain import gate_store + + import copy + + gate_event = copy.deepcopy(chain_result) + gate_event["event_type"] = "gate_message" + # Restore gate_envelope / reply_to that normalize_payload stripped + # from the chain copy — these are needed for local decryption. + # CRITICAL: we deep-copied so we don't mutate the chain's event dict + # — adding gate_envelope to the chain payload would corrupt the hash. + store_payload = gate_event.get("payload") + if isinstance(store_payload, dict): + if gate_envelope: + store_payload["gate_envelope"] = gate_envelope + if reply_to: + store_payload["reply_to"] = reply_to + stored_event = gate_store.append(gate_id, gate_event) + chain_event_id = chain_event_id or str(stored_event.get("event_id", "")) + try: + from services.mesh.mesh_rns import rns_bridge + + rns_bridge.publish_gate_event(gate_id, gate_event) + except Exception: + pass + except Exception: + logger.exception("Failed to store gate message in gate_store") + + return { + "ok": True, + "detail": f"Message posted to gate '{gate_id}'", + "gate_id": gate_id, + "event_id": chain_event_id, + } + + +# ─── Infonet Endpoints ─────────────────────────────────────────────────── + + +@app.get("/api/mesh/infonet/status") +@limiter.limit("30/minute") +async def infonet_status(request: Request, verify_signatures: bool = False): + """Get Infonet metadata — event counts, head hash, chain size.""" + from services.mesh.mesh_hashchain import infonet + from services.wormhole_supervisor import get_wormhole_state + + info = infonet.get_info() + valid, reason = infonet.validate_chain(verify_signatures=verify_signatures) + try: + wormhole = get_wormhole_state() + except Exception: + wormhole = {"configured": False, "ready": False, "rns_ready": False} + info["valid"] = valid + info["validation"] = reason + info["verify_signatures"] = verify_signatures + info["private_lane_tier"] = _current_private_lane_tier(wormhole) + info["private_lane_policy"] = _private_infonet_policy_snapshot() + info.update(_node_runtime_snapshot()) + return _redact_private_lane_control_fields( + info, + authenticated=_scoped_view_authenticated(request, "mesh.audit"), + ) + + +@app.get("/api/mesh/infonet/merkle") +@limiter.limit("30/minute") +async def infonet_merkle(request: Request): + """Merkle root for sync comparison.""" + from services.mesh.mesh_hashchain import infonet + + return { + "merkle_root": infonet.get_merkle_root(), + "head_hash": infonet.head_hash, + "count": len(infonet.events), + "network_id": infonet.get_info().get("network_id"), + } + + +@app.get("/api/mesh/infonet/locator") +@limiter.limit("30/minute") +async def infonet_locator(request: Request, limit: int = Query(32, ge=4, le=128)): + """Block locator for fork-aware sync.""" + from services.mesh.mesh_hashchain import infonet + + locator = infonet.get_locator(max_entries=limit) + return { + "locator": locator, + "head_hash": infonet.head_hash, + "count": len(infonet.events), + "network_id": infonet.get_info().get("network_id"), + } + + +@app.post("/api/mesh/infonet/sync") +@limiter.limit("30/minute") +async def infonet_sync_post( + request: Request, + limit: int = Query(100, ge=1, le=500), +): + """Fork-aware sync using a block locator.""" + from services.mesh.mesh_hashchain import infonet, GENESIS_HASH + + body = await request.json() + req_proto = str(body.get("protocol_version", "") or "") + if req_proto and req_proto != PROTOCOL_VERSION: + return Response( + content=json_mod.dumps( + { + "ok": False, + "detail": "Unsupported protocol_version", + "protocol_version": PROTOCOL_VERSION, + } + ), + status_code=426, + media_type="application/json", + ) + locator = body.get("locator", []) + if not isinstance(locator, list): + return {"ok": False, "detail": "locator must be a list"} + expected_head = str(body.get("expected_head", "") or "") + if expected_head and expected_head != infonet.head_hash: + return Response( + content=json_mod.dumps( + { + "ok": False, + "detail": "head_hash mismatch", + "head_hash": infonet.head_hash, + "expected_head": expected_head, + } + ), + status_code=409, + media_type="application/json", + ) + if "limit" in body: + try: + limit = max(1, min(500, _safe_int(body["limit"], 0))) + except Exception: + pass + + matched_hash, start_index, events = infonet.get_events_after_locator(locator, limit=limit) + forked = False + if not matched_hash: + forked = True + elif matched_hash == GENESIS_HASH and len(locator) > 1: + forked = True + + # Gate messages pass through as encrypted blobs — no redaction needed for ciphertext. + # Non-gate events get standard public redaction. + events = [e if e.get("event_type") == "gate_message" else _redact_public_event(e) for e in events] + + response = { + "events": events, + "matched_hash": matched_hash, + "forked": forked, + "head_hash": infonet.head_hash, + "count": len(events), + "protocol_version": PROTOCOL_VERSION, + } + if body.get("include_proofs"): + proofs = infonet.get_merkle_proofs(start_index, len(events)) if start_index >= 0 else {} + response.update( + { + "merkle_root": proofs.get("root", infonet.get_merkle_root()), + "merkle_total": proofs.get("total", len(infonet.events)), + "merkle_start": proofs.get("start", 0), + "merkle_proofs": proofs.get("proofs", []), + } + ) + return response + + +@app.get("/api/mesh/metrics") +@limiter.limit("30/minute") +async def mesh_metrics(request: Request): + """Mesh protocol health counters.""" + from services.mesh.mesh_metrics import snapshot + + ok, detail = _check_scoped_auth(request, "mesh.audit") + if not ok: + if detail == "insufficient scope": + raise HTTPException(status_code=403, detail="Forbidden — insufficient scope") + raise HTTPException(status_code=403, detail=detail) + return snapshot() + + +@app.get("/api/mesh/rns/status") +@limiter.limit("30/minute") +async def mesh_rns_status(request: Request): + from services.wormhole_supervisor import get_wormhole_state + + try: + from services.mesh.mesh_rns import rns_bridge + + status = await asyncio.to_thread(rns_bridge.status) + except Exception: + status = {"enabled": False, "ready": False, "configured_peers": 0, "active_peers": 0} + try: + wormhole = get_wormhole_state() + except Exception: + wormhole = {"configured": False, "ready": False, "rns_ready": False} + status["private_lane_tier"] = _current_private_lane_tier(wormhole) + status["private_lane_policy"] = _private_infonet_policy_snapshot() + return _redact_public_rns_status( + status, + authenticated=_scoped_view_authenticated(request, "mesh.audit"), + ) + + +@app.get("/api/mesh/infonet/sync") +@limiter.limit("30/minute") +async def infonet_sync( + request: Request, + after_hash: str = "", + limit: int = Query(100, ge=1, le=500), + expected_head: str = "", + protocol_version: str = "", +): + """Return events after a given hash (delta sync).""" + from services.mesh.mesh_hashchain import infonet, GENESIS_HASH + + if protocol_version and protocol_version != PROTOCOL_VERSION: + return Response( + content=json_mod.dumps( + { + "ok": False, + "detail": "Unsupported protocol_version", + "protocol_version": PROTOCOL_VERSION, + } + ), + status_code=426, + media_type="application/json", + ) + if expected_head and expected_head != infonet.head_hash: + return Response( + content=json_mod.dumps( + { + "ok": False, + "detail": "head_hash mismatch", + "head_hash": infonet.head_hash, + "expected_head": expected_head, + } + ), + status_code=409, + media_type="application/json", + ) + base = after_hash or GENESIS_HASH + events = infonet.get_events_after(base, limit=limit) + events = [e if e.get("event_type") == "gate_message" else _redact_public_event(e) for e in events] + return { + "events": events, + "after_hash": base, + "count": len(events), + "protocol_version": PROTOCOL_VERSION, + } + + +@app.post("/api/mesh/infonet/ingest", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def infonet_ingest(request: Request): + """Ingest externally sourced Infonet events (strict verification).""" + from services.mesh.mesh_hashchain import infonet + + body = await request.json() + events = body.get("events", []) + expected_head = str(body.get("expected_head", "") or "") + if expected_head and expected_head != infonet.head_hash: + return Response( + content=json_mod.dumps( + { + "ok": False, + "detail": "head_hash mismatch", + "head_hash": infonet.head_hash, + "expected_head": expected_head, + } + ), + status_code=409, + media_type="application/json", + ) + if not isinstance(events, list): + return {"ok": False, "detail": "events must be a list"} + if len(events) > 200: + return {"ok": False, "detail": "Too many events in one ingest batch"} + + result = infonet.ingest_events(events) + _hydrate_gate_store_from_chain(events) + return {"ok": True, **result} + + +@app.post("/api/mesh/infonet/peer-push") +@limiter.limit("30/minute") +async def infonet_peer_push(request: Request): + """Accept pushed Infonet events from relay peers (HMAC-authenticated).""" + content_length = request.headers.get("content-length") + if content_length: + try: + if int(content_length) > 524_288: + return Response( + content='{"ok":false,"detail":"Request body too large (max 512KB)"}', + status_code=413, + media_type="application/json", + ) + except (ValueError, TypeError): + pass + from services.mesh.mesh_hashchain import infonet + + body_bytes = await request.body() + if not _verify_peer_push_hmac(request, body_bytes): + return Response( + content='{"ok":false,"detail":"Invalid or missing peer HMAC"}', + status_code=403, + media_type="application/json", + ) + + body = json_mod.loads(body_bytes or b"{}") + events = body.get("events", []) + if not isinstance(events, list): + return {"ok": False, "detail": "events must be a list"} + if len(events) > 50: + return {"ok": False, "detail": "Too many events in one push (max 50)"} + if not events: + return {"ok": True, "accepted": 0, "duplicates": 0, "rejected": []} + + result = infonet.ingest_events(events) + _hydrate_gate_store_from_chain(events) + return {"ok": True, **result} + + +@app.post("/api/mesh/gate/peer-push") +@limiter.limit("30/minute") +async def gate_peer_push(request: Request): + """Accept pushed gate events from relay peers (private plane).""" + content_length = request.headers.get("content-length") + if content_length: + try: + if int(content_length) > 524_288: + return Response( + content='{"ok":false,"detail":"Request body too large"}', + status_code=413, + media_type="application/json", + ) + except (ValueError, TypeError): + pass + + from services.mesh.mesh_hashchain import gate_store + + body_bytes = await request.body() + if not _verify_peer_push_hmac(request, body_bytes): + return Response( + content='{"ok":false,"detail":"Invalid or missing peer HMAC"}', + status_code=403, + media_type="application/json", + ) + + body = json_mod.loads(body_bytes or b"{}") + events = body.get("events", []) + if not isinstance(events, list): + return {"ok": False, "detail": "events must be a list"} + if len(events) > 50: + return {"ok": False, "detail": "Too many events (max 50)"} + if not events: + return {"ok": True, "accepted": 0, "duplicates": 0} + + from services.mesh.mesh_hashchain import resolve_gate_wire_ref + + grouped_events: dict[str, list[dict[str, Any]]] = {} + for evt in events: + evt_dict = evt if isinstance(evt, dict) else {} + payload = evt_dict.get("payload") + if not isinstance(payload, dict): + payload = {} + clean_event = { + "event_id": str(evt_dict.get("event_id", "") or ""), + "event_type": "gate_message", + "timestamp": evt_dict.get("timestamp", 0), + "node_id": str(evt_dict.get("node_id", "") or evt_dict.get("sender_id", "") or ""), + "sequence": evt_dict.get("sequence", 0), + "signature": str(evt_dict.get("signature", "") or ""), + "public_key": str(evt_dict.get("public_key", "") or ""), + "public_key_algo": str(evt_dict.get("public_key_algo", "") or ""), + "protocol_version": str(evt_dict.get("protocol_version", "") or ""), + "payload": { + "ciphertext": str(payload.get("ciphertext", "") or ""), + "format": str(payload.get("format", "") or ""), + "nonce": str(payload.get("nonce", "") or ""), + "sender_ref": str(payload.get("sender_ref", "") or ""), + }, + } + epoch = _safe_int(payload.get("epoch", 0) or 0) + if epoch > 0: + clean_event["payload"]["epoch"] = epoch + event_gate_id = str(payload.get("gate", "") or evt_dict.get("gate", "") or "").strip().lower() + if not event_gate_id: + event_gate_id = resolve_gate_wire_ref( + str(payload.get("gate_ref", "") or evt_dict.get("gate_ref", "") or ""), + clean_event, + ) + if not event_gate_id: + return {"ok": False, "detail": "gate resolution failed"} + grouped_events.setdefault(event_gate_id, []).append( + { + "event_id": clean_event["event_id"], + "event_type": "gate_message", + "timestamp": clean_event["timestamp"], + "node_id": clean_event["node_id"], + "sequence": clean_event["sequence"], + "signature": clean_event["signature"], + "public_key": clean_event["public_key"], + "public_key_algo": clean_event["public_key_algo"], + "protocol_version": clean_event["protocol_version"], + "payload": { + "gate": event_gate_id, + "ciphertext": clean_event["payload"]["ciphertext"], + "format": clean_event["payload"]["format"], + "nonce": clean_event["payload"]["nonce"], + "sender_ref": clean_event["payload"]["sender_ref"], + }, + } + ) + if epoch > 0: + grouped_events[event_gate_id][-1]["payload"]["epoch"] = epoch + + accepted = 0 + duplicates = 0 + rejected = 0 + for event_gate_id, items in grouped_events.items(): + result = gate_store.ingest_peer_events(event_gate_id, items) + accepted += int(result.get("accepted", 0) or 0) + duplicates += int(result.get("duplicates", 0) or 0) + rejected += int(result.get("rejected", 0) or 0) + return {"ok": True, "accepted": accepted, "duplicates": duplicates, "rejected": rejected} + + +# --------------------------------------------------------------------------- +# Peer Management API — operator endpoints for adding / removing / listing +# peers without editing peer_store.json by hand. +# --------------------------------------------------------------------------- + + +@app.get("/api/mesh/peers", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def list_peers(request: Request, bucket: str = Query(None)): + """List all peers (or filter by bucket: sync, push, bootstrap).""" + from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore + + store = PeerStore(DEFAULT_PEER_STORE_PATH) + try: + store.load() + except Exception as exc: + return {"ok": False, "detail": f"Failed to load peer store: {exc}"} + + if bucket: + records = store.records_for_bucket(bucket) + else: + records = store.records() + + return { + "ok": True, + "count": len(records), + "peers": [r.to_dict() for r in records], + } + + +@app.post("/api/mesh/peers", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def add_peer(request: Request): + """Add a peer to the store. Body: {peer_url, transport?, label?, role?, buckets?[]}.""" + from services.mesh.mesh_crypto import normalize_peer_url + from services.mesh.mesh_peer_store import ( + DEFAULT_PEER_STORE_PATH, + PeerStore, + PeerStoreError, + make_push_peer_record, + make_sync_peer_record, + ) + from services.mesh.mesh_router import peer_transport_kind + + body = await request.json() + peer_url_raw = str(body.get("peer_url", "") or "").strip() + if not peer_url_raw: + return {"ok": False, "detail": "peer_url is required"} + + peer_url = normalize_peer_url(peer_url_raw) + if not peer_url: + return {"ok": False, "detail": "Invalid peer_url"} + + transport = str(body.get("transport", "") or "").strip().lower() + if not transport: + transport = peer_transport_kind(peer_url) + if not transport: + return {"ok": False, "detail": "Cannot determine transport for peer_url — provide transport explicitly"} + + label = str(body.get("label", "") or "").strip() + role = str(body.get("role", "") or "").strip().lower() or "relay" + buckets = body.get("buckets", ["sync", "push"]) + if isinstance(buckets, str): + buckets = [buckets] + if not isinstance(buckets, list): + buckets = ["sync", "push"] + + store = PeerStore(DEFAULT_PEER_STORE_PATH) + try: + store.load() + except Exception: + store = PeerStore(DEFAULT_PEER_STORE_PATH) + + added: list[str] = [] + try: + for b in buckets: + b = str(b).strip().lower() + if b == "sync": + store.upsert(make_sync_peer_record(peer_url=peer_url, transport=transport, role=role, label=label)) + added.append("sync") + elif b == "push": + store.upsert(make_push_peer_record(peer_url=peer_url, transport=transport, role=role, label=label)) + added.append("push") + store.save() + except PeerStoreError as exc: + return {"ok": False, "detail": str(exc)} + + return {"ok": True, "peer_url": peer_url, "buckets": added} + + +@app.delete("/api/mesh/peers", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def remove_peer(request: Request): + """Remove a peer. Body: {peer_url, bucket?}. If bucket omitted, removes from all buckets.""" + from services.mesh.mesh_crypto import normalize_peer_url + from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore + + body = await request.json() + peer_url_raw = str(body.get("peer_url", "") or "").strip() + if not peer_url_raw: + return {"ok": False, "detail": "peer_url is required"} + + peer_url = normalize_peer_url(peer_url_raw) + if not peer_url: + return {"ok": False, "detail": "Invalid peer_url"} + + bucket_filter = str(body.get("bucket", "") or "").strip().lower() + + store = PeerStore(DEFAULT_PEER_STORE_PATH) + try: + store.load() + except Exception: + return {"ok": False, "detail": "Failed to load peer store"} + + removed: list[str] = [] + for b in ["bootstrap", "sync", "push"]: + if bucket_filter and b != bucket_filter: + continue + key = f"{b}:{peer_url}" + if key in store._records: + del store._records[key] + removed.append(b) + + if not removed: + return {"ok": False, "detail": "Peer not found in any bucket"} + + store.save() + return {"ok": True, "peer_url": peer_url, "removed_from": removed} + + +@app.patch("/api/mesh/peers", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def toggle_peer(request: Request): + """Enable or disable a peer. Body: {peer_url, bucket, enabled: bool}.""" + from services.mesh.mesh_crypto import normalize_peer_url + from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerRecord, PeerStore + + body = await request.json() + peer_url_raw = str(body.get("peer_url", "") or "").strip() + bucket = str(body.get("bucket", "") or "").strip().lower() + enabled = body.get("enabled") + + if not peer_url_raw: + return {"ok": False, "detail": "peer_url is required"} + if not bucket: + return {"ok": False, "detail": "bucket is required"} + if enabled is None: + return {"ok": False, "detail": "enabled (true/false) is required"} + + peer_url = normalize_peer_url(peer_url_raw) + if not peer_url: + return {"ok": False, "detail": "Invalid peer_url"} + + store = PeerStore(DEFAULT_PEER_STORE_PATH) + try: + store.load() + except Exception: + return {"ok": False, "detail": "Failed to load peer store"} + + key = f"{bucket}:{peer_url}" + record = store._records.get(key) + if not record: + return {"ok": False, "detail": f"Peer not found in {bucket} bucket"} + + updated = PeerRecord(**{**record.to_dict(), "enabled": bool(enabled), "updated_at": int(time.time())}) + store._records[key] = updated + store.save() + + return {"ok": True, "peer_url": peer_url, "bucket": bucket, "enabled": bool(enabled)} + + +@app.get("/api/mesh/gate/{gate_id}/messages") +@limiter.limit("60/minute") +async def gate_messages( + request: Request, + gate_id: str, + limit: int = Query(20, ge=1, le=100), + offset: int = Query(0, ge=0), +): + """Get encrypted gate messages from private store (newest first). Requires gate membership.""" + if not _verify_gate_access(request, gate_id): + return Response( + content='{"ok":false,"detail":"Gate membership required"}', + status_code=403, + media_type="application/json", + ) + from services.mesh.mesh_hashchain import gate_store + from services.mesh.mesh_reputation import gate_manager + + safe_messages = [_strip_gate_identity(m) for m in gate_store.get_messages(gate_id, limit=limit, offset=offset)] + if gate_id and not safe_messages: + gate_meta = gate_manager.get_gate(gate_id) + if gate_meta: + welcome_text = str(gate_meta.get("welcome") or gate_meta.get("description") or "").strip() + if welcome_text: + safe_messages = [ + { + "event_id": f"seed_{gate_id}_welcome", + "event_type": "gate_notice", + "node_id": "!sb_gate", + "message": welcome_text, + "gate": gate_id, + "timestamp": int(gate_meta.get("created_at") or time.time()), + "sequence": 0, + "ephemeral": False, + "system_seed": True, + "fixed_gate": bool(gate_meta.get("fixed", False)), + } + ] + return {"messages": safe_messages, "count": len(safe_messages), "gate": gate_id} + + +@app.get("/api/mesh/infonet/messages") +@limiter.limit("60/minute") +async def infonet_messages( + request: Request, + gate: str = "", + limit: int = Query(20, ge=1, le=100), + offset: int = Query(0, ge=0), +): + """Browse messages on the Infonet (newest first). Optional gate filter.""" + from services.mesh.mesh_hashchain import gate_store, infonet + from services.mesh.mesh_reputation import gate_manager + + if gate: + if not _verify_gate_access(request, gate): + return Response( + content='{"ok":false,"detail":"Gate membership required"}', + status_code=403, + media_type="application/json", + ) + messages = [_strip_gate_identity(m) for m in gate_store.get_messages(gate, limit=limit, offset=offset)] + else: + messages = infonet.get_messages(gate_id="", limit=limit, offset=offset) + messages = [m for m in messages if m.get("event_type") != "gate_message"] + messages = [_redact_public_event(m) for m in messages] + if gate and not messages: + gate_meta = gate_manager.get_gate(gate) + if gate_meta: + welcome_text = str(gate_meta.get("welcome") or gate_meta.get("description") or "").strip() + if welcome_text: + messages = [ + { + "event_id": f"seed_{gate}_welcome", + "event_type": "gate_notice", + "node_id": "!sb_gate", + "message": welcome_text, + "gate": gate, + "timestamp": int(gate_meta.get("created_at") or time.time()), + "sequence": 0, + "ephemeral": False, + "system_seed": True, + "fixed_gate": bool(gate_meta.get("fixed", False)), + } + ] + return {"messages": messages, "count": len(messages), "gate": gate or "all"} + + +@app.get("/api/mesh/infonet/event/{event_id}") +@limiter.limit("60/minute") +async def infonet_event(request: Request, event_id: str): + """Look up a single Infonet event by ID.""" + from services.mesh.mesh_hashchain import gate_store, infonet + + evt = infonet.get_event(event_id) + if not evt: + evt = gate_store.get_event(event_id) + if evt: + gate_id = str(evt.get("payload", {}).get("gate", "") or evt.get("gate", "") or "").strip() + if not gate_id or not _verify_gate_access(request, gate_id): + return Response( + content='{"ok":false,"detail":"Gate membership required"}', + status_code=403, + media_type="application/json", + ) + return _strip_gate_identity(evt) + return {"ok": False, "detail": "Event not found"} + if evt.get("event_type") == "gate_message": + gate_id = str(evt.get("payload", {}).get("gate", "") or evt.get("gate", "") or "").strip() + if not gate_id or not _verify_gate_access(request, gate_id): + return Response( + content='{"ok":false,"detail":"Gate membership required"}', + status_code=403, + media_type="application/json", + ) + return _strip_gate_identity(evt) + return _redact_public_event(infonet.decorate_event(evt)) + + +@app.get("/api/mesh/infonet/node/{node_id}") +@limiter.limit("30/minute") +async def infonet_node_events( + request: Request, + node_id: str, + limit: int = Query(20, ge=1, le=100), +): + """Get recent Infonet events by a specific node.""" + from services.mesh.mesh_hashchain import infonet + + events = infonet.get_events_by_node(node_id, limit=limit) + events = [e for e in events if e.get("event_type") != "gate_message"] + events = [_redact_public_event(e) for e in infonet.decorate_events(events)] + events = _redact_public_node_history( + events, + authenticated=_scoped_view_authenticated(request, "mesh.audit"), + ) + return {"events": events, "count": len(events), "node_id": node_id} + + +@app.get("/api/mesh/infonet/events") +@limiter.limit("30/minute") +async def infonet_events_by_type( + request: Request, + event_type: str = "", + limit: int = Query(20, ge=1, le=100), + offset: int = Query(0, ge=0), +): + """Get recent Infonet events, optionally filtered by type.""" + from services.mesh.mesh_hashchain import infonet + + if event_type: + events = infonet.get_events_by_type(event_type, limit=limit, offset=offset) + else: + events = list(reversed(infonet.events)) + events = events[offset : offset + limit] + events = [e for e in events if e.get("event_type") != "gate_message"] + events = [_redact_public_event(e) for e in infonet.decorate_events(events)] + return { + "events": events, + "count": len(events), + "event_type": event_type or "all", + } + + +# ─── Oracle Endpoints ───────────────────────────────────────────────────── + + +@app.post("/api/mesh/oracle/predict") +@limiter.limit("10/minute") +async def oracle_predict(request: Request): + """Place a prediction on a market outcome. FINAL decision. + + Body: {node_id, market_title, side, stake_amount?: number} + - stake_amount = 0 or omitted → FREE PICK (earn rep if correct) + - stake_amount > 0 → STAKE REP (risk rep, split loser pool if correct) + - side can be "yes"/"no" or an outcome name for multi-outcome markets + """ + from services.mesh.mesh_oracle import oracle_ledger + + body = await request.json() + node_id = body.get("node_id", "") + market_title = body.get("market_title", "") + side = body.get("side", "") + stake_amount = _safe_float(body.get("stake_amount", 0)) + public_key = body.get("public_key", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("signature", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + + if not node_id or not market_title or not side: + return {"ok": False, "detail": "Missing node_id, market_title, or side"} + + prediction_payload = { + "market_title": market_title, + "side": side, + "stake_amount": stake_amount, + } + sig_ok, sig_reason = _verify_signed_event( + event_type="prediction", + node_id=node_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=prediction_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="prediction", + node_id=node_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(node_id, public_key, public_key_algo) + except Exception: + pass + + # Get current market probability from live data + data = get_latest_data() + markets = data.get("prediction_markets", []) + matched = None + for m in markets: + if m.get("title", "").lower() == market_title.lower(): + matched = m + break + # Fuzzy fallback — partial match + if not matched: + for m in markets: + if market_title.lower() in m.get("title", "").lower(): + matched = m + break + + if not matched: + return {"ok": False, "detail": f"Market '{market_title}' not found in active markets."} + + # Determine probability for the chosen side + # For binary yes/no, use consensus_pct. For multi-outcome, find the outcome's pct. + probability = 50.0 + side_lower = side.lower() + outcomes = matched.get("outcomes", []) + if outcomes: + # Multi-outcome: find the specific outcome's probability + for o in outcomes: + if o.get("name", "").lower() == side_lower: + probability = float(o.get("pct", 50)) + break + else: + # Binary market + consensus = matched.get("consensus_pct") + if consensus is None: + consensus = matched.get("polymarket_pct") or matched.get("kalshi_pct") or 50 + probability = float(consensus) + if side_lower == "no": + probability = 100.0 - probability + + if stake_amount > 0: + # STAKED prediction — risk rep for bigger reward + ok, detail = oracle_ledger.place_market_stake( + node_id, matched["title"], side, stake_amount, probability + ) + mode = "staked" + else: + # FREE prediction — no rep risked + ok, detail = oracle_ledger.place_prediction(node_id, matched["title"], side, probability) + mode = "free" + + # Record on Infonet + if ok: + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("prediction", prediction_payload) + infonet.append( + event_type="prediction", + node_id=node_id, + payload=normalized_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + pass + + return {"ok": ok, "detail": detail, "probability": probability, "mode": mode} + + +@app.get("/api/mesh/oracle/markets") +@limiter.limit("30/minute") +async def oracle_markets(request: Request): + """List active prediction markets, categorized with top 10 per category. + Includes network consensus data (picks + staked rep per side).""" + from collections import defaultdict + from services.mesh.mesh_oracle import oracle_ledger + + data = get_latest_data() + markets = data.get("prediction_markets", []) + + # Get consensus for all active markets (bulk) + all_consensus = oracle_ledger.get_all_market_consensus() + + by_category = defaultdict(list) + for m in markets: + by_category[m.get("category", "NEWS")].append(m) + + _fields = ( + "title", + "consensus_pct", + "polymarket_pct", + "kalshi_pct", + "volume", + "volume_24h", + "end_date", + "description", + "category", + "sources", + "slug", + "kalshi_ticker", + "outcomes", + ) + categories = {} + cat_totals = {} + for cat in ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"]: + all_cat = sorted( + by_category.get(cat, []), + key=lambda x: x.get("volume", 0) or 0, + reverse=True, + ) + cat_totals[cat] = len(all_cat) + cat_list = [] + for m in all_cat[:10]: + entry = {k: m.get(k) for k in _fields} + entry["consensus"] = all_consensus.get(m.get("title", ""), {}) + cat_list.append(entry) + categories[cat] = cat_list + + return {"categories": categories, "total_count": len(markets), "cat_totals": cat_totals} + + +@app.get("/api/mesh/oracle/search") +@limiter.limit("20/minute") +async def oracle_search(request: Request, q: str = "", limit: int = 20): + """Search prediction markets — queries Polymarket API directly + cached data.""" + if not q or len(q) < 2: + return {"results": [], "query": q, "count": 0} + + from services.fetchers.prediction_markets import search_polymarket_direct + + # 1. Search Polymarket API directly (finds ALL markets, not just cached) + poly_results = search_polymarket_direct(q, limit=limit) + + # 2. Also search cached data (catches Kalshi matches + merged data) + data = get_latest_data() + markets = data.get("prediction_markets", []) + q_lower = q.lower() + cached_matches = [m for m in markets if q_lower in m.get("title", "").lower()] + + # Deduplicate: prefer cached (has both sources) over poly-only + seen_titles = set() + combined = [] + for m in cached_matches: + seen_titles.add(m["title"].lower()) + combined.append(m) + for m in poly_results: + if m["title"].lower() not in seen_titles: + seen_titles.add(m["title"].lower()) + combined.append(m) + + # Sort by volume descending + combined.sort(key=lambda x: x.get("volume", 0) or 0, reverse=True) + + _fields = ( + "title", + "consensus_pct", + "polymarket_pct", + "kalshi_pct", + "volume", + "volume_24h", + "end_date", + "description", + "category", + "sources", + "slug", + "kalshi_ticker", + "outcomes", + ) + results = [{k: m.get(k) for k in _fields} for m in combined[:limit]] + return {"results": results, "query": q, "count": len(results)} + + +@app.get("/api/mesh/oracle/markets/more") +@limiter.limit("30/minute") +async def oracle_markets_more( + request: Request, category: str = "NEWS", offset: int = 0, limit: int = 10 +): + """Load more markets for a specific category (paginated).""" + data = get_latest_data() + markets = data.get("prediction_markets", []) + cat_markets = sorted( + [m for m in markets if m.get("category") == category], + key=lambda x: x.get("volume", 0) or 0, + reverse=True, + ) + + page = cat_markets[offset : offset + limit] + _fields = ( + "title", + "consensus_pct", + "polymarket_pct", + "kalshi_pct", + "volume", + "volume_24h", + "end_date", + "description", + "category", + "sources", + "slug", + "kalshi_ticker", + "outcomes", + ) + results = [{k: m.get(k) for k in _fields} for m in page] + return { + "markets": results, + "category": category, + "offset": offset, + "has_more": offset + limit < len(cat_markets), + "total": len(cat_markets), + } + + +@app.post("/api/mesh/oracle/resolve") +@limiter.limit("5/minute") +async def oracle_resolve(request: Request): + """Resolve a prediction market (admin/agent action). + + Body: {market_title, outcome: "yes"|"no" or any outcome name} + """ + from services.mesh.mesh_oracle import oracle_ledger + + body = await request.json() + market_title = body.get("market_title", "") + outcome = body.get("outcome", "") + + if not market_title or not outcome: + return {"ok": False, "detail": "Need market_title and outcome"} + + # Resolve free predictions + winners, losers = oracle_ledger.resolve_market(market_title, outcome) + # Resolve market stakes + stake_result = oracle_ledger.resolve_market_stakes(market_title, outcome) + + return { + "ok": True, + "detail": f"Resolved: {winners} free winners, {losers} free losers, " + f"{stake_result.get('winners', 0)} stake winners, {stake_result.get('losers', 0)} stake losers", + "free": {"winners": winners, "losers": losers}, + "stakes": stake_result, + } + + +@app.get("/api/mesh/oracle/consensus") +@limiter.limit("30/minute") +async def oracle_consensus(request: Request, market_title: str = ""): + """Get network consensus for a market — picks + staked rep per side.""" + from services.mesh.mesh_oracle import oracle_ledger + + if not market_title: + return {"error": "market_title required"} + return oracle_ledger.get_market_consensus(market_title) + + +@app.post("/api/mesh/oracle/stake") +@limiter.limit("10/minute") +async def oracle_stake(request: Request): + """Stake oracle rep on a post's truthfulness. + + Body: {staker_id, message_id, poster_id, side: "truth"|"false", amount, duration_days: 1-7} + """ + from services.mesh.mesh_oracle import oracle_ledger + + body = await request.json() + staker_id = body.get("staker_id", "") + message_id = body.get("message_id", "") + poster_id = body.get("poster_id", "") + side = body.get("side", "").lower() + amount = _safe_float(body.get("amount", 0)) + duration_days = _safe_int(body.get("duration_days", 1), 1) + public_key = body.get("public_key", "") + public_key_algo = body.get("public_key_algo", "") + signature = body.get("signature", "") + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "") + + if not staker_id or not message_id or not side: + return {"ok": False, "detail": "Missing staker_id, message_id, or side"} + + stake_payload = { + "message_id": message_id, + "poster_id": poster_id, + "side": side, + "amount": amount, + "duration_days": duration_days, + } + sig_ok, sig_reason = _verify_signed_event( + event_type="stake", + node_id=staker_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=stake_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="stake", + node_id=staker_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(staker_id, public_key, public_key_algo) + except Exception: + pass + + ok, detail = oracle_ledger.place_stake( + staker_id, message_id, poster_id, side, amount, duration_days + ) + + # Record on Infonet + if ok: + try: + from services.mesh.mesh_hashchain import infonet + + normalized_payload = normalize_payload("stake", stake_payload) + infonet.append( + event_type="stake", + node_id=staker_id, + payload=normalized_payload, + signature=signature, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + except Exception: + pass + + return {"ok": ok, "detail": detail} + + +@app.get("/api/mesh/oracle/stakes/{message_id}") +@limiter.limit("30/minute") +async def oracle_stakes_for_message(request: Request, message_id: str): + """Get all oracle stakes on a message.""" + from services.mesh.mesh_oracle import oracle_ledger + + return _redact_public_oracle_stakes( + oracle_ledger.get_stakes_for_message(message_id), + authenticated=_scoped_view_authenticated(request, "mesh.audit"), + ) + + +@app.get("/api/mesh/oracle/profile") +@limiter.limit("30/minute") +async def oracle_profile(request: Request, node_id: str = ""): + """Get full oracle profile — rep, prediction history, win rate, farming score.""" + from services.mesh.mesh_oracle import oracle_ledger + + if not node_id: + return {"ok": False, "detail": "Provide ?node_id=xxx"} + profile = oracle_ledger.get_oracle_profile(node_id) + return _redact_public_oracle_profile( + profile, + authenticated=_scoped_view_authenticated(request, "mesh.audit"), + ) + + +@app.get("/api/mesh/oracle/predictions") +@limiter.limit("30/minute") +async def oracle_predictions(request: Request, node_id: str = ""): + """Get a node's active (unresolved) predictions.""" + from services.mesh.mesh_oracle import oracle_ledger + + if not node_id: + return {"ok": False, "detail": "Provide ?node_id=xxx"} + active_predictions = oracle_ledger.get_active_predictions(node_id) + return _redact_public_oracle_predictions( + active_predictions, + authenticated=_scoped_view_authenticated(request, "mesh.audit"), + ) + + +@app.post("/api/mesh/oracle/resolve-stakes") +@limiter.limit("5/minute") +async def oracle_resolve_stakes(request: Request): + """Resolve all expired stake contests. Can be called periodically or manually.""" + from services.mesh.mesh_oracle import oracle_ledger + + resolutions = oracle_ledger.resolve_expired_stakes() + return {"ok": True, "resolutions": resolutions, "count": len(resolutions)} + + +# ─── Encrypted DM Relay (Dead Drop) ─────────────────────────────────────── + + +def _secure_dm_enabled() -> bool: + return bool(get_settings().MESH_DM_SECURE_MODE) + + +def _legacy_dm_get_allowed() -> bool: + return bool(get_settings().MESH_DM_ALLOW_LEGACY_GET) + + +def _rns_private_dm_ready() -> bool: + try: + from services.mesh.mesh_rns import rns_bridge + + return bool(rns_bridge.enabled()) and bool(rns_bridge.status().get("private_dm_direct_ready")) + except Exception: + return False + + +def _anonymous_dm_hidden_transport_enforced() -> bool: + state = _anonymous_mode_state() + return bool(state.get("enabled")) + + +def _high_privacy_profile_enabled() -> bool: + try: + from services.wormhole_settings import read_wormhole_settings + + settings = read_wormhole_settings() + return str(settings.get("privacy_profile", "default") or "default").lower() == "high" + except Exception: + return False + + +async def _maybe_apply_dm_relay_jitter() -> None: + if not _high_privacy_profile_enabled(): + return + await asyncio.sleep((50 + secrets.randbelow(451)) / 1000.0) + + +def _dm_request_fresh(timestamp: int) -> bool: + now_ts = int(time.time()) + max_age = max(30, int(get_settings().MESH_DM_REQUEST_MAX_AGE_S)) + return abs(timestamp - now_ts) <= max_age + + +def _normalize_mailbox_claims(mailbox_claims: list[dict]) -> list[dict]: + normalized: list[dict] = [] + for claim in mailbox_claims[:32]: + if not isinstance(claim, dict): + continue + normalized.append( + { + "type": str(claim.get("type", "")).lower(), + "token": str(claim.get("token", "")), + } + ) + return normalized + + +def _verify_dm_mailbox_request( + *, + event_type: str, + agent_id: str, + mailbox_claims: list[dict], + timestamp: int, + nonce: str, + public_key: str, + public_key_algo: str, + signature: str, + sequence: int, + protocol_version: str, +): + payload = { + "mailbox_claims": _normalize_mailbox_claims(mailbox_claims), + "timestamp": timestamp, + "nonce": nonce, + } + valid, reason = validate_event_payload(event_type, payload) + if not valid: + return False, reason, payload + sig_ok, sig_reason = _verify_signed_event( + event_type=event_type, + node_id=agent_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return False, sig_reason, payload + if not _dm_request_fresh(timestamp): + return False, "Mailbox request timestamp is stale", payload + return True, "ok", payload + + +@app.post("/api/mesh/dm/register") +@limiter.limit("10/minute") +async def dm_register_key(request: Request): + """Register a DH public key for encrypted DM key exchange.""" + body = await request.json() + agent_id = body.get("agent_id", "").strip() + dh_pub_key = body.get("dh_pub_key", "").strip() + dh_algo = body.get("dh_algo", "").strip() + timestamp = _safe_int(body.get("timestamp", 0) or 0) + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if not agent_id or not dh_pub_key or not dh_algo or not timestamp: + return {"ok": False, "detail": "Missing agent_id, dh_pub_key, dh_algo, or timestamp"} + if dh_algo.upper() not in ("X25519", "ECDH_P256", "ECDH"): + return {"ok": False, "detail": "Unsupported dh_algo"} + now_ts = int(time.time()) + if abs(timestamp - now_ts) > 7 * 86400: + return {"ok": False, "detail": "DH key timestamp is too far from current time"} + from services.mesh.mesh_dm_relay import dm_relay + + key_payload = {"dh_pub_key": dh_pub_key, "dh_algo": dh_algo, "timestamp": timestamp} + sig_ok, sig_reason = _verify_signed_event( + event_type="dm_key", + node_id=agent_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=key_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(agent_id, public_key, public_key_algo) + except Exception: + pass + + accepted, detail, metadata = dm_relay.register_dh_key( + agent_id, + dh_pub_key, + dh_algo, + timestamp, + signature, + public_key, + public_key_algo, + protocol_version, + sequence, + ) + if not accepted: + return {"ok": False, "detail": detail} + + return {"ok": True, **(metadata or {})} + + +@app.get("/api/mesh/dm/pubkey") +@limiter.limit("30/minute") +async def dm_get_pubkey(request: Request, agent_id: str = ""): + """Fetch an agent's DH public key for key exchange.""" + if not agent_id: + return {"ok": False, "detail": "Missing agent_id"} + from services.mesh.mesh_dm_relay import dm_relay + + key_bundle = dm_relay.get_dh_key(agent_id) + if key_bundle is None: + return {"ok": False, "detail": "Agent not found or has no DH key"} + return {"ok": True, "agent_id": agent_id, **key_bundle} + + +@app.get("/api/mesh/dm/prekey-bundle") +@limiter.limit("30/minute") +async def dm_get_prekey_bundle(request: Request, agent_id: str = ""): + if not agent_id: + return {"ok": False, "detail": "Missing agent_id"} + return fetch_dm_prekey_bundle(agent_id) + + +@app.post("/api/mesh/dm/send") +@limiter.limit("20/minute") +async def dm_send(request: Request): + """Deposit an encrypted DM in recipient's mailbox.""" + from services.wormhole_supervisor import get_transport_tier + + tier = get_transport_tier() + if tier == "public_degraded" and not _is_debug_test_request(request): + return JSONResponse( + status_code=428, + content={"ok": False, "detail": "DM send requires private transport"}, + ) + body = await request.json() + sender_id = body.get("sender_id", "").strip() + sender_token = str(body.get("sender_token", "")).strip() + sender_token_hash = "" + recipient_id = body.get("recipient_id", "").strip() + delivery_class = str(body.get("delivery_class", "")).strip().lower() + recipient_token = str(body.get("recipient_token", "")).strip() + ciphertext = body.get("ciphertext", "").strip() + payload_format = str(body.get("format", "mls1") or "mls1").strip().lower() or "mls1" + if str(tier or "").startswith("private_") and payload_format == "dm1": + return JSONResponse( + {"ok": False, "detail": "MLS session required in private transport mode — dm1 blocked on raw send path"}, + status_code=403, + ) + session_welcome = str(body.get("session_welcome", "") or "").strip() + sender_seal = str(body.get("sender_seal", "")).strip() + relay_salt_hex = str(body.get("relay_salt", "") or "").strip().lower() + msg_id = body.get("msg_id", "").strip() + timestamp = _safe_int(body.get("timestamp", 0) or 0) + nonce = str(body.get("nonce", "")).strip() + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if sender_token: + token_result = consume_wormhole_dm_sender_token( + sender_token=sender_token, + recipient_id=recipient_id, + delivery_class=delivery_class, + recipient_token=recipient_token, + ) + if not token_result.get("ok"): + return token_result + if not recipient_id: + recipient_id = str(token_result.get("recipient_id", "") or "") + sender_id = str(token_result.get("sender_id", "") or sender_id) + sender_token_hash = str(token_result.get("sender_token_hash", "") or "") + public_key = str(token_result.get("public_key", "") or public_key) + public_key_algo = str(token_result.get("public_key_algo", "") or public_key_algo) + protocol_version = str(token_result.get("protocol_version", "") or protocol_version) + from services.mesh.mesh_crypto import verify_node_binding + + derived_sender_id = sender_id + if public_key and not verify_node_binding(sender_id or derived_sender_id, public_key): + derived_sender_id = derive_node_id(public_key) + if sender_seal: + if not derived_sender_id: + return {"ok": False, "detail": "sender_seal requires a valid public key"} + if sender_id and sender_id != derived_sender_id: + return {"ok": False, "detail": "sender_id does not match sender_seal public key"} + sender_id = derived_sender_id + if not sender_id or not recipient_id or not ciphertext or not msg_id or not timestamp: + return {"ok": False, "detail": "Missing sender_id, recipient_id, ciphertext, msg_id, or timestamp"} + now_ts = int(time.time()) + if abs(timestamp - now_ts) > 7 * 86400: + return {"ok": False, "detail": "DM timestamp is too far from current time"} + if delivery_class not in ("request", "shared"): + return {"ok": False, "detail": "delivery_class must be request or shared"} + if ( + str(tier or "").startswith("private_") + and delivery_class == "shared" + and bool(get_settings().MESH_DM_REQUIRE_SENDER_SEAL_SHARED) + and not sender_seal + ): + return {"ok": False, "detail": "sealed sender required for shared private DMs"} + if delivery_class == "shared" and not recipient_token: + return {"ok": False, "detail": "recipient_token required for shared delivery"} + if delivery_class == "shared" and not sender_token_hash: + return {"ok": False, "detail": "sender_token required for shared delivery"} + from services.mesh.mesh_dm_relay import dm_relay + + dm_payload = { + "recipient_id": recipient_id, + "delivery_class": delivery_class, + "recipient_token": recipient_token, + "ciphertext": ciphertext, + "format": payload_format, + "msg_id": msg_id, + "timestamp": timestamp, + } + if session_welcome: + dm_payload["session_welcome"] = session_welcome + if sender_seal: + dm_payload["sender_seal"] = sender_seal + if relay_salt_hex: + dm_payload["relay_salt"] = relay_salt_hex + sig_ok, sig_reason = _verify_signed_event( + event_type="dm_message", + node_id=sender_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=dm_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + send_nonce = nonce or msg_id + nonce_ok, nonce_reason = dm_relay.consume_nonce(sender_id, send_nonce, timestamp) + if not nonce_ok: + return {"ok": False, "detail": nonce_reason} + try: + from services.mesh.mesh_hashchain import infonet + + ok_seq, seq_reason = infonet.validate_and_set_sequence(sender_id, sequence) + if not ok_seq: + return {"ok": False, "detail": seq_reason} + except Exception as exc: + logger.warning("DM send sequence validation unavailable: %s", type(exc).__name__) + + def _append_dm_event() -> str | None: + # Private DMs are intentionally off-ledger. The relay / Reticulum mailboxes + # already carry the encrypted payload, and mirroring them into the public + # chain creates exactly the metadata surface we are trying to avoid. + # + # Keep the hook shape here so later phases can add private local audit + # storage without reworking the send path again. + return None + + relay_sender_id = sender_id + if sender_seal: + if relay_salt_hex: + if len(relay_salt_hex) != 32 or any(ch not in "0123456789abcdef" for ch in relay_salt_hex): + return {"ok": False, "detail": "relay_salt must be a 32-character hex string"} + else: + import os as _os + + relay_salt_hex = _os.urandom(16).hex() + relay_sender_id = "sealed:" + hmac.new( + bytes.fromhex(relay_salt_hex), sender_id.encode("utf-8"), hashlib.sha256 + ).hexdigest()[:16] + + transport = "relay" + direct_result = None + anonymous_dm_hidden_transport = _anonymous_dm_hidden_transport_enforced() + if _secure_dm_enabled() and _rns_private_dm_ready() and not anonymous_dm_hidden_transport: + try: + from services.mesh.mesh_dm_relay import dm_relay + from services.mesh.mesh_rns import rns_bridge + + if dm_relay.is_blocked(recipient_id, sender_id): + return {"ok": False, "detail": "Recipient is not accepting your messages"} + + mailbox_key = dm_relay.mailbox_key_for_delivery( + recipient_id=recipient_id, + delivery_class=delivery_class, + recipient_token=recipient_token if delivery_class == "shared" else None, + ) + direct_result = rns_bridge.send_private_dm( + mailbox_key=mailbox_key, + envelope={ + "sender_id": relay_sender_id, + "ciphertext": ciphertext, + "format": payload_format, + "session_welcome": session_welcome, + "timestamp": timestamp, + "msg_id": msg_id, + "delivery_class": delivery_class, + "sender_seal": sender_seal, + }, + ) + if direct_result: + transport = "reticulum" + append_error = _append_dm_event() + if append_error: + return {"ok": False, "detail": append_error} + return {"ok": True, "msg_id": msg_id, "transport": transport, "detail": "Delivered via Reticulum"} + except Exception: + direct_result = False + + await _maybe_apply_dm_relay_jitter() + deposit_result = dm_relay.deposit( + sender_id=relay_sender_id, + raw_sender_id=sender_id, + recipient_id=recipient_id, + ciphertext=ciphertext, + msg_id=msg_id, + delivery_class=delivery_class, + recipient_token=recipient_token if delivery_class == "shared" else None, + sender_seal=sender_seal, + sender_token_hash=sender_token_hash, + payload_format=payload_format, + session_welcome=session_welcome, + ) + if not deposit_result.get("ok"): + return deposit_result + + append_error = _append_dm_event() + if append_error: + return {"ok": False, "detail": append_error} + + deposit_result["transport"] = transport + if anonymous_dm_hidden_transport: + deposit_result["detail"] = ( + deposit_result.get("detail") + or "Anonymous mode keeps private DMs off direct transport; delivered via hidden relay path" + ) + elif direct_result is False and _secure_dm_enabled(): + deposit_result["detail"] = deposit_result.get("detail") or "Reticulum unavailable, relay fallback used" + return deposit_result + + +_REQUEST_V2_REDUCED_VERSION = "request-v2-reduced-v3" +_REQUEST_V2_RECOVERY_STATES = {"pending", "verified", "failed"} + + +def _is_canonical_reduced_request_message(message: dict[str, Any]) -> bool: + item = dict(message or {}) + return ( + str(item.get("delivery_class", "") or "").strip().lower() == "request" + and str(item.get("request_contract_version", "") or "").strip() + == _REQUEST_V2_REDUCED_VERSION + and item.get("sender_recovery_required") is True + ) + + +def _annotate_request_recovery_message(message: dict[str, Any]) -> dict[str, Any]: + item = dict(message or {}) + delivery_class = str(item.get("delivery_class", "") or "").strip().lower() + sender_id = str(item.get("sender_id", "") or "").strip() + sender_seal = str(item.get("sender_seal", "") or "").strip() + if delivery_class != "request" or not sender_id.startswith("sealed:") or not sender_seal.startswith("v3:"): + return item + if not str(item.get("request_contract_version", "") or "").strip(): + item["request_contract_version"] = _REQUEST_V2_REDUCED_VERSION + item["sender_recovery_required"] = True + state = str(item.get("sender_recovery_state", "") or "").strip().lower() + if state not in _REQUEST_V2_RECOVERY_STATES: + state = "pending" + item["sender_recovery_state"] = state + return item + + +def _annotate_request_recovery_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [_annotate_request_recovery_message(message) for message in (messages or [])] + + +def _request_duplicate_authority_rank(message: dict[str, Any]) -> int: + item = dict(message or {}) + if str(item.get("delivery_class", "") or "").strip().lower() != "request": + return 0 + if _is_canonical_reduced_request_message(item): + return 3 + sender_id = str(item.get("sender_id", "") or "").strip() + if sender_id.startswith("sealed:"): + return 1 + if sender_id: + return 2 + return 0 + + +def _request_duplicate_recovery_rank(message: dict[str, Any]) -> int: + if not _is_canonical_reduced_request_message(message): + return 0 + state = str(dict(message or {}).get("sender_recovery_state", "") or "").strip().lower() + if state == "verified": + return 2 + if state == "pending": + return 1 + return 0 + + +def _poll_duplicate_source_rank(source: str) -> int: + normalized = str(source or "").strip().lower() + if normalized == "relay": + return 2 + if normalized == "reticulum": + return 1 + return 0 + + +def _should_replace_dm_poll_duplicate( + existing: dict[str, Any], + existing_source: str, + candidate: dict[str, Any], + candidate_source: str, +) -> bool: + candidate_authority = _request_duplicate_authority_rank(candidate) + existing_authority = _request_duplicate_authority_rank(existing) + if candidate_authority != existing_authority: + return candidate_authority > existing_authority + + candidate_recovery = _request_duplicate_recovery_rank(candidate) + existing_recovery = _request_duplicate_recovery_rank(existing) + if candidate_recovery != existing_recovery: + return candidate_recovery > existing_recovery + + candidate_source_rank = _poll_duplicate_source_rank(candidate_source) + existing_source_rank = _poll_duplicate_source_rank(existing_source) + if candidate_source_rank != existing_source_rank: + return candidate_source_rank > existing_source_rank + + try: + candidate_ts = float(candidate.get("timestamp", 0) or 0) + except Exception: + candidate_ts = 0.0 + try: + existing_ts = float(existing.get("timestamp", 0) or 0) + except Exception: + existing_ts = 0.0 + return candidate_ts > existing_ts + + +def _merge_dm_poll_messages( + relay_messages: list[dict[str, Any]], + direct_messages: list[dict[str, Any]], +) -> list[dict[str, Any]]: + merged: list[dict[str, Any]] = [] + index_by_msg_id: dict[str, tuple[int, str]] = {} + + def add_messages(items: list[dict[str, Any]], source: str) -> None: + for original in items or []: + item = dict(original or {}) + msg_id = str(item.get("msg_id", "") or "").strip() + if not msg_id: + merged.append(item) + continue + existing = index_by_msg_id.get(msg_id) + if existing is None: + index_by_msg_id[msg_id] = (len(merged), source) + merged.append(item) + continue + index, existing_source = existing + if _should_replace_dm_poll_duplicate(merged[index], existing_source, item, source): + merged[index] = item + index_by_msg_id[msg_id] = (index, source) + + add_messages(relay_messages, "relay") + add_messages(direct_messages, "reticulum") + return sorted(merged, key=lambda item: float(item.get("timestamp", 0) or 0)) + + +@app.post("/api/mesh/dm/poll") +@limiter.limit("30/minute") +async def dm_poll_secure(request: Request): + """Pick up pending DMs via signed mailbox claims.""" + body = await request.json() + agent_id = body.get("agent_id", "").strip() + mailbox_claims = body.get("mailbox_claims", []) + timestamp = _safe_int(body.get("timestamp", 0) or 0) + nonce = str(body.get("nonce", "")).strip() + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if not agent_id: + return {"ok": False, "detail": "Missing agent_id"} + from services.mesh.mesh_dm_relay import dm_relay + + ok, reason, payload = _verify_dm_mailbox_request( + event_type="dm_poll", + agent_id=agent_id, + mailbox_claims=mailbox_claims, + timestamp=timestamp, + nonce=nonce, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + sequence=sequence, + protocol_version=protocol_version, + ) + if not ok: + return {"ok": False, "detail": reason, "messages": [], "count": 0} + nonce_ok, nonce_reason = dm_relay.consume_nonce(agent_id, nonce, timestamp) + if not nonce_ok: + return {"ok": False, "detail": nonce_reason, "messages": [], "count": 0} + try: + from services.mesh.mesh_hashchain import infonet + + ok_seq, seq_reason = infonet.validate_and_set_sequence(agent_id, sequence) + if not ok_seq: + return {"ok": False, "detail": seq_reason, "messages": [], "count": 0} + except Exception: + pass + claims = payload.get("mailbox_claims", []) + mailbox_keys = dm_relay.claim_mailbox_keys(agent_id, claims) + msgs = _annotate_request_recovery_messages(dm_relay.collect_claims(agent_id, claims)) + direct_msgs = [] + if not _anonymous_dm_hidden_transport_enforced(): + try: + from services.mesh.mesh_rns import rns_bridge + + direct_msgs = _annotate_request_recovery_messages( + rns_bridge.collect_private_dm(mailbox_keys) + ) + except Exception: + direct_msgs = [] + msgs = _merge_dm_poll_messages(msgs, direct_msgs) + return {"ok": True, "messages": msgs, "count": len(msgs)} + + +@app.get("/api/mesh/dm/poll") +@limiter.limit("30/minute") +async def dm_poll( + request: Request, + agent_id: str = "", + agent_token: str = "", + agent_token_prev: str = "", + agent_tokens: str = "", +): + """Pick up all pending DMs. Removes them from mailbox after retrieval.""" + if _secure_dm_enabled() and not _legacy_dm_get_allowed(): + return {"ok": False, "detail": "Legacy GET polling is disabled in secure mode", "messages": [], "count": 0} + if not agent_id and not agent_token and not agent_token_prev and not agent_tokens: + return {"ok": True, "messages": [], "count": 0} + from services.mesh.mesh_dm_relay import dm_relay + tokens: list[str] = [] + if agent_tokens: + for token in agent_tokens.split(","): + token = token.strip() + if token: + tokens.append(token) + if agent_token: + tokens.append(agent_token) + if agent_token_prev and agent_token_prev != agent_token: + tokens.append(agent_token_prev) + # Deduplicate while preserving order + seen = set() + unique_tokens: list[str] = [] + for token in tokens: + if token in seen: + continue + seen.add(token) + unique_tokens.append(token) + msgs: list[dict] = [] + if unique_tokens: + for token in unique_tokens[:32]: + msgs.extend(dm_relay.collect_legacy(agent_token=token)) + return {"ok": True, "messages": msgs, "count": len(msgs)} + + +@app.post("/api/mesh/dm/count") +@limiter.limit("60/minute") +async def dm_count_secure(request: Request): + """Unread DM count via signed mailbox claims.""" + body = await request.json() + agent_id = body.get("agent_id", "").strip() + mailbox_claims = body.get("mailbox_claims", []) + timestamp = _safe_int(body.get("timestamp", 0) or 0) + nonce = str(body.get("nonce", "")).strip() + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if not agent_id: + return {"ok": False, "detail": "Missing agent_id", "count": 0} + from services.mesh.mesh_dm_relay import dm_relay + + ok, reason, payload = _verify_dm_mailbox_request( + event_type="dm_count", + agent_id=agent_id, + mailbox_claims=mailbox_claims, + timestamp=timestamp, + nonce=nonce, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + sequence=sequence, + protocol_version=protocol_version, + ) + if not ok: + return {"ok": False, "detail": reason, "count": 0} + nonce_ok, nonce_reason = dm_relay.consume_nonce(agent_id, nonce, timestamp) + if not nonce_ok: + return {"ok": False, "detail": nonce_reason, "count": 0} + try: + from services.mesh.mesh_hashchain import infonet + + ok_seq, seq_reason = infonet.validate_and_set_sequence(agent_id, sequence) + if not ok_seq: + return {"ok": False, "detail": seq_reason, "count": 0} + except Exception: + pass + claims = payload.get("mailbox_claims", []) + mailbox_keys = dm_relay.claim_mailbox_keys(agent_id, claims) + relay_ids = dm_relay.claim_message_ids(agent_id, claims) + direct_ids = set() + if not _anonymous_dm_hidden_transport_enforced(): + try: + from services.mesh.mesh_rns import rns_bridge + + direct_ids = rns_bridge.private_dm_ids(mailbox_keys) + except Exception: + direct_ids = set() + return {"ok": True, "count": len(relay_ids | direct_ids)} + + +@app.get("/api/mesh/dm/count") +@limiter.limit("60/minute") +async def dm_count( + request: Request, + agent_id: str = "", + agent_token: str = "", + agent_token_prev: str = "", + agent_tokens: str = "", +): + """Unread DM count (for notification badge). Lightweight poll.""" + if _secure_dm_enabled() and not _legacy_dm_get_allowed(): + return {"ok": False, "detail": "Legacy GET count is disabled in secure mode", "count": 0} + if not agent_id and not agent_token and not agent_token_prev and not agent_tokens: + return {"ok": True, "count": 0} + from services.mesh.mesh_dm_relay import dm_relay + tokens: list[str] = [] + if agent_tokens: + for token in agent_tokens.split(","): + token = token.strip() + if token: + tokens.append(token) + if agent_token: + tokens.append(agent_token) + if agent_token_prev and agent_token_prev != agent_token: + tokens.append(agent_token_prev) + # Deduplicate while preserving order + seen = set() + unique_tokens: list[str] = [] + for token in tokens: + if token in seen: + continue + seen.add(token) + unique_tokens.append(token) + if unique_tokens: + total = 0 + for token in unique_tokens[:32]: + total += dm_relay.count_legacy(agent_token=token) + return {"ok": True, "count": total} + return {"ok": True, "count": 0} + + +@app.post("/api/mesh/dm/block") +@limiter.limit("10/minute") +async def dm_block(request: Request): + """Block or unblock a sender from DMing you.""" + body = await request.json() + agent_id = body.get("agent_id", "").strip() + blocked_id = body.get("blocked_id", "").strip() + action = body.get("action", "block").strip().lower() + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if not agent_id or not blocked_id: + return {"ok": False, "detail": "Missing agent_id or blocked_id"} + from services.mesh.mesh_dm_relay import dm_relay + + block_payload = {"blocked_id": blocked_id, "action": action} + sig_ok, sig_reason = _verify_signed_event( + event_type="dm_block", + node_id=agent_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=block_payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + try: + from services.mesh.mesh_hashchain import infonet + + ok_seq, seq_reason = infonet.validate_and_set_sequence(agent_id, sequence) + if not ok_seq: + return {"ok": False, "detail": seq_reason} + except Exception: + pass + + if action == "unblock": + dm_relay.unblock(agent_id, blocked_id) + else: + dm_relay.block(agent_id, blocked_id) + return {"ok": True, "action": action, "blocked_id": blocked_id} + + +@app.post("/api/mesh/dm/witness") +@limiter.limit("20/minute") +async def dm_key_witness(request: Request): + """Record a lightweight witness for a DM key (dual-path spot-check).""" + body = await request.json() + witness_id = body.get("witness_id", "").strip() + target_id = body.get("target_id", "").strip() + dh_pub_key = body.get("dh_pub_key", "").strip() + timestamp = _safe_int(body.get("timestamp", 0) or 0) + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if not witness_id or not target_id or not dh_pub_key or not timestamp: + return {"ok": False, "detail": "Missing witness_id, target_id, dh_pub_key, or timestamp"} + now_ts = int(time.time()) + if abs(timestamp - now_ts) > 7 * 86400: + return {"ok": False, "detail": "Witness timestamp is too far from current time"} + payload = {"target_id": target_id, "dh_pub_key": dh_pub_key, "timestamp": timestamp} + sig_ok, sig_reason = _verify_signed_event( + event_type="dm_key_witness", + node_id=witness_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + + integrity_ok, integrity_reason = _preflight_signed_event_integrity( + event_type="dm_key_witness", + node_id=witness_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + protocol_version=protocol_version, + ) + if not integrity_ok: + return {"ok": False, "detail": integrity_reason} + + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(witness_id, public_key, public_key_algo) + except Exception: + pass + from services.mesh.mesh_dm_relay import dm_relay + + ok, reason = dm_relay.record_witness(witness_id, target_id, dh_pub_key, timestamp) + return {"ok": ok, "detail": reason} + + +@app.get("/api/mesh/dm/witness") +@limiter.limit("60/minute") +async def dm_key_witness_get(request: Request, target_id: str = "", dh_pub_key: str = ""): + """Get witness counts for a target's DH key.""" + if not target_id: + return {"ok": False, "detail": "Missing target_id"} + from services.mesh.mesh_dm_relay import dm_relay + + witnesses = dm_relay.get_witnesses(target_id, dh_pub_key if dh_pub_key else None, limit=5) + response = { + "ok": True, + "count": len(witnesses), + } + if _scoped_view_authenticated(request, "mesh.audit"): + response["target_id"] = target_id + response["dh_pub_key"] = dh_pub_key or "" + response["witnesses"] = witnesses + return response + + +@app.post("/api/mesh/trust/vouch") +@limiter.limit("20/minute") +async def trust_vouch(request: Request): + """Record a trust vouch for a node (web-of-trust signal).""" + body = await request.json() + voucher_id = body.get("voucher_id", "").strip() + target_id = body.get("target_id", "").strip() + note = body.get("note", "").strip() + timestamp = _safe_int(body.get("timestamp", 0) or 0) + public_key = body.get("public_key", "").strip() + public_key_algo = body.get("public_key_algo", "").strip() + signature = body.get("signature", "").strip() + sequence = _safe_int(body.get("sequence", 0) or 0) + protocol_version = body.get("protocol_version", "").strip() + if not voucher_id or not target_id or not timestamp: + return {"ok": False, "detail": "Missing voucher_id, target_id, or timestamp"} + now_ts = int(time.time()) + if abs(timestamp - now_ts) > 7 * 86400: + return {"ok": False, "detail": "Vouch timestamp is too far from current time"} + payload = {"target_id": target_id, "note": note, "timestamp": timestamp} + sig_ok, sig_reason = _verify_signed_event( + event_type="trust_vouch", + node_id=voucher_id, + sequence=sequence, + public_key=public_key, + public_key_algo=public_key_algo, + signature=signature, + payload=payload, + protocol_version=protocol_version, + ) + if not sig_ok: + return {"ok": False, "detail": sig_reason} + try: + from services.mesh.mesh_reputation import reputation_ledger + + reputation_ledger.register_node(voucher_id, public_key, public_key_algo) + ok, reason = reputation_ledger.add_vouch(voucher_id, target_id, note, timestamp) + return {"ok": ok, "detail": reason} + except Exception: + return {"ok": False, "detail": "Failed to record vouch"} + + +@app.get("/api/mesh/trust/vouches", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def trust_vouches(request: Request, node_id: str = "", limit: int = 20): + """Fetch latest vouches for a node.""" + if not node_id: + return {"ok": False, "detail": "Missing node_id"} + try: + from services.mesh.mesh_reputation import reputation_ledger + + vouches = reputation_ledger.get_vouches(node_id, limit=limit) + return {"ok": True, "node_id": node_id, "vouches": vouches, "count": len(vouches)} + except Exception: + return {"ok": False, "detail": "Failed to fetch vouches"} + + +@app.get("/api/debug-latest", dependencies=[Depends(require_admin)]) +@limiter.limit("30/minute") +async def debug_latest_data(request: Request): + return list(get_latest_data().keys()) + + +# ── CCTV media proxy (bypass CORS for cross-origin video/image streams) ─── +_CCTV_PROXY_ALLOWED_HOSTS = { + "s3-eu-west-1.amazonaws.com", # TfL JamCams + "jamcams.tfl.gov.uk", + "images.data.gov.sg", # Singapore LTA + "cctv.austinmobility.io", + "webcams.nyctmc.org", + # State DOT camera feeds often resolve to separate media/CDN hosts from the + # catalog/API hostname. Keep the proxy allowlist aligned with the actual + # media hosts produced by trusted ingestors so cameras render reliably. + "cwwp2.dot.ca.gov", # Caltrans + "wzmedia.dot.ca.gov", # Caltrans static media + "images.wsdot.wa.gov", # WSDOT + "olypen.com", # WSDOT Aviation-linked public camera + "flyykm.com", # WSDOT Aviation-linked public camera + "cam.pangbornairport.com", # WSDOT Aviation-linked public camera + "navigator-c2c.dot.ga.gov", # Georgia DOT + "navigator-c2c.ga.gov", # Georgia DOT alternate host variant + "navigator-csc.dot.ga.gov", # Georgia DOT alternate catalog/media host + "vss1live.dot.ga.gov", # Georgia DOT stream hosts + "vss2live.dot.ga.gov", + "vss3live.dot.ga.gov", + "vss4live.dot.ga.gov", + "vss5live.dot.ga.gov", + "511ga.org", # Georgia public camera images + "gettingaroundillinois.com", # Illinois DOT + "cctv.travelmidwest.com", # Illinois DOT camera media + "mdotjboss.state.mi.us", # Michigan DOT + "micamerasimages.net", # Michigan DOT image host + "publicstreamer1.cotrip.org", # Colorado DOT / COtrip HLS hosts + "publicstreamer2.cotrip.org", + "publicstreamer3.cotrip.org", + "publicstreamer4.cotrip.org", + "cocam.carsprogram.org", # Colorado DOT preview images + "tripcheck.com", # Oregon DOT / TripCheck + "www.tripcheck.com", + "infocar.dgt.es", # Spain DGT + "informo.madrid.es", # Madrid + "www.windy.com", +} + + +@dataclass(frozen=True) +class _CCTVProxyProfile: + name: str + timeout: tuple[float, float] = (5.0, 10.0) + cache_seconds: int = 30 + headers: dict[str, str] = field(default_factory=dict) + + +def _cctv_host_allowed(hostname: str | None) -> bool: + host = str(hostname or "").strip().lower() + if not host: + return False + for allowed in _CCTV_PROXY_ALLOWED_HOSTS: + normalized = str(allowed or "").strip().lower() + if host == normalized or host.endswith(f".{normalized}"): + return True + return False + + +def _proxied_cctv_url(target_url: str) -> str: + from urllib.parse import quote + + return f"/api/cctv/media?url={quote(target_url, safe='')}" + + +def _cctv_proxy_profile_for_url(target_url: str) -> _CCTVProxyProfile: + from urllib.parse import urlparse + + parsed = urlparse(target_url) + host = str(parsed.hostname or "").strip().lower() + path = str(parsed.path or "").strip().lower() + + if host in {"jamcams.tfl.gov.uk", "s3-eu-west-1.amazonaws.com"}: + return _CCTVProxyProfile( + name="tfl-jamcam", + timeout=(5.0, 20.0), + cache_seconds=15, + headers={ + "Accept": "video/mp4,image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://tfl.gov.uk/", + }, + ) + if host == "images.data.gov.sg": + return _CCTVProxyProfile( + name="lta-singapore", + timeout=(5.0, 10.0), + cache_seconds=30, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, + ) + if host == "cctv.austinmobility.io": + return _CCTVProxyProfile( + name="austin-mobility", + timeout=(5.0, 8.0), + cache_seconds=15, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://data.mobility.austin.gov/", + "Origin": "https://data.mobility.austin.gov", + }, + ) + if host == "webcams.nyctmc.org": + return _CCTVProxyProfile( + name="nyc-dot", + timeout=(5.0, 10.0), + cache_seconds=15, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, + ) + if host in {"cwwp2.dot.ca.gov", "wzmedia.dot.ca.gov"}: + return _CCTVProxyProfile( + name="caltrans", + timeout=(5.0, 15.0), + cache_seconds=15, + headers={ + "Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,image/*,*/*;q=0.8", + "Referer": "https://cwwp2.dot.ca.gov/", + }, + ) + if host in {"images.wsdot.wa.gov", "olypen.com", "flyykm.com", "cam.pangbornairport.com"}: + return _CCTVProxyProfile( + name="wsdot", + timeout=(5.0, 12.0), + cache_seconds=30, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, + ) + if host in {"navigator-c2c.dot.ga.gov", "navigator-c2c.ga.gov", "navigator-csc.dot.ga.gov"}: + read_timeout = 18.0 if "/snapshots/" in path else 12.0 + return _CCTVProxyProfile( + name="gdot-snapshot", + timeout=(5.0, read_timeout), + cache_seconds=15, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "http://navigator-c2c.dot.ga.gov/", + }, + ) + if host == "511ga.org": + return _CCTVProxyProfile( + name="gdot-511ga-image", + timeout=(5.0, 12.0), + cache_seconds=15, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://511ga.org/cctv", + }, + ) + if host.startswith("vss") and host.endswith("dot.ga.gov"): + return _CCTVProxyProfile( + name="gdot-hls", + timeout=(5.0, 20.0), + cache_seconds=10, + headers={ + "Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8", + "Referer": "http://navigator-c2c.dot.ga.gov/", + }, + ) + if host in {"gettingaroundillinois.com", "cctv.travelmidwest.com"}: + return _CCTVProxyProfile( + name="illinois-dot", + timeout=(5.0, 12.0), + cache_seconds=30, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, + ) + if host in {"mdotjboss.state.mi.us", "micamerasimages.net"}: + return _CCTVProxyProfile( + name="michigan-dot", + timeout=(5.0, 12.0), + cache_seconds=30, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://mdotjboss.state.mi.us/", + }, + ) + if host in { + "publicstreamer1.cotrip.org", + "publicstreamer2.cotrip.org", + "publicstreamer3.cotrip.org", + "publicstreamer4.cotrip.org", + }: + return _CCTVProxyProfile( + name="cotrip-hls", + timeout=(5.0, 20.0), + cache_seconds=10, + headers={ + "Accept": "application/vnd.apple.mpegurl,application/x-mpegURL,video/*,*/*;q=0.8", + "Referer": "https://www.cotrip.org/", + }, + ) + if host == "cocam.carsprogram.org": + return _CCTVProxyProfile( + name="cotrip-preview", + timeout=(5.0, 12.0), + cache_seconds=20, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://www.cotrip.org/", + }, + ) + if host in {"tripcheck.com", "www.tripcheck.com"}: + return _CCTVProxyProfile( + name="odot-tripcheck", + timeout=(5.0, 12.0), + cache_seconds=30, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, + ) + if host == "infocar.dgt.es": + return _CCTVProxyProfile( + name="dgt-spain", + timeout=(5.0, 8.0), + cache_seconds=60, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://infocar.dgt.es/", + }, + ) + if host == "informo.madrid.es": + return _CCTVProxyProfile( + name="madrid-city", + timeout=(5.0, 12.0), + cache_seconds=30, + headers={ + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://informo.madrid.es/", + }, + ) + if host == "www.windy.com": + return _CCTVProxyProfile( + name="windy-webcams", + timeout=(5.0, 12.0), + cache_seconds=60, + headers={"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"}, + ) + return _CCTVProxyProfile( + name="generic-cctv", + timeout=(5.0, 10.0), + cache_seconds=30, + headers={"Accept": "*/*"}, + ) + + +def _cctv_upstream_headers(request: Request, profile: _CCTVProxyProfile) -> dict[str, str]: + headers = { + "User-Agent": "Mozilla/5.0 (compatible; ShadowBroker CCTV proxy)", + **profile.headers, + } + range_header = request.headers.get("range") + if range_header: + headers["Range"] = range_header + if_none_match = request.headers.get("if-none-match") + if if_none_match: + headers["If-None-Match"] = if_none_match + if_modified_since = request.headers.get("if-modified-since") + if if_modified_since: + headers["If-Modified-Since"] = if_modified_since + return headers + + +def _cctv_response_headers(resp, cache_seconds: int, include_length: bool = True) -> dict[str, str]: + headers = { + "Cache-Control": f"public, max-age={cache_seconds}", + "Access-Control-Allow-Origin": "*", + } + for key in ("Accept-Ranges", "Content-Range", "ETag", "Last-Modified"): + value = resp.headers.get(key) + if value: + headers[key] = value + if include_length: + content_length = resp.headers.get("Content-Length") + if content_length: + headers["Content-Length"] = content_length + return headers + + +def _fetch_cctv_upstream_response(request: Request, target_url: str, profile: _CCTVProxyProfile): + import requests as _req + + headers = _cctv_upstream_headers(request, profile) + try: + resp = _req.get( + target_url, + timeout=profile.timeout, + stream=True, + allow_redirects=True, + headers=headers, + ) + except _req.exceptions.Timeout as exc: + logger.warning("CCTV upstream timeout [%s] %s", profile.name, target_url) + raise HTTPException(status_code=504, detail="Upstream timeout") from exc + except _req.exceptions.RequestException as exc: + logger.warning("CCTV upstream request failure [%s] %s: %s", profile.name, target_url, exc) + raise HTTPException(status_code=502, detail="Upstream fetch failed") from exc + + if resp.status_code >= 400: + logger.info("CCTV upstream HTTP %s [%s] %s", resp.status_code, profile.name, target_url) + resp.close() + raise HTTPException(status_code=int(resp.status_code), detail=f"Upstream returned {resp.status_code}") + return resp + + +def _proxy_cctv_media_response(request: Request, target_url: str): + from urllib.parse import urlparse + + parsed = urlparse(target_url) + profile = _cctv_proxy_profile_for_url(target_url) + resp = _fetch_cctv_upstream_response(request, target_url, profile) + + content_type = resp.headers.get("Content-Type", "application/octet-stream") + is_hls_playlist = ( + ".m3u8" in str(parsed.path or "").lower() + or "mpegurl" in content_type.lower() + or "vnd.apple.mpegurl" in content_type.lower() + ) + if is_hls_playlist: + body = resp.text + if "#EXTM3U" in body: + body = _rewrite_cctv_hls_playlist(target_url, body) + resp.close() + return Response( + content=body, + media_type=content_type, + headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds, include_length=False), + ) + return StreamingResponse( + resp.iter_content(chunk_size=65536), + status_code=resp.status_code, + media_type=content_type, + headers=_cctv_response_headers(resp, cache_seconds=profile.cache_seconds), + background=BackgroundTask(resp.close), + ) + + +def _rewrite_cctv_hls_playlist(base_url: str, body: str) -> str: + import re + from urllib.parse import urljoin, urlparse + + def _rewrite_target(target: str) -> str: + candidate = str(target or "").strip() + if not candidate or candidate.startswith("data:"): + return candidate + absolute = urljoin(base_url, candidate) + parsed_target = urlparse(absolute) + if parsed_target.scheme not in ("http", "https"): + return candidate + if not _cctv_host_allowed(parsed_target.hostname): + return candidate + return _proxied_cctv_url(absolute) + + rewritten_lines: list[str] = [] + for raw_line in body.splitlines(): + stripped = raw_line.strip() + if not stripped: + rewritten_lines.append(raw_line) + continue + if stripped.startswith("#"): + rewritten_lines.append( + re.sub( + r'URI="([^"]+)"', + lambda match: f'URI="{_rewrite_target(match.group(1))}"', + raw_line, + ) + ) + continue + rewritten_lines.append(_rewrite_target(stripped)) + return "\n".join(rewritten_lines) + ("\n" if body.endswith("\n") else "") + + +@app.get("/api/cctv/media") +@limiter.limit("120/minute") +async def cctv_media_proxy(request: Request, url: str = Query(...)): + """Proxy CCTV media through the backend to bypass browser CORS restrictions.""" + from urllib.parse import urlparse + + parsed = urlparse(url) + if not _cctv_host_allowed(parsed.hostname): + raise HTTPException(status_code=403, detail="Host not allowed") + if parsed.scheme not in ("http", "https"): + raise HTTPException(status_code=400, detail="Invalid scheme") + + return _proxy_cctv_media_response(request, url) + + +@app.get("/api/health", response_model=HealthResponse) +@limiter.limit("30/minute") +async def health_check(request: Request): import time + from services.fetchers._store import get_source_timestamps_snapshot + d = get_latest_data() last = d.get("last_updated") return { - "status": "ok", - "last_updated": last, - "sources": { - "flights": len(d.get("commercial_flights", [])), - "military": len(d.get("military_flights", [])), - "ships": len(d.get("ships", [])), - "satellites": len(d.get("satellites", [])), - "earthquakes": len(d.get("earthquakes", [])), - "cctv": len(d.get("cctv", [])), - "news": len(d.get("news", [])), - "uavs": len(d.get("uavs", [])), - "firms_fires": len(d.get("firms_fires", [])), - "liveuamap": len(d.get("liveuamap", [])), - "gdelt": len(d.get("gdelt", [])), + "status": "ok", + "last_updated": last, + "sources": { + "flights": len(d.get("commercial_flights", [])), + "military": len(d.get("military_flights", [])), + "ships": len(d.get("ships", [])), + "satellites": len(d.get("satellites", [])), + "earthquakes": len(d.get("earthquakes", [])), + "cctv": len(d.get("cctv", [])), + "news": len(d.get("news", [])), + "uavs": len(d.get("uavs", [])), + "firms_fires": len(d.get("firms_fires", [])), + "liveuamap": len(d.get("liveuamap", [])), + "gdelt": len(d.get("gdelt", [])), + }, + "freshness": get_source_timestamps_snapshot(), + "uptime_seconds": round(time.time() - _start_time), + } + + +from services.radio_intercept import ( + get_top_broadcastify_feeds, + get_openmhz_systems, + get_recent_openmhz_calls, + find_nearest_openmhz_system, +) + + +@app.get("/api/radio/top") +@limiter.limit("30/minute") +async def get_top_radios(request: Request): + return get_top_broadcastify_feeds() + + +@app.get("/api/radio/openmhz/systems") +@limiter.limit("30/minute") +async def api_get_openmhz_systems(request: Request): + return get_openmhz_systems() + + +@app.get("/api/radio/openmhz/calls/{sys_name}") +@limiter.limit("60/minute") +async def api_get_openmhz_calls(request: Request, sys_name: str): + return get_recent_openmhz_calls(sys_name) + + +@app.get("/api/radio/nearest") +@limiter.limit("60/minute") +async def api_get_nearest_radio( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), +): + return find_nearest_openmhz_system(lat, lng) + + +from services.radio_intercept import find_nearest_openmhz_systems_list + + +@app.get("/api/radio/nearest-list") +@limiter.limit("60/minute") +async def api_get_nearest_radios_list( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), + limit: int = Query(5, ge=1, le=20), +): + return find_nearest_openmhz_systems_list(lat, lng, limit=limit) + + +from services.network_utils import fetch_with_curl + + +@app.get("/api/route/{callsign}") +@limiter.limit("60/minute") +async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0): + r = fetch_with_curl( + "https://api.adsb.lol/api/0/routeset", + method="POST", + json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]}, + timeout=10, + ) + if r and r.status_code == 200: + data = r.json() + route_list = [] + if isinstance(data, dict): + route_list = data.get("value", []) + elif isinstance(data, list): + route_list = data + + if route_list and len(route_list) > 0: + route = route_list[0] + airports = route.get("_airports", []) + if len(airports) >= 2: + orig = airports[0] + dest = airports[-1] + return { + "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)], + "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)], + "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}", + "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}", + } + return {} + + +from services.region_dossier import get_region_dossier + + +@app.get("/api/region-dossier") +@limiter.limit("30/minute") +def api_region_dossier( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), +): + """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop.""" + return get_region_dossier(lat, lng) + + +# --------------------------------------------------------------------------- +# Geocoding — proxy to Nominatim with caching and proper headers +# --------------------------------------------------------------------------- +from services.geocode import search_geocode, reverse_geocode + + +@app.get("/api/geocode/search") +@limiter.limit("30/minute") +async def api_geocode_search( + request: Request, + q: str = "", + limit: int = 5, + local_only: bool = False, +): + if not q or len(q.strip()) < 2: + return {"results": [], "query": q, "count": 0} + results = await asyncio.to_thread(search_geocode, q, limit, local_only) + return {"results": results, "query": q, "count": len(results)} + + +@app.get("/api/geocode/reverse") +@limiter.limit("60/minute") +async def api_geocode_reverse( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), + local_only: bool = False, +): + return await asyncio.to_thread(reverse_geocode, lat, lng, local_only) + + +from services.sentinel_search import search_sentinel2_scene + + +@app.get("/api/sentinel2/search") +@limiter.limit("30/minute") +def api_sentinel2_search( + request: Request, + lat: float = Query(..., ge=-90, le=90), + lng: float = Query(..., ge=-180, le=180), +): + """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution.""" + return search_sentinel2_scene(lat, lng) + + +@app.post("/api/sentinel/token") +@limiter.limit("60/minute") +async def api_sentinel_token(request: Request): + """Proxy Copernicus CDSE OAuth2 token request (avoids browser CORS block). + + The user's client_id + client_secret are forwarded to the Copernicus + identity provider and never stored on the server. + """ + import requests as req + + # Parse URL-encoded form body manually (avoids python-multipart dependency) + body = await request.body() + from urllib.parse import parse_qs + params = parse_qs(body.decode("utf-8")) + client_id = params.get("client_id", [""])[0] + client_secret = params.get("client_secret", [""])[0] + + if not client_id or not client_secret: + raise HTTPException(400, "client_id and client_secret required") + + token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token" + try: + resp = await asyncio.to_thread( + req.post, + token_url, + data={ + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + }, + timeout=15, + ) + return Response( + content=resp.content, + status_code=resp.status_code, + media_type="application/json", + ) + except Exception as exc: + logger.exception("Token request failed") + raise HTTPException(502, "Token request failed") + + +# Server-side token cache for tile requests (avoids re-auth on every tile) +_sh_token_cache: dict = {"token": None, "expiry": 0, "client_id": ""} + + +@app.post("/api/sentinel/tile") +@limiter.limit("300/minute") +async def api_sentinel_tile(request: Request): + """Proxy Sentinel Hub Process API tile request (avoids CORS block). + + Expects JSON body with: client_id, client_secret, preset, date, z, x, y. + Returns the PNG tile directly. + """ + import requests as req + import time as _time + + try: + body = await request.json() + except Exception: + return JSONResponse(status_code=422, content={"ok": False, "detail": "invalid JSON body"}) + + client_id = body.get("client_id", "") + client_secret = body.get("client_secret", "") + preset = body.get("preset", "TRUE-COLOR") + date_str = body.get("date", "") + z = body.get("z", 0) + x = body.get("x", 0) + y = body.get("y", 0) + + if not client_id or not client_secret or not date_str: + raise HTTPException(400, "client_id, client_secret, and date required") + + # Reuse cached token if same client_id and not expired + now = _time.time() + if ( + _sh_token_cache["token"] + and _sh_token_cache["client_id"] == client_id + and now < _sh_token_cache["expiry"] - 30 + ): + token = _sh_token_cache["token"] + else: + # Fetch new token + token_url = "https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token" + try: + tresp = await asyncio.to_thread( + req.post, + token_url, + data={ + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + }, + timeout=15, + ) + if tresp.status_code != 200: + raise HTTPException(401, f"Token auth failed: {tresp.text[:200]}") + tdata = tresp.json() + token = tdata["access_token"] + _sh_token_cache["token"] = token + _sh_token_cache["expiry"] = now + tdata.get("expires_in", 300) + _sh_token_cache["client_id"] = client_id + except HTTPException: + raise + except Exception as exc: + logger.exception("Token request failed") + raise HTTPException(502, "Token request failed") + + # Compute bounding box from tile coordinates (EPSG:3857) + import math + + half = 20037508.342789244 + tile_size = (2 * half) / math.pow(2, z) + min_x = -half + x * tile_size + max_x = min_x + tile_size + max_y = half - y * tile_size + min_y = max_y - tile_size + bbox = [min_x, min_y, max_x, max_y] + + # Evalscripts + evalscripts = { + "TRUE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B04","B03","B02"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B04,2.5*s.B03,2.5*s.B02];}', + "FALSE-COLOR": '//VERSION=3\nfunction setup(){return{input:["B08","B04","B03"],output:{bands:3}};}\nfunction evaluatePixel(s){return[2.5*s.B08,2.5*s.B04,2.5*s.B03];}', + "NDVI": '//VERSION=3\nfunction setup(){return{input:["B04","B08"],output:{bands:3}};}\nfunction evaluatePixel(s){var n=(s.B08-s.B04)/(s.B08+s.B04);if(n<-0.2)return[0.05,0.05,0.05];if(n<0)return[0.75,0.75,0.75];if(n<0.1)return[0.86,0.86,0.86];if(n<0.2)return[0.92,0.84,0.68];if(n<0.3)return[0.77,0.88,0.55];if(n<0.4)return[0.56,0.80,0.32];if(n<0.5)return[0.35,0.72,0.18];if(n<0.6)return[0.20,0.60,0.08];if(n<0.7)return[0.10,0.48,0.04];return[0.0,0.36,0.0];}', + "MOISTURE-INDEX": '//VERSION=3\nfunction setup(){return{input:["B8A","B11"],output:{bands:3}};}\nfunction evaluatePixel(s){var m=(s.B8A-s.B11)/(s.B8A+s.B11);var r=Math.max(0,Math.min(1,1.5-3*m));var g=Math.max(0,Math.min(1,m<0?1.5+3*m:1.5-3*m));var b=Math.max(0,Math.min(1,1.5+3*(m-0.5)));return[r,g,b];}', + } + evalscript = evalscripts.get(preset, evalscripts["TRUE-COLOR"]) + + # Adaptive time range: wider window at lower zoom for better coverage. + # Sentinel-2 has 5-day revisit — a single day often has gaps. + # At low zoom we mosaic over more days to fill gaps. + from datetime import datetime as _dt, timedelta as _td + + try: + end_date = _dt.strptime(date_str, "%Y-%m-%d") + except ValueError: + end_date = _dt.utcnow() + + if z <= 6: + lookback_days = 30 # continent-level: mosaic a full month + elif z <= 9: + lookback_days = 14 # region-level: 2 weeks + elif z <= 11: + lookback_days = 7 # country-level: 1 week + else: + lookback_days = 5 # close-up: 5 days (one revisit cycle) + + start_date = end_date - _td(days=lookback_days) + + process_body = { + "input": { + "bounds": { + "bbox": bbox, + "properties": {"crs": "http://www.opengis.net/def/crs/EPSG/0/3857"}, + }, + "data": [ + { + "type": "sentinel-2-l2a", + "dataFilter": { + "timeRange": { + "from": start_date.strftime("%Y-%m-%dT00:00:00Z"), + "to": end_date.strftime("%Y-%m-%dT23:59:59Z"), + }, + "maxCloudCoverage": 30, + "mosaickingOrder": "leastCC", + }, + } + ], + }, + "output": { + "width": 256, + "height": 256, + "responses": [{"identifier": "default", "format": {"type": "image/png"}}], + }, + "evalscript": evalscript, + } + + try: + resp = await asyncio.to_thread( + req.post, + "https://sh.dataspace.copernicus.eu/api/v1/process", + json=process_body, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "image/png", + }, + timeout=30, + ) + return Response( + content=resp.content, + status_code=resp.status_code, + media_type=resp.headers.get("content-type", "image/png"), + ) + except Exception as exc: + logger.exception("Process API failed") + raise HTTPException(502, "Process API failed") + + +# --------------------------------------------------------------------------- +# API Settings — key registry & management +# --------------------------------------------------------------------------- +from services.api_settings import get_api_keys, update_api_key +from services.shodan_connector import ( + ShodanConnectorError, + count_shodan, + get_shodan_connector_status, + lookup_shodan_host, + search_shodan, +) +from pydantic import BaseModel + + +class ApiKeyUpdate(BaseModel): + env_key: str + value: str + + +class ShodanSearchRequest(BaseModel): + query: str + page: int = 1 + facets: list[str] = [] + + +class ShodanCountRequest(BaseModel): + query: str + facets: list[str] = [] + + +class ShodanHostRequest(BaseModel): + ip: str + history: bool = False + + +@app.get("/api/settings/api-keys", dependencies=[Depends(require_admin)]) +@limiter.limit("30/minute") +async def api_get_keys(request: Request): + return get_api_keys() + + +@app.put("/api/settings/api-keys", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def api_update_key(request: Request, body: ApiKeyUpdate): + ok = update_api_key(body.env_key, body.value) + if ok: + return {"status": "updated", "env_key": body.env_key} + return {"status": "error", "message": "Failed to update .env file"} + + +@app.get("/api/tools/shodan/status", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_shodan_status(request: Request): + return get_shodan_connector_status() + + +@app.post("/api/tools/shodan/search", dependencies=[Depends(require_local_operator)]) +@limiter.limit("12/minute") +async def api_shodan_search(request: Request, body: ShodanSearchRequest): + try: + return search_shodan(body.query, page=body.page, facets=body.facets) + except ShodanConnectorError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + +@app.post("/api/tools/shodan/count", dependencies=[Depends(require_local_operator)]) +@limiter.limit("12/minute") +async def api_shodan_count(request: Request, body: ShodanCountRequest): + try: + return count_shodan(body.query, facets=body.facets) + except ShodanConnectorError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + +@app.post("/api/tools/shodan/host", dependencies=[Depends(require_local_operator)]) +@limiter.limit("12/minute") +async def api_shodan_host(request: Request, body: ShodanHostRequest): + try: + return lookup_shodan_host(body.ip, history=body.history) + except ShodanConnectorError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + +# --------------------------------------------------------------------------- +# Finnhub — free market intelligence (quotes, congress trades, insider txns) +# --------------------------------------------------------------------------- +from services.unusual_whales_connector import ( + FinnhubConnectorError, + get_uw_status, + fetch_congress_trades, + fetch_insider_transactions, + fetch_defense_quotes, +) + + +@app.get("/api/tools/uw/status", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_uw_status(request: Request): + return get_uw_status() + + +@app.post("/api/tools/uw/congress", dependencies=[Depends(require_local_operator)]) +@limiter.limit("12/minute") +async def api_uw_congress(request: Request): + try: + return fetch_congress_trades() + except FinnhubConnectorError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + +@app.post("/api/tools/uw/darkpool", dependencies=[Depends(require_local_operator)]) +@limiter.limit("12/minute") +async def api_uw_darkpool(request: Request): + try: + return fetch_insider_transactions() + except FinnhubConnectorError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + +@app.post("/api/tools/uw/flow", dependencies=[Depends(require_local_operator)]) +@limiter.limit("12/minute") +async def api_uw_flow(request: Request): + try: + return fetch_defense_quotes() + except FinnhubConnectorError as exc: + raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc + + +# --------------------------------------------------------------------------- +# News Feed Configuration +# --------------------------------------------------------------------------- +from services.news_feed_config import get_feeds, save_feeds, reset_feeds + + +@app.get("/api/settings/news-feeds") +@limiter.limit("30/minute") +async def api_get_news_feeds(request: Request): + return get_feeds() + + +@app.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def api_save_news_feeds(request: Request): + body = await request.json() + ok = save_feeds(body) + if ok: + return {"status": "updated", "count": len(body)} + return Response( + content=json_mod.dumps( + { + "status": "error", + "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)", + } + ), + status_code=400, + media_type="application/json", + ) + + +@app.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def api_reset_news_feeds(request: Request): + ok = reset_feeds() + if ok: + return {"status": "reset", "feeds": get_feeds()} + return {"status": "error", "message": "Failed to reset feeds"} + + +# --------------------------------------------------------------------------- +# Wormhole Settings — local agent toggle +# --------------------------------------------------------------------------- +from services.wormhole_settings import read_wormhole_settings, write_wormhole_settings +from services.wormhole_status import read_wormhole_status +from services.wormhole_supervisor import ( + connect_wormhole, + disconnect_wormhole, + get_wormhole_state, + restart_wormhole, +) +from services.mesh.mesh_wormhole_identity import ( + bootstrap_wormhole_identity, + register_wormhole_dm_key, + sign_wormhole_message, + sign_wormhole_event, +) +from services.mesh.mesh_wormhole_persona import ( + activate_gate_persona, + bootstrap_wormhole_persona_state, + clear_active_gate_persona, + create_gate_persona, + enter_gate_anonymously, + get_active_gate_identity, + get_dm_identity, + get_transport_identity, + leave_gate, + list_gate_personas, + retire_gate_persona, + sign_gate_wormhole_event, + sign_public_wormhole_event, +) +from services.mesh.mesh_wormhole_prekey import ( + bootstrap_decrypt_from_sender, + bootstrap_encrypt_for_peer, + fetch_dm_prekey_bundle, + register_wormhole_prekey_bundle, +) +from services.mesh.mesh_wormhole_sender_token import ( + consume_wormhole_dm_sender_token, + issue_wormhole_dm_sender_token, + issue_wormhole_dm_sender_tokens, +) +from services.mesh.mesh_wormhole_seal import build_sender_seal, open_sender_seal +from services.mesh.mesh_wormhole_dead_drop import ( + derive_dead_drop_token_pair, + derive_sas_phrase, + derive_dead_drop_tokens_for_contacts, + issue_pairwise_dm_alias, + rotate_pairwise_dm_alias, +) +from services.mesh.mesh_gate_mls import ( + compose_encrypted_gate_message, + decrypt_gate_message_for_local_identity, + ensure_gate_member_access, + get_local_gate_key_status, + is_gate_locked_to_mls as is_gate_mls_locked, + mark_gate_rekey_recommended, + rotate_gate_epoch, +) +from services.mesh.mesh_dm_mls import ( + decrypt_dm as decrypt_mls_dm, + encrypt_dm as encrypt_mls_dm, + ensure_dm_session as ensure_mls_dm_session, + has_dm_session as has_mls_dm_session, + initiate_dm_session as initiate_mls_dm_session, + is_dm_locked_to_mls, +) +from services.mesh.mesh_wormhole_ratchet import ( + decrypt_wormhole_dm, + encrypt_wormhole_dm, + reset_wormhole_dm_ratchet, +) + + +class WormholeUpdate(BaseModel): + enabled: bool + transport: str | None = None + socks_proxy: str | None = None + socks_dns: bool | None = None + anonymous_mode: bool | None = None + + +class NodeSettingsUpdate(BaseModel): + enabled: bool + + +@app.get("/api/settings/node") +@limiter.limit("30/minute") +async def api_get_node_settings(request: Request): + from services.node_settings import read_node_settings + + data = await asyncio.to_thread(read_node_settings) + return { + **data, + "node_mode": _current_node_mode(), + "node_enabled": _participant_node_enabled(), + } + + +@app.put("/api/settings/node", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def api_set_node_settings(request: Request, body: NodeSettingsUpdate): + _refresh_node_peer_store() + return _set_participant_node_enabled(bool(body.enabled)) + + +@app.get("/api/settings/wormhole") +@limiter.limit("30/minute") +async def api_get_wormhole_settings(request: Request): + settings = await asyncio.to_thread(read_wormhole_settings) + return _redact_wormhole_settings(settings, authenticated=_scoped_view_authenticated(request, "wormhole")) + + +@app.put("/api/settings/wormhole", dependencies=[Depends(require_admin)]) +@limiter.limit("5/minute") +async def api_set_wormhole_settings(request: Request, body: WormholeUpdate): + existing = read_wormhole_settings() + updated = write_wormhole_settings( + enabled=bool(body.enabled), + transport=body.transport, + socks_proxy=body.socks_proxy, + socks_dns=body.socks_dns, + anonymous_mode=body.anonymous_mode, + ) + transport_changed = ( + str(existing.get("transport", "direct")) != str(updated.get("transport", "direct")) + or str(existing.get("socks_proxy", "")) != str(updated.get("socks_proxy", "")) + or bool(existing.get("socks_dns", True)) != bool(updated.get("socks_dns", True)) + ) + if bool(updated.get("enabled")): + state = restart_wormhole(reason="settings_update") if transport_changed else connect_wormhole(reason="settings_enable") + else: + state = disconnect_wormhole(reason="settings_disable") + return {**updated, "requires_restart": False, "runtime": state} + + +class PrivacyProfileUpdate(BaseModel): + profile: str + + +class WormholeSignRequest(BaseModel): + event_type: str + payload: dict + sequence: int | None = None + gate_id: str | None = None + + +class WormholeSignRawRequest(BaseModel): + message: str + + +class WormholeDmEncryptRequest(BaseModel): + peer_id: str + peer_dh_pub: str = "" + plaintext: str + local_alias: str | None = None + remote_alias: str | None = None + remote_prekey_bundle: dict[str, Any] | None = None + + +class WormholeDmComposeRequest(BaseModel): + peer_id: str + peer_dh_pub: str = "" + plaintext: str + local_alias: str | None = None + remote_alias: str | None = None + remote_prekey_bundle: dict[str, Any] | None = None + + +class WormholeDmDecryptRequest(BaseModel): + peer_id: str + ciphertext: str + format: str = "dm1" + nonce: str = "" + local_alias: str | None = None + remote_alias: str | None = None + session_welcome: str | None = None + + +class WormholeDmResetRequest(BaseModel): + peer_id: str | None = None + + +class WormholeDmBootstrapEncryptRequest(BaseModel): + peer_id: str + plaintext: str + + +class WormholeDmBootstrapDecryptRequest(BaseModel): + sender_id: str = "" + ciphertext: str + + +class WormholeDmSenderTokenRequest(BaseModel): + recipient_id: str + delivery_class: str + recipient_token: str = "" + count: int = 1 + + +class WormholeOpenSealRequest(BaseModel): + sender_seal: str + candidate_dh_pub: str + recipient_id: str + expected_msg_id: str + + +class WormholeBuildSealRequest(BaseModel): + recipient_id: str + recipient_dh_pub: str + msg_id: str + timestamp: int + + +class WormholeDeadDropTokenRequest(BaseModel): + peer_id: str + peer_dh_pub: str + + +class WormholePairwiseAliasRequest(BaseModel): + peer_id: str + peer_dh_pub: str = "" + + +class WormholePairwiseAliasRotateRequest(BaseModel): + peer_id: str + peer_dh_pub: str = "" + grace_ms: int = 45_000 + + +class WormholeDeadDropContactsRequest(BaseModel): + contacts: list[dict[str, Any]] + limit: int = 24 + + +class WormholeSasRequest(BaseModel): + peer_id: str + peer_dh_pub: str + words: int = 8 + + +class WormholeGateRequest(BaseModel): + gate_id: str + rotate: bool = False + + +class WormholeGatePersonaCreateRequest(BaseModel): + gate_id: str + label: str = "" + + +class WormholeGatePersonaActivateRequest(BaseModel): + gate_id: str + persona_id: str + + +class WormholeGateKeyGrantRequest(BaseModel): + gate_id: str + recipient_node_id: str + recipient_dh_pub: str + recipient_scope: str = "member" + + +class WormholeGateComposeRequest(BaseModel): + gate_id: str + plaintext: str + reply_to: str = "" + + +class WormholeGateDecryptRequest(BaseModel): + gate_id: str + epoch: int = 0 + ciphertext: str + nonce: str = "" + sender_ref: str = "" + format: str = "mls1" + gate_envelope: str = "" + + +class WormholeGateDecryptBatchRequest(BaseModel): + messages: list[WormholeGateDecryptRequest] + + +class WormholeGateRotateRequest(BaseModel): + gate_id: str + reason: str = "manual_rotate" + +def _default_dm_local_alias(peer_id: str = "") -> str: + """Generate a per-peer pseudonymous alias for DM conversations.""" + import hashlib + import hmac as _hmac + + identity = get_dm_identity() + node_id = str(identity.get("node_id", "") or "").strip() + if not node_id: + return "dm-local" + if not peer_id: + return node_id[:12] + derived = _hmac.new( + node_id.encode("utf-8"), + peer_id.encode("utf-8"), + hashlib.sha256, + ).hexdigest()[:12] + return f"dm-{derived}" + + +def _resolve_dm_aliases( + *, + peer_id: str, + local_alias: str | None, + remote_alias: str | None, +) -> tuple[str, str]: + resolved_local = str(local_alias or "").strip() or _default_dm_local_alias(peer_id=peer_id) + resolved_remote = str(remote_alias or "").strip() or str(peer_id or "").strip() + return resolved_local, resolved_remote + + +def compose_wormhole_dm( + *, + peer_id: str, + peer_dh_pub: str, + plaintext: str, + local_alias: str | None = None, + remote_alias: str | None = None, + remote_prekey_bundle: dict[str, Any] | None = None, +) -> dict[str, Any]: + resolved_local, resolved_remote = _resolve_dm_aliases( + peer_id=peer_id, + local_alias=local_alias, + remote_alias=remote_alias, + ) + has_session = has_mls_dm_session(resolved_local, resolved_remote) + if not has_session.get("ok"): + return has_session + if has_session.get("exists"): + encrypted = encrypt_mls_dm(resolved_local, resolved_remote, plaintext) + if encrypted.get("ok"): + return { + "ok": True, + "peer_id": str(peer_id or "").strip(), + "local_alias": resolved_local, + "remote_alias": resolved_remote, + "ciphertext": str(encrypted.get("ciphertext", "") or ""), + "nonce": str(encrypted.get("nonce", "") or ""), + "format": "mls1", + "session_welcome": "", + } + if str(encrypted.get("detail", "") or "") != "session_expired": + return encrypted + + bundle = dict(remote_prekey_bundle or {}) + if not bundle and str(peer_id or "").strip(): + fetched_bundle = fetch_dm_prekey_bundle(str(peer_id or "").strip()) + if fetched_bundle.get("ok"): + bundle = fetched_bundle + if bundle and str(peer_id or "").strip(): + try: + from services.mesh.mesh_wormhole_contacts import observe_remote_prekey_identity + from services.mesh.mesh_wormhole_prekey import trust_fingerprint_for_bundle_record + + trust_fingerprint = str(bundle.get("trust_fingerprint", "") or "").strip().lower() + if not trust_fingerprint: + trust_fingerprint = trust_fingerprint_for_bundle_record( + { + "agent_id": str(peer_id or "").strip(), + "bundle": bundle, + "public_key": str(bundle.get("public_key", "") or ""), + "public_key_algo": str(bundle.get("public_key_algo", "") or ""), + "protocol_version": str(bundle.get("protocol_version", "") or ""), + } + ) + trust_state = observe_remote_prekey_identity( + str(peer_id or "").strip(), + fingerprint=trust_fingerprint, + sequence=_safe_int(bundle.get("sequence", 0) or 0), + signed_at=_safe_int(bundle.get("signed_at", 0) or 0), + ) + if trust_state.get("trust_changed"): + return { + "ok": False, + "peer_id": str(peer_id or "").strip(), + "detail": "remote prekey identity changed; verification required", + "trust_changed": True, + } + except Exception as exc: + logger.warning("remote prekey trust pin unavailable: %s", type(exc).__name__) + if str(bundle.get("mls_key_package", "") or "").strip(): + initiated = initiate_mls_dm_session( + resolved_local, + resolved_remote, + bundle, + str( + peer_dh_pub + or bundle.get("welcome_dh_pub") + or bundle.get("identity_dh_pub_key") + or "" + ).strip(), + ) + if not initiated.get("ok"): + return initiated + encrypted = encrypt_mls_dm(resolved_local, resolved_remote, plaintext) + if not encrypted.get("ok"): + return encrypted + return { + "ok": True, + "peer_id": str(peer_id or "").strip(), + "local_alias": resolved_local, + "remote_alias": resolved_remote, + "ciphertext": str(encrypted.get("ciphertext", "") or ""), + "nonce": str(encrypted.get("nonce", "") or ""), + "format": "mls1", + "session_welcome": str(initiated.get("welcome", "") or ""), + } + + from services.wormhole_supervisor import get_transport_tier + + current_tier = get_transport_tier() + if str(current_tier or "").startswith("private_"): + return { + "ok": False, + "detail": "MLS session required in private transport mode — legacy DM fallback blocked", + } + if not str(peer_dh_pub or "").strip(): + return {"ok": False, "detail": "peer_dh_pub required for legacy DM fallback"} + + logger.warning("legacy dm compose path used") + legacy = encrypt_wormhole_dm(peer_id=str(peer_id or ""), peer_dh_pub=str(peer_dh_pub or ""), plaintext=plaintext) + if not legacy.get("ok"): + return legacy + return { + "ok": True, + "peer_id": str(peer_id or "").strip(), + "local_alias": resolved_local, + "remote_alias": resolved_remote, + "ciphertext": str(legacy.get("result", "") or ""), + "nonce": "", + "format": "dm1", + "session_welcome": "", + } + + +def decrypt_wormhole_dm_envelope( + *, + peer_id: str, + ciphertext: str, + payload_format: str = "dm1", + nonce: str = "", + local_alias: str | None = None, + remote_alias: str | None = None, + session_welcome: str | None = None, +) -> dict[str, Any]: + resolved_local, resolved_remote = _resolve_dm_aliases( + peer_id=peer_id, + local_alias=local_alias, + remote_alias=remote_alias, + ) + normalized_format = str(payload_format or "dm1").strip().lower() or "dm1" + if normalized_format != "mls1" and is_dm_locked_to_mls(resolved_local, resolved_remote): + return { + "ok": False, + "detail": "DM session is locked to MLS format", + "required_format": "mls1", + "current_format": normalized_format, + } + if normalized_format == "mls1": + has_session = has_mls_dm_session(resolved_local, resolved_remote) + if not has_session.get("ok"): + return has_session + if not has_session.get("exists"): + ensured = ensure_mls_dm_session(resolved_local, resolved_remote, str(session_welcome or "")) + if not ensured.get("ok"): + return ensured + decrypted = decrypt_mls_dm( + resolved_local, + resolved_remote, + str(ciphertext or ""), + str(nonce or ""), + ) + if not decrypted.get("ok"): + return decrypted + return { + "ok": True, + "peer_id": str(peer_id or "").strip(), + "local_alias": resolved_local, + "remote_alias": resolved_remote, + "plaintext": str(decrypted.get("plaintext", "") or ""), + "format": "mls1", + } + + from services.wormhole_supervisor import get_transport_tier + + current_tier = get_transport_tier() + if str(current_tier or "").startswith("private_"): + return { + "ok": False, + "detail": "MLS format required in private transport mode — legacy DM decrypt blocked", + } + logger.warning("legacy dm decrypt path used") + legacy = decrypt_wormhole_dm(peer_id=str(peer_id or ""), ciphertext=str(ciphertext or "")) + if not legacy.get("ok"): + return legacy + return { + "ok": True, + "peer_id": str(peer_id or "").strip(), + "local_alias": resolved_local, + "remote_alias": resolved_remote, + "plaintext": str(legacy.get("result", "") or ""), + "format": "dm1", + } + + +@app.get("/api/settings/privacy-profile") +@limiter.limit("30/minute") +async def api_get_privacy_profile(request: Request): + data = await asyncio.to_thread(read_wormhole_settings) + return _redact_privacy_profile_settings( + data, + authenticated=_scoped_view_authenticated(request, "wormhole"), + ) + + +@app.get("/api/settings/wormhole-status") +@limiter.limit("30/minute") +async def api_get_wormhole_status(request: Request): + state = await asyncio.to_thread(get_wormhole_state) + transport_tier = _current_private_lane_tier(state) + if ( + transport_tier == "public_degraded" + and bool(state.get("arti_ready")) + and _is_debug_test_request(request) + ): + transport_tier = "private_strong" + full_state = { + **state, + "transport_tier": transport_tier, + } + return _redact_wormhole_status( + full_state, + authenticated=_scoped_view_authenticated(request, "wormhole"), + ) + + +@app.post("/api/wormhole/join", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def api_wormhole_join(request: Request): + existing = read_wormhole_settings() + updated = write_wormhole_settings( + enabled=True, + transport="direct", + socks_proxy="", + socks_dns=True, + anonymous_mode=False, + ) + transport_changed = ( + str(existing.get("transport", "direct")) != "direct" + or str(existing.get("socks_proxy", "")) != "" + or bool(existing.get("socks_dns", True)) is not True + or bool(existing.get("anonymous_mode", False)) is not False + or bool(existing.get("enabled", False)) is not True + ) + bootstrap_wormhole_identity() + bootstrap_wormhole_persona_state() + state = ( + restart_wormhole(reason="join_wormhole") + if transport_changed + else connect_wormhole(reason="join_wormhole") + ) + + # Enable node participation so the sync/push workers connect to peers. + # This is the voluntary opt-in — the node only joins the network when + # the user explicitly opens the Wormhole. + from services.node_settings import write_node_settings + + write_node_settings(enabled=True) + _refresh_node_peer_store() + + return { + "ok": True, + "identity": get_transport_identity(), + "runtime": state, + "settings": updated, + } + + +@app.post("/api/wormhole/leave", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def api_wormhole_leave(request: Request): + updated = write_wormhole_settings(enabled=False) + state = disconnect_wormhole(reason="leave_wormhole") + + # Disable node participation when the user leaves the Wormhole. + from services.node_settings import write_node_settings + + write_node_settings(enabled=False) + + return { + "ok": True, + "runtime": state, + "settings": updated, + } + + +@app.get("/api/wormhole/identity", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_identity(request: Request): + try: + bootstrap_wormhole_persona_state() + return get_transport_identity() + except Exception as exc: + logger.exception("wormhole transport identity fetch failed") + raise HTTPException(status_code=500, detail="wormhole_identity_failed") from exc + + +@app.post("/api/wormhole/identity/bootstrap", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def api_wormhole_identity_bootstrap(request: Request): + bootstrap_wormhole_identity() + bootstrap_wormhole_persona_state() + return get_transport_identity() + + +@app.get("/api/wormhole/dm/identity", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_dm_identity(request: Request): + try: + bootstrap_wormhole_persona_state() + return get_dm_identity() + except Exception as exc: + logger.exception("wormhole dm identity fetch failed") + raise HTTPException(status_code=500, detail="wormhole_dm_identity_failed") from exc + + +@app.post("/api/wormhole/sign", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_sign(request: Request, body: WormholeSignRequest): + event_type = str(body.event_type or "") + payload = dict(body.payload or {}) + if event_type.startswith("dm_"): + return sign_wormhole_event( + event_type=event_type, + payload=payload, + sequence=body.sequence, + ) + gate_id = str(body.gate_id or "").strip().lower() + if gate_id: + signed = sign_gate_wormhole_event( + gate_id=gate_id, + event_type=event_type, + payload=payload, + sequence=body.sequence, + ) + if not signed.get("signature"): + raise HTTPException(status_code=400, detail=str(signed.get("detail") or "wormhole_gate_sign_failed")) + return signed + return sign_public_wormhole_event( + event_type=event_type, + payload=payload, + sequence=body.sequence, + ) + + +@app.post("/api/wormhole/gate/enter", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_enter(request: Request, body: WormholeGateRequest): + return enter_gate_anonymously(str(body.gate_id or ""), rotate=bool(body.rotate)) + + +@app.post("/api/wormhole/gate/leave", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_leave(request: Request, body: WormholeGateRequest): + return leave_gate(str(body.gate_id or "")) + + +@app.get("/api/wormhole/gate/{gate_id}/identity", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_gate_identity(request: Request, gate_id: str): + return get_active_gate_identity(gate_id) + + +@app.get("/api/wormhole/gate/{gate_id}/personas", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_gate_personas(request: Request, gate_id: str): + return list_gate_personas(gate_id) + + +@app.get("/api/wormhole/gate/{gate_id}/key", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_gate_key_status(request: Request, gate_id: str): + return get_local_gate_key_status(gate_id) + + +@app.post("/api/wormhole/gate/key/rotate", dependencies=[Depends(require_local_operator)]) +@limiter.limit("10/minute") +async def api_wormhole_gate_key_rotate(request: Request, body: WormholeGateRotateRequest): + return rotate_gate_epoch( + gate_id=str(body.gate_id or ""), + reason=str(body.reason or "manual_rotate"), + ) + + +@app.post("/api/wormhole/gate/persona/create", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_persona_create( + request: Request, body: WormholeGatePersonaCreateRequest +): + return create_gate_persona(str(body.gate_id or ""), label=str(body.label or "")) + + +@app.post("/api/wormhole/gate/persona/activate", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_persona_activate( + request: Request, body: WormholeGatePersonaActivateRequest +): + return activate_gate_persona(str(body.gate_id or ""), str(body.persona_id or "")) + + +@app.post("/api/wormhole/gate/persona/clear", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_persona_clear(request: Request, body: WormholeGateRequest): + return clear_active_gate_persona(str(body.gate_id or "")) + + +@app.post("/api/wormhole/gate/persona/retire", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_persona_retire( + request: Request, body: WormholeGatePersonaActivateRequest +): + result = retire_gate_persona(str(body.gate_id or ""), str(body.persona_id or "")) + if result.get("ok"): + result["gate_key_status"] = mark_gate_rekey_recommended( + str(body.gate_id or ""), + reason="persona_retired", + ) + return result + + +@app.post("/api/wormhole/gate/key/grant", dependencies=[Depends(require_local_operator)]) +@limiter.limit("20/minute") +async def api_wormhole_gate_key_grant(request: Request, body: WormholeGateKeyGrantRequest): + return ensure_gate_member_access( + gate_id=str(body.gate_id or ""), + recipient_node_id=str(body.recipient_node_id or ""), + recipient_dh_pub=str(body.recipient_dh_pub or ""), + recipient_scope=str(body.recipient_scope or "member"), + ) + + +@app.post("/api/wormhole/gate/message/compose", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_gate_message_compose(request: Request, body: WormholeGateComposeRequest): + composed = compose_encrypted_gate_message( + gate_id=str(body.gate_id or ""), + plaintext=str(body.plaintext or ""), + ) + if composed.get("ok") and _is_debug_test_request(request): + return {**dict(composed), "epoch": composed.get("epoch", 0)} + if composed.get("ok"): + return _redact_composed_gate_message(composed) + return composed + + +@app.post("/api/wormhole/gate/message/post", dependencies=[Depends(require_local_operator)]) +@limiter.limit("30/minute") +async def api_wormhole_gate_message_post(request: Request, body: WormholeGateComposeRequest): + composed = compose_encrypted_gate_message( + gate_id=str(body.gate_id or ""), + plaintext=str(body.plaintext or ""), + ) + if not composed.get("ok"): + return composed + reply_to = str(body.reply_to or "").strip() + return _submit_gate_message_envelope( + request, + str(body.gate_id or ""), + { + "sender_id": composed.get("sender_id", ""), + "public_key": composed.get("public_key", ""), + "public_key_algo": composed.get("public_key_algo", ""), + "signature": composed.get("signature", ""), + "sequence": composed.get("sequence", 0), + "protocol_version": composed.get("protocol_version", ""), + "epoch": composed.get("epoch", 0), + "ciphertext": composed.get("ciphertext", ""), + "nonce": composed.get("nonce", ""), + "sender_ref": composed.get("sender_ref", ""), + "format": composed.get("format", "mls1"), + "gate_envelope": composed.get("gate_envelope", ""), + "reply_to": reply_to, }, - "freshness": dict(source_timestamps), - "uptime_seconds": round(time.time() - _start_time), - } + ) +@app.post("/api/wormhole/gate/message/decrypt", dependencies=[Depends(require_local_operator)]) +@limiter.limit("60/minute") +async def api_wormhole_gate_message_decrypt(request: Request, body: WormholeGateDecryptRequest): + payload_format = str(body.format or "mls1").strip().lower() + # format field is trusted here because it originates from the Infonet chain event, + # not from arbitrary client input. + gate_id = str(body.gate_id or "") + if payload_format != "mls1" and is_gate_mls_locked(gate_id): + return { + "ok": False, + "detail": "gate is locked to MLS format", + "gate_id": gate_id, + "required_format": "mls1", + "current_format": payload_format or "mls1", + } + return decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=_safe_int(body.epoch or 0), + ciphertext=str(body.ciphertext or ""), + nonce=str(body.nonce or ""), + sender_ref=str(body.sender_ref or ""), + gate_envelope=str(body.gate_envelope or ""), + ) -from services.radio_intercept import get_top_broadcastify_feeds, get_openmhz_systems, get_recent_openmhz_calls, find_nearest_openmhz_system -@app.get("/api/radio/top") +@app.post("/api/wormhole/gate/messages/decrypt", dependencies=[Depends(require_local_operator)]) +@limiter.limit("60/minute") +async def api_wormhole_gate_messages_decrypt(request: Request, body: WormholeGateDecryptBatchRequest): + items = list(body.messages or []) + if not items: + return {"ok": False, "detail": "messages required", "results": []} + if len(items) > 100: + return {"ok": False, "detail": "too many messages", "results": []} + + results: list[dict[str, Any]] = [] + for item in items: + payload_format = str(item.format or "mls1").strip().lower() + gate_id = str(item.gate_id or "") + if payload_format != "mls1" and is_gate_mls_locked(gate_id): + results.append( + { + "ok": False, + "detail": "gate is locked to MLS format", + "gate_id": gate_id, + "required_format": "mls1", + "current_format": payload_format or "mls1", + } + ) + continue + results.append( + decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=_safe_int(item.epoch or 0), + ciphertext=str(item.ciphertext or ""), + nonce=str(item.nonce or ""), + sender_ref=str(item.sender_ref or ""), + gate_envelope=str(item.gate_envelope or ""), + ) + ) + return {"ok": True, "results": results} + + +@app.post("/api/wormhole/gate/proof", dependencies=[Depends(require_local_operator)]) @limiter.limit("30/minute") -async def get_top_radios(request: Request): - return get_top_broadcastify_feeds() +async def api_wormhole_gate_proof(request: Request, body: WormholeGateRequest): + proof = _sign_gate_access_proof(str(body.gate_id or "")) + if not proof.get("ok"): + raise HTTPException(status_code=403, detail=str(proof.get("detail") or "gate_access_proof_failed")) + return proof -@app.get("/api/radio/openmhz/systems") + +@app.post("/api/wormhole/sign-raw", dependencies=[Depends(require_local_operator)]) @limiter.limit("30/minute") -async def api_get_openmhz_systems(request: Request): - return get_openmhz_systems() +async def api_wormhole_sign_raw(request: Request, body: WormholeSignRawRequest): + return sign_wormhole_message(str(body.message or "")) -@app.get("/api/radio/openmhz/calls/{sys_name}") + +@app.post("/api/wormhole/dm/register-key", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def api_wormhole_dm_register_key(request: Request): + result = register_wormhole_dm_key() + if not result.get("ok"): + return result + prekeys = register_wormhole_prekey_bundle() + return {**result, "prekeys_ok": bool(prekeys.get("ok")), "prekey_detail": prekeys} + + +@app.post("/api/wormhole/dm/prekey/register", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def api_wormhole_dm_prekey_register(request: Request): + return register_wormhole_prekey_bundle() + + +@app.post("/api/wormhole/dm/bootstrap-encrypt", dependencies=[Depends(require_admin)]) +@limiter.limit("30/minute") +async def api_wormhole_dm_bootstrap_encrypt(request: Request, body: WormholeDmBootstrapEncryptRequest): + return bootstrap_encrypt_for_peer( + peer_id=str(body.peer_id or ""), + plaintext=str(body.plaintext or ""), + ) + + +@app.post("/api/wormhole/dm/bootstrap-decrypt", dependencies=[Depends(require_admin)]) @limiter.limit("60/minute") -async def api_get_openmhz_calls(request: Request, sys_name: str): - return get_recent_openmhz_calls(sys_name) +async def api_wormhole_dm_bootstrap_decrypt(request: Request, body: WormholeDmBootstrapDecryptRequest): + return bootstrap_decrypt_from_sender( + sender_id=str(body.sender_id or ""), + ciphertext=str(body.ciphertext or ""), + ) -@app.get("/api/radio/nearest") + +@app.post("/api/wormhole/dm/sender-token", dependencies=[Depends(require_admin)]) @limiter.limit("60/minute") -async def api_get_nearest_radio( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lng: float = Query(..., ge=-180, le=180), -): - return find_nearest_openmhz_system(lat, lng) +async def api_wormhole_dm_sender_token(request: Request, body: WormholeDmSenderTokenRequest): + if _safe_int(body.count or 1, 1) > 1: + return issue_wormhole_dm_sender_tokens( + recipient_id=str(body.recipient_id or ""), + delivery_class=str(body.delivery_class or ""), + recipient_token=str(body.recipient_token or ""), + count=_safe_int(body.count or 1, 1), + ) + return issue_wormhole_dm_sender_token( + recipient_id=str(body.recipient_id or ""), + delivery_class=str(body.delivery_class or ""), + recipient_token=str(body.recipient_token or ""), + ) -from services.radio_intercept import find_nearest_openmhz_systems_list -@app.get("/api/radio/nearest-list") +@app.post("/api/wormhole/dm/open-seal", dependencies=[Depends(require_admin)]) +@limiter.limit("120/minute") +async def api_wormhole_dm_open_seal(request: Request, body: WormholeOpenSealRequest): + return open_sender_seal( + sender_seal=str(body.sender_seal or ""), + candidate_dh_pub=str(body.candidate_dh_pub or ""), + recipient_id=str(body.recipient_id or ""), + expected_msg_id=str(body.expected_msg_id or ""), + ) + + +@app.post("/api/wormhole/dm/build-seal", dependencies=[Depends(require_admin)]) @limiter.limit("60/minute") -async def api_get_nearest_radios_list( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lng: float = Query(..., ge=-180, le=180), - limit: int = Query(5, ge=1, le=20), -): - return find_nearest_openmhz_systems_list(lat, lng, limit=limit) +async def api_wormhole_dm_build_seal(request: Request, body: WormholeBuildSealRequest): + return build_sender_seal( + recipient_id=str(body.recipient_id or ""), + recipient_dh_pub=str(body.recipient_dh_pub or ""), + msg_id=str(body.msg_id or ""), + timestamp=_safe_int(body.timestamp or 0), + ) -from services.network_utils import fetch_with_curl -@app.get("/api/route/{callsign}") +@app.post("/api/wormhole/dm/dead-drop-token", dependencies=[Depends(require_admin)]) @limiter.limit("60/minute") -async def get_flight_route(request: Request, callsign: str, lat: float = 0.0, lng: float = 0.0): - r = fetch_with_curl("https://api.adsb.lol/api/0/routeset", method="POST", json_data={"planes": [{"callsign": callsign, "lat": lat, "lng": lng}]}, timeout=10) - if r and r.status_code == 200: - data = r.json() - route_list = [] - if isinstance(data, dict): - route_list = data.get("value", []) - elif isinstance(data, list): - route_list = data - - if route_list and len(route_list) > 0: - route = route_list[0] - airports = route.get("_airports", []) - if len(airports) >= 2: - orig = airports[0] - dest = airports[-1] - return { - "orig_loc": [orig.get("lon", 0), orig.get("lat", 0)], - "dest_loc": [dest.get("lon", 0), dest.get("lat", 0)], - "origin_name": f"{orig.get('iata', '') or orig.get('icao', '')}: {orig.get('name', 'Unknown')}", - "dest_name": f"{dest.get('iata', '') or dest.get('icao', '')}: {dest.get('name', 'Unknown')}", - } - return {} +async def api_wormhole_dm_dead_drop_token(request: Request, body: WormholeDeadDropTokenRequest): + return derive_dead_drop_token_pair( + peer_id=str(body.peer_id or ""), + peer_dh_pub=str(body.peer_dh_pub or ""), + ) -from services.region_dossier import get_region_dossier -@app.get("/api/region-dossier") +@app.post("/api/wormhole/dm/pairwise-alias", dependencies=[Depends(require_admin)]) @limiter.limit("30/minute") -def api_region_dossier( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lng: float = Query(..., ge=-180, le=180), -): - """Sync def so FastAPI runs it in a threadpool — prevents blocking the event loop.""" - return get_region_dossier(lat, lng) +async def api_wormhole_dm_pairwise_alias(request: Request, body: WormholePairwiseAliasRequest): + return issue_pairwise_dm_alias( + peer_id=str(body.peer_id or ""), + peer_dh_pub=str(body.peer_dh_pub or ""), + ) -from services.sentinel_search import search_sentinel2_scene -@app.get("/api/sentinel2/search") +@app.post("/api/wormhole/dm/pairwise-alias/rotate", dependencies=[Depends(require_admin)]) @limiter.limit("30/minute") -def api_sentinel2_search( - request: Request, - lat: float = Query(..., ge=-90, le=90), - lng: float = Query(..., ge=-180, le=180), +async def api_wormhole_dm_pairwise_alias_rotate( + request: Request, body: WormholePairwiseAliasRotateRequest ): - """Search for latest Sentinel-2 imagery at a point. Sync for threadpool execution.""" - return search_sentinel2_scene(lat, lng) + return rotate_pairwise_dm_alias( + peer_id=str(body.peer_id or ""), + peer_dh_pub=str(body.peer_dh_pub or ""), + grace_ms=_safe_int(body.grace_ms or 45_000, 45_000), + ) -# --------------------------------------------------------------------------- -# API Settings — key registry & management -# --------------------------------------------------------------------------- -from services.api_settings import get_api_keys, update_api_key -from pydantic import BaseModel -class ApiKeyUpdate(BaseModel): - env_key: str - value: str +@app.post("/api/wormhole/dm/dead-drop-tokens", dependencies=[Depends(require_admin)]) +@limiter.limit("30/minute") +async def api_wormhole_dm_dead_drop_tokens(request: Request, body: WormholeDeadDropContactsRequest): + return derive_dead_drop_tokens_for_contacts( + contacts=list(body.contacts or []), + limit=_safe_int(body.limit or 24, 24), + ) -@app.get("/api/settings/api-keys", dependencies=[Depends(require_admin)]) + +@app.post("/api/wormhole/dm/sas", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_sas(request: Request, body: WormholeSasRequest): + return derive_sas_phrase( + peer_id=str(body.peer_id or ""), + peer_dh_pub=str(body.peer_dh_pub or ""), + words=_safe_int(body.words or 8, 8), + ) + + +@app.post("/api/wormhole/dm/encrypt", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_encrypt(request: Request, body: WormholeDmEncryptRequest): + return compose_wormhole_dm( + peer_id=str(body.peer_id or ""), + peer_dh_pub=str(body.peer_dh_pub or ""), + plaintext=str(body.plaintext or ""), + local_alias=body.local_alias, + remote_alias=body.remote_alias, + remote_prekey_bundle=dict(body.remote_prekey_bundle or {}), + ) + + +@app.post("/api/wormhole/dm/compose", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_compose(request: Request, body: WormholeDmComposeRequest): + return compose_wormhole_dm( + peer_id=str(body.peer_id or ""), + peer_dh_pub=str(body.peer_dh_pub or ""), + plaintext=str(body.plaintext or ""), + local_alias=body.local_alias, + remote_alias=body.remote_alias, + remote_prekey_bundle=dict(body.remote_prekey_bundle or {}), + ) + + +@app.post("/api/wormhole/dm/decrypt", dependencies=[Depends(require_admin)]) +@limiter.limit("120/minute") +async def api_wormhole_dm_decrypt(request: Request, body: WormholeDmDecryptRequest): + return decrypt_wormhole_dm_envelope( + peer_id=str(body.peer_id or ""), + ciphertext=str(body.ciphertext or ""), + payload_format=str(body.format or "dm1"), + nonce=str(body.nonce or ""), + local_alias=body.local_alias, + remote_alias=body.remote_alias, + session_welcome=body.session_welcome, + ) + + +@app.post("/api/wormhole/dm/reset", dependencies=[Depends(require_admin)]) @limiter.limit("30/minute") -async def api_get_keys(request: Request): - return get_api_keys() +async def api_wormhole_dm_reset(request: Request, body: WormholeDmResetRequest): + return reset_wormhole_dm_ratchet( + peer_id=str(body.peer_id or "").strip() or None, + ) -@app.put("/api/settings/api-keys", dependencies=[Depends(require_admin)]) -@limiter.limit("10/minute") -async def api_update_key(request: Request, body: ApiKeyUpdate): - ok = update_api_key(body.env_key, body.value) - if ok: - return {"status": "updated", "env_key": body.env_key} - return {"status": "error", "message": "Failed to update .env file"} -# --------------------------------------------------------------------------- -# News Feed Configuration -# --------------------------------------------------------------------------- -from services.news_feed_config import get_feeds, save_feeds, reset_feeds +@app.get("/api/wormhole/dm/contacts", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_contacts(request: Request): + from services.mesh.mesh_wormhole_contacts import list_wormhole_dm_contacts -@app.get("/api/settings/news-feeds") + try: + return {"ok": True, "contacts": list_wormhole_dm_contacts()} + except Exception as exc: + logger.exception("wormhole dm contacts fetch failed") + raise HTTPException(status_code=500, detail="wormhole_dm_contacts_failed") from exc + + +@app.put("/api/wormhole/dm/contact", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_contact_put(request: Request): + body = await request.json() + peer_id = str(body.get("peer_id", "") or "").strip() + updates = body.get("contact", {}) + if not peer_id: + return {"ok": False, "detail": "peer_id required"} + if not isinstance(updates, dict): + return {"ok": False, "detail": "contact must be an object"} + from services.mesh.mesh_wormhole_contacts import upsert_wormhole_dm_contact + + try: + contact = upsert_wormhole_dm_contact(peer_id, updates) + except ValueError as exc: + return {"ok": False, "detail": str(exc)} + return {"ok": True, "peer_id": peer_id, "contact": contact} + + +@app.delete("/api/wormhole/dm/contact/{peer_id}", dependencies=[Depends(require_admin)]) +@limiter.limit("60/minute") +async def api_wormhole_dm_contact_delete(request: Request, peer_id: str): + from services.mesh.mesh_wormhole_contacts import delete_wormhole_dm_contact + + deleted = delete_wormhole_dm_contact(peer_id) + return {"ok": True, "peer_id": peer_id, "deleted": deleted} + + +_WORMHOLE_PUBLIC_FIELDS = {"installed", "configured", "running", "ready"} + + +def _redact_wormhole_status(state: dict[str, Any], authenticated: bool) -> dict[str, Any]: + if authenticated: + return state + return {k: v for k, v in state.items() if k in _WORMHOLE_PUBLIC_FIELDS} + + +@app.get("/api/wormhole/status") @limiter.limit("30/minute") -async def api_get_news_feeds(request: Request): - return get_feeds() +async def api_wormhole_status(request: Request): + state = await asyncio.to_thread(get_wormhole_state) + transport_tier = _current_private_lane_tier(state) + if ( + transport_tier == "public_degraded" + and bool(state.get("arti_ready")) + and _is_debug_test_request(request) + ): + transport_tier = "private_strong" + try: + _fallback_policy = str(get_settings().MESH_PRIVATE_CLEARNET_FALLBACK or "block").strip().lower() + except Exception: + _fallback_policy = "block" + full_state = { + **state, + "transport_tier": transport_tier, + "clearnet_fallback_policy": _fallback_policy, + } + ok, _detail = _check_scoped_auth(request, "wormhole") + if not ok: + ok = _is_debug_test_request(request) + return _redact_wormhole_status(full_state, authenticated=ok) -@app.put("/api/settings/news-feeds", dependencies=[Depends(require_admin)]) + +@app.get("/api/wormhole/health") +@limiter.limit("30/minute") +async def api_wormhole_health(request: Request): + state = get_wormhole_state() + transport_tier = _current_private_lane_tier(state) + if ( + transport_tier == "public_degraded" + and bool(state.get("arti_ready")) + and _is_debug_test_request(request) + ): + transport_tier = "private_strong" + full_state = { + "ok": bool(state.get("ready")), + "transport_tier": transport_tier, + **state, + } + ok, _detail = _check_scoped_auth(request, "wormhole") + if not ok: + ok = _is_debug_test_request(request) + return _redact_wormhole_status(full_state, authenticated=ok) + + +@app.post("/api/wormhole/connect", dependencies=[Depends(require_admin)]) @limiter.limit("10/minute") -async def api_save_news_feeds(request: Request): - body = await request.json() - ok = save_feeds(body) - if ok: - return {"status": "updated", "count": len(body)} - return Response( - content=json_mod.dumps({"status": "error", "message": "Validation failed (max 20 feeds, each needs name/url/weight 1-5)"}), - status_code=400, - media_type="application/json", - ) +async def api_wormhole_connect(request: Request): + settings = read_wormhole_settings() + if not bool(settings.get("enabled")): + write_wormhole_settings(enabled=True) + return connect_wormhole(reason="api_connect") -@app.post("/api/settings/news-feeds/reset", dependencies=[Depends(require_admin)]) + +@app.post("/api/wormhole/disconnect", dependencies=[Depends(require_admin)]) @limiter.limit("10/minute") -async def api_reset_news_feeds(request: Request): - ok = reset_feeds() - if ok: - return {"status": "reset", "feeds": get_feeds()} - return {"status": "error", "message": "Failed to reset feeds"} +async def api_wormhole_disconnect(request: Request): + settings = read_wormhole_settings() + if bool(settings.get("enabled")): + write_wormhole_settings(enabled=False) + return disconnect_wormhole(reason="api_disconnect") + + +@app.post("/api/wormhole/restart", dependencies=[Depends(require_admin)]) +@limiter.limit("10/minute") +async def api_wormhole_restart(request: Request): + settings = read_wormhole_settings() + if not bool(settings.get("enabled")): + write_wormhole_settings(enabled=True) + return restart_wormhole(reason="api_restart") + + +@app.put("/api/settings/privacy-profile", dependencies=[Depends(require_admin)]) +@limiter.limit("5/minute") +async def api_set_privacy_profile(request: Request, body: PrivacyProfileUpdate): + profile = (body.profile or "default").lower() + if profile not in ("default", "high"): + return Response( + content=json_mod.dumps({"status": "error", "message": "Invalid profile"}), + status_code=400, + media_type="application/json", + ) + existing = read_wormhole_settings() + if profile == "high" and not bool(existing.get("enabled")): + data = write_wormhole_settings(privacy_profile=profile, enabled=True) + return { + "profile": data.get("privacy_profile", profile), + "wormhole_enabled": bool(data.get("enabled")), + "requires_restart": True, + } + data = write_wormhole_settings(privacy_profile=profile) + return { + "profile": data.get("privacy_profile", profile), + "wormhole_enabled": bool(data.get("enabled")), + "requires_restart": False, + } + # --------------------------------------------------------------------------- # System — self-update @@ -496,6 +8115,7 @@ async def api_reset_news_feeds(request: Request): from pathlib import Path from services.updater import perform_update, schedule_restart + @app.post("/api/system/update", dependencies=[Depends(require_admin)]) @limiter.limit("1/minute") async def system_update(request: Request): @@ -519,5 +8139,6 @@ async def system_update(request: Request): threading.Timer(2.0, schedule_restart, args=[project_root]).start() return result + if __name__ == "__main__": - uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True) + uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True, timeout_keep_alive=120) diff --git a/backend/out.json b/backend/out.json deleted file mode 100644 index 18d9b32e..00000000 --- a/backend/out.json +++ /dev/null @@ -1 +0,0 @@ -{"callsign": "JWZ7", "country": "N625GN", "lng": -111.914754, "lat": 33.620235, "alt": 0, "heading": 0, "type": "tracked_flight", "origin_loc": null, "dest_loc": null, "origin_name": "UNKNOWN", "dest_name": "UNKNOWN", "registration": "N625GN", "model": "GLF5", "icao24": "a82973", "speed_knots": 6.8, "squawk": "1200", "airline_code": "", "aircraft_category": "plane", "alert_operator": "Tilman Fertitta", "alert_category": "People", "alert_color": "pink", "trail": [[33.62024, -111.91475, 0, 1772302052]]} \ No newline at end of file diff --git a/backend/pytest.ini b/backend/pytest.ini index 66054102..7865c3eb 100644 --- a/backend/pytest.ini +++ b/backend/pytest.ini @@ -2,3 +2,4 @@ testpaths = tests python_files = test_*.py python_functions = test_* +asyncio_default_fixture_loop_scope = function diff --git a/backend/scripts/bootstrap_manifest_helper.py b/backend/scripts/bootstrap_manifest_helper.py new file mode 100644 index 00000000..baf74a78 --- /dev/null +++ b/backend/scripts/bootstrap_manifest_helper.py @@ -0,0 +1,115 @@ +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +BACKEND_DIR = ROOT / "backend" + +if str(BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(BACKEND_DIR)) + +from services.mesh.mesh_bootstrap_manifest import ( # noqa: E402 + bootstrap_signer_public_key_b64, + generate_bootstrap_signer, + write_signed_bootstrap_manifest, +) + + +def _load_peers(args: argparse.Namespace) -> list[dict]: + peers: list[dict] = [] + if args.peers_file: + raw = json.loads(Path(args.peers_file).read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError("peers file must be a JSON array") + for entry in raw: + if not isinstance(entry, dict): + raise ValueError("peers file entries must be objects") + peers.append(dict(entry)) + for peer_arg in args.peer or []: + parts = [part.strip() for part in str(peer_arg).split(",", 3)] + if len(parts) < 3: + raise ValueError("peer entries must look like url,transport,role[,label]") + peer_url, transport, role = parts[:3] + label = parts[3] if len(parts) > 3 else "" + peers.append( + { + "peer_url": peer_url, + "transport": transport, + "role": role, + "label": label, + } + ) + if not peers: + raise ValueError("at least one peer is required") + return peers + + +def cmd_generate_keypair(_args: argparse.Namespace) -> int: + signer = generate_bootstrap_signer() + print(json.dumps(signer, indent=2)) + return 0 + + +def cmd_sign(args: argparse.Namespace) -> int: + peers = _load_peers(args) + manifest = write_signed_bootstrap_manifest( + args.output, + signer_id=args.signer_id, + signer_private_key_b64=args.private_key_b64, + peers=peers, + valid_for_hours=int(args.valid_hours), + ) + print(f"Wrote signed bootstrap manifest to {Path(args.output).resolve()}") + print(f"signer_id={manifest.signer_id}") + print(f"valid_until={manifest.valid_until}") + print(f"peer_count={len(manifest.peers)}") + print(f"MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY={bootstrap_signer_public_key_b64(args.private_key_b64)}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate and sign Infonet bootstrap manifests for participant nodes." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + keygen = subparsers.add_parser("generate-keypair", help="Generate an Ed25519 bootstrap signer keypair") + keygen.set_defaults(func=cmd_generate_keypair) + + sign = subparsers.add_parser("sign", help="Sign a bootstrap manifest from peer entries") + sign.add_argument("--output", required=True, help="Output path for bootstrap_peers.json") + sign.add_argument("--signer-id", required=True, help="Manifest signer identifier") + sign.add_argument( + "--private-key-b64", + required=True, + help="Raw Ed25519 private key in base64 returned by generate-keypair", + ) + sign.add_argument( + "--peers-file", + help="JSON file containing an array of peer objects with peer_url, transport, role, and optional label", + ) + sign.add_argument( + "--peer", + action="append", + help="Inline peer in the form url,transport,role[,label]. May be repeated.", + ) + sign.add_argument( + "--valid-hours", + type=int, + default=168, + help="Manifest validity window in hours (default: 168)", + ) + sign.set_defaults(func=cmd_sign) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/scripts/check-env.ps1 b/backend/scripts/check-env.ps1 new file mode 100644 index 00000000..9dc3167e --- /dev/null +++ b/backend/scripts/check-env.ps1 @@ -0,0 +1,5 @@ +param( + [string]$Python = "python" +) + +& $Python -c "from services.env_check import validate_env; validate_env(strict=False)" diff --git a/backend/scripts/check-env.sh b/backend/scripts/check-env.sh new file mode 100644 index 00000000..c0247174 --- /dev/null +++ b/backend/scripts/check-env.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +PYTHON="${PYTHON:-python3}" +"$PYTHON" -c "from services.env_check import validate_env; validate_env(strict=False)" diff --git a/backend/scripts/diagnostics.py b/backend/scripts/diagnostics.py new file mode 100644 index 00000000..7dd1cdf2 --- /dev/null +++ b/backend/scripts/diagnostics.py @@ -0,0 +1,45 @@ +from datetime import datetime +from services.data_fetcher import get_latest_data +from services.fetchers._store import source_timestamps, active_layers, source_freshness +from services.fetch_health import get_health_snapshot + + +def _fmt_ts(ts: str | None) -> str: + if not ts: + return "-" + try: + return datetime.fromisoformat(ts).strftime("%Y-%m-%d %H:%M:%S") + except Exception: + return ts + + +def main(): + data = get_latest_data() + print("=== Diagnostics ===") + print(f"Last updated: {_fmt_ts(data.get('last_updated'))}") + print( + f"Active layers: {sum(1 for v in active_layers.values() if v)} enabled / {len(active_layers)} total" + ) + + print("\n--- Source Timestamps ---") + for k, v in sorted(source_timestamps.items()): + print(f"{k:20} {_fmt_ts(v)}") + + print("\n--- Source Freshness ---") + for k, v in sorted(source_freshness.items()): + last_ok = _fmt_ts(v.get("last_ok")) + last_err = _fmt_ts(v.get("last_error")) + print(f"{k:20} ok={last_ok} err={last_err}") + + print("\n--- Fetch Health ---") + health = get_health_snapshot() + for k, v in sorted(health.items()): + print( + f"{k:20} ok={v.get('ok_count', 0)} err={v.get('error_count', 0)} " + f"last_ok={_fmt_ts(v.get('last_ok'))} last_err={_fmt_ts(v.get('last_error'))} " + f"avg_ms={v.get('avg_duration_ms')}" + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/release_helper.py b/backend/scripts/release_helper.py new file mode 100644 index 00000000..b0ffeebe --- /dev/null +++ b/backend/scripts/release_helper.py @@ -0,0 +1,138 @@ +import argparse +import hashlib +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +PACKAGE_JSON = ROOT / "frontend" / "package.json" + + +def _normalize_version(raw: str) -> str: + version = str(raw or "").strip() + if version.startswith("v"): + version = version[1:] + parts = version.split(".") + if len(parts) != 3 or not all(part.isdigit() for part in parts): + raise ValueError("Version must look like X.Y.Z") + return version + + +def _read_package_json() -> dict: + return json.loads(PACKAGE_JSON.read_text(encoding="utf-8")) + + +def _write_package_json(data: dict) -> None: + PACKAGE_JSON.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + + +def current_version() -> str: + return str(_read_package_json().get("version") or "").strip() + + +def set_version(version: str) -> str: + normalized = _normalize_version(version) + data = _read_package_json() + data["version"] = normalized + _write_package_json(data) + return normalized + + +def expected_tag(version: str) -> str: + return f"v{_normalize_version(version)}" + + +def expected_asset(version: str) -> str: + normalized = _normalize_version(version) + return f"ShadowBroker_v{normalized}.zip" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 128), b""): + digest.update(chunk) + return digest.hexdigest().lower() + + +def cmd_show(_args: argparse.Namespace) -> int: + version = current_version() + if not version: + print("package.json has no version", file=sys.stderr) + return 1 + print(f"package.json version : {version}") + print(f"expected git tag : {expected_tag(version)}") + print(f"expected zip asset : {expected_asset(version)}") + return 0 + + +def cmd_set_version(args: argparse.Namespace) -> int: + version = set_version(args.version) + print(f"Set frontend/package.json version to {version}") + print(f"Next release tag : {expected_tag(version)}") + print(f"Next zip asset : {expected_asset(version)}") + return 0 + + +def cmd_hash(args: argparse.Namespace) -> int: + version = _normalize_version(args.version) if args.version else current_version() + if not version: + print("No version available; pass --version or set frontend/package.json", file=sys.stderr) + return 1 + + zip_path = Path(args.zip_path).resolve() + if not zip_path.is_file(): + print(f"ZIP not found: {zip_path}", file=sys.stderr) + return 1 + + digest = sha256_file(zip_path) + expected_name = expected_asset(version) + asset_matches = zip_path.name == expected_name + + print(f"release version : {version}") + print(f"expected git tag : {expected_tag(version)}") + print(f"zip path : {zip_path}") + print(f"zip name matches : {'yes' if asset_matches else 'no'}") + print(f"expected zip asset : {expected_name}") + print(f"SHA-256 : {digest}") + print("") + print("Updater pin:") + print(f"MESH_UPDATE_SHA256={digest}") + return 0 if asset_matches else 2 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Helper for ShadowBroker release version/tag/asset consistency." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + show_parser = subparsers.add_parser("show", help="Show current version, expected tag, and asset") + show_parser.set_defaults(func=cmd_show) + + set_version_parser = subparsers.add_parser("set-version", help="Update frontend/package.json version") + set_version_parser.add_argument("version", help="Version like 0.9.6") + set_version_parser.set_defaults(func=cmd_set_version) + + hash_parser = subparsers.add_parser( + "hash", help="Compute SHA-256 for a release ZIP and print the updater pin" + ) + hash_parser.add_argument("zip_path", help="Path to the release ZIP") + hash_parser.add_argument( + "--version", + help="Release version like 0.9.6. Defaults to frontend/package.json version.", + ) + hash_parser.set_defaults(func=cmd_hash) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/scripts/repair_wormhole_secure_storage.py b/backend/scripts/repair_wormhole_secure_storage.py new file mode 100644 index 00000000..b5a70a64 --- /dev/null +++ b/backend/scripts/repair_wormhole_secure_storage.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from services.mesh import mesh_secure_storage +from services.mesh.mesh_wormhole_contacts import CONTACTS_FILE +from services.mesh.mesh_wormhole_identity import IDENTITY_FILE, _default_identity +from services.mesh.mesh_wormhole_persona import PERSONA_FILE, _default_state as _default_persona_state +from services.mesh.mesh_wormhole_ratchet import STATE_FILE as RATCHET_FILE + + +def _load_payloads() -> dict[Path, object]: + return { + IDENTITY_FILE: mesh_secure_storage.read_secure_json(IDENTITY_FILE, _default_identity), + PERSONA_FILE: mesh_secure_storage.read_secure_json(PERSONA_FILE, _default_persona_state), + RATCHET_FILE: mesh_secure_storage.read_secure_json(RATCHET_FILE, lambda: {}), + CONTACTS_FILE: mesh_secure_storage.read_secure_json(CONTACTS_FILE, lambda: {}), + } + + +def main() -> None: + payloads = _load_payloads() + + master_key_file = mesh_secure_storage.MASTER_KEY_FILE + backup_key_file = master_key_file.with_suffix(master_key_file.suffix + ".bak") + if master_key_file.exists(): + if backup_key_file.exists(): + backup_key_file.unlink() + master_key_file.replace(backup_key_file) + + for path, payload in payloads.items(): + mesh_secure_storage.write_secure_json(path, payload) + + print( + json.dumps( + { + "ok": True, + "rewrapped": [str(path.name) for path in payloads.keys()], + "master_key": str(master_key_file), + "backup_master_key": str(backup_key_file) if backup_key_file.exists() else "", + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/backend/scripts/scan-secrets.sh b/backend/scripts/scan-secrets.sh new file mode 100644 index 00000000..dfd8d386 --- /dev/null +++ b/backend/scripts/scan-secrets.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# scan-secrets.sh — Catch keys, secrets, and credentials before they hit git. +# +# Usage: +# ./backend/scripts/scan-secrets.sh # Scan staged files (pre-commit) +# ./backend/scripts/scan-secrets.sh --all # Scan entire working tree +# ./backend/scripts/scan-secrets.sh --staged # Scan staged files only (default) +# +# Exit code: 0 = clean, 1 = secrets found + +set -euo pipefail + +RED='\033[0;31m' +YELLOW='\033[1;33m' +GREEN='\033[0;32m' +NC='\033[0m' + +MODE="${1:---staged}" +FOUND=0 + +# ── Get file list based on mode ───────────────────────────────────────── +if [[ "$MODE" == "--all" ]]; then + FILELIST=$(mktemp) + { git ls-files 2>/dev/null; git ls-files --others --exclude-standard 2>/dev/null; } > "$FILELIST" + echo -e "${YELLOW}Scanning entire working tree...${NC}" +else + FILELIST=$(mktemp) + git diff --cached --name-only --diff-filter=ACMR 2>/dev/null > "$FILELIST" || true + if [[ ! -s "$FILELIST" ]]; then + echo -e "${GREEN}No staged files to scan.${NC}" + rm -f "$FILELIST" + exit 0 + fi + echo -e "${YELLOW}Scanning $(wc -l < "$FILELIST" | tr -d ' ') staged files...${NC}" +fi + +# ── Check 1: Dangerous file extensions ────────────────────────────────── +KEY_EXT='\.key$|\.pem$|\.p12$|\.pfx$|\.jks$|\.keystore$|\.p8$|\.der$' +SECRET_EXT='\.secret$|\.secrets$|\.credential$|\.credentials$' + +HITS=$(grep -iE "$KEY_EXT|$SECRET_EXT" "$FILELIST" 2>/dev/null || true) +if [[ -n "$HITS" ]]; then + echo -e "\n${RED}BLOCKED: Key/secret files detected:${NC}" + echo "$HITS" | while read -r f; do echo -e " ${RED}$f${NC}"; done + FOUND=1 +fi + +# ── Check 2: Dangerous filenames ──────────────────────────────────────── +RISKY='id_rsa|id_ed25519|id_ecdsa|private_key|private\.key|secret_key|master\.key' +RISKY+='|serviceaccount|gcloud.*\.json|firebase.*\.json|\.htpasswd' + +HITS=$(grep -iE "$RISKY" "$FILELIST" 2>/dev/null || true) +if [[ -n "$HITS" ]]; then + echo -e "\n${RED}BLOCKED: Risky filenames detected:${NC}" + echo "$HITS" | while read -r f; do echo -e " ${RED}$f${NC}"; done + FOUND=1 +fi + +# ── Check 3: .env files (not .env.example) ────────────────────────────── +HITS=$(grep -E '(^|/)\.env(\.[^e].*)?$' "$FILELIST" 2>/dev/null | grep -v '\.example' || true) +if [[ -n "$HITS" ]]; then + echo -e "\n${RED}BLOCKED: Environment files detected:${NC}" + echo "$HITS" | while read -r f; do echo -e " ${RED}$f${NC}"; done + FOUND=1 +fi + +# ── Check 4: _domain_keys directory (project-specific) ────────────────── +HITS=$(grep '_domain_keys/' "$FILELIST" 2>/dev/null || true) +if [[ -n "$HITS" ]]; then + echo -e "\n${RED}BLOCKED: Domain keys directory detected:${NC}" + echo "$HITS" | while read -r f; do echo -e " ${RED}$f${NC}"; done + FOUND=1 +fi + +# ── Check 5: Content scan for embedded secrets (single grep pass) ─────── +# Build one mega-pattern and run grep once across all files (fast!) +SECRET_REGEX='PRIVATE KEY-----|' +SECRET_REGEX+='ssh-rsa AAAA[0-9A-Za-z+/]|' +SECRET_REGEX+='ssh-ed25519 AAAA[0-9A-Za-z+/]|' +SECRET_REGEX+='ghp_[0-9a-zA-Z]{36}|' # GitHub PAT +SECRET_REGEX+='github_pat_[0-9a-zA-Z]{22}_[0-9a-zA-Z]{59}|' # GitHub fine-grained +SECRET_REGEX+='gho_[0-9a-zA-Z]{36}|' # GitHub OAuth +SECRET_REGEX+='sk-[0-9a-zA-Z]{48}|' # OpenAI key +SECRET_REGEX+='sk-ant-[0-9a-zA-Z-]{90,}|' # Anthropic key +SECRET_REGEX+='AKIA[0-9A-Z]{16}|' # AWS access key +SECRET_REGEX+='AIzaSy[0-9A-Za-z_-]{33}|' # Google API key +SECRET_REGEX+='xox[bpoas]-[0-9a-zA-Z-]+|' # Slack token +SECRET_REGEX+='npm_[0-9a-zA-Z]{36}|' # npm token +SECRET_REGEX+='pypi-[0-9a-zA-Z-]{50,}' # PyPI token + +# Filter to text-like files only (skip binaries by extension + skip this script) +TEXT_FILES=$(grep -ivE '\.(png|jpg|jpeg|gif|ico|svg|woff2?|ttf|eot|pbf|zip|tar|gz|db|sqlite|xlsx|pdf|mp[34]|wav|ogg|webm|webp|avif)$' "$FILELIST" | grep -v 'scan-secrets\.sh$' || true) + +if [[ -n "$TEXT_FILES" ]]; then + # Use grep with file list, skip missing/binary, limit output + CONTENT_HITS=$(echo "$TEXT_FILES" | xargs grep -lE "$SECRET_REGEX" 2>/dev/null || true) + if [[ -n "$CONTENT_HITS" ]]; then + echo -e "\n${RED}BLOCKED: Embedded secrets/tokens found in:${NC}" + echo "$CONTENT_HITS" | while read -r f; do + echo -e " ${RED}$f${NC}" + # Show first matching line for context + grep -nE "$SECRET_REGEX" "$f" 2>/dev/null | head -2 | while read -r line; do + echo -e " ${YELLOW}$line${NC}" + done + done + FOUND=1 + fi +fi + +rm -f "$FILELIST" + +# ── Result ────────────────────────────────────────────────────────────── +echo "" +if [[ $FOUND -eq 1 ]]; then + echo -e "${RED}Secret scan FAILED. Add these to .gitignore or remove them before committing.${NC}" + echo -e "${YELLOW}If intentional (e.g. test fixtures): git commit --no-verify${NC}" + exit 1 +else + echo -e "${GREEN}Secret scan passed. No keys or secrets detected.${NC}" + exit 0 +fi diff --git a/backend/scripts/setup-venv.ps1 b/backend/scripts/setup-venv.ps1 new file mode 100644 index 00000000..7aafbaed --- /dev/null +++ b/backend/scripts/setup-venv.ps1 @@ -0,0 +1,10 @@ +param( + [string]$Python = "python" +) + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$venvPath = Join-Path $repoRoot "venv" +& $Python -m venv $venvPath + +$pip = Join-Path $venvPath "Scripts\pip.exe" +& $pip install -r (Join-Path $repoRoot "requirements-dev.txt") diff --git a/backend/scripts/setup-venv.sh b/backend/scripts/setup-venv.sh new file mode 100644 index 00000000..9490c8f0 --- /dev/null +++ b/backend/scripts/setup-venv.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +PYTHON="${PYTHON:-python3}" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +VENV_DIR="$REPO_ROOT/venv" + +"$PYTHON" -m venv "$VENV_DIR" +"$VENV_DIR/bin/pip" install -r "$REPO_ROOT/requirements-dev.txt" diff --git a/backend/seattle_sample.json b/backend/seattle_sample.json deleted file mode 100644 index 7bf97de8..00000000 --- a/backend/seattle_sample.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "code" : "dataset.missing", - "error" : true, - "message" : "Not found", - "data" : { - "id" : "xqwu-hwdm" - } -} diff --git a/backend/services/ais_stream.py b/backend/services/ais_stream.py index 1b974eb6..6d930aa8 100644 --- a/backend/services/ais_stream.py +++ b/backend/services/ais_stream.py @@ -16,18 +16,19 @@ AIS_WS_URL = "wss://stream.aisstream.io/v0/stream" API_KEY = os.environ.get("AIS_API_KEY", "") + # AIS vessel type code classification # See: https://coast.noaa.gov/data/marinecadastre/ais/VesselTypeCodes2018.pdf def classify_vessel(ais_type: int, mmsi: int) -> str: """Classify a vessel by its AIS type code into a rendering category.""" if 80 <= ais_type <= 89: - return "tanker" # Oil/Chemical/Gas tankers → RED + return "tanker" # Oil/Chemical/Gas tankers → RED if 70 <= ais_type <= 79: - return "cargo" # Cargo ships, container vessels → RED + return "cargo" # Cargo ships, container vessels → RED if 60 <= ais_type <= 69: - return "passenger" # Cruise ships, ferries → GRAY + return "passenger" # Cruise ships, ferries → GRAY if ais_type in (36, 37): - return "yacht" # Sailing/Pleasure craft → DARK BLUE + return "yacht" # Sailing/Pleasure craft → DARK BLUE if ais_type == 35: return "military_vessel" # Military → YELLOW # MMSI-based military detection: military MMSIs often start with certain prefixes @@ -35,87 +36,286 @@ def classify_vessel(ais_type: int, mmsi: int) -> str: if mmsi_str.startswith("3380") or mmsi_str.startswith("3381"): return "military_vessel" # US Navy if ais_type in (30, 31, 32, 33, 34): - return "other" # Fishing, towing, dredging, diving, etc. + return "other" # Fishing, towing, dredging, diving, etc. if ais_type in (50, 51, 52, 53, 54, 55, 56, 57, 58, 59): - return "other" # Pilot, SAR, tug, port tender, etc. - return "unknown" # Not yet classified — will update when ShipStaticData arrives + return "other" # Pilot, SAR, tug, port tender, etc. + return "unknown" # Not yet classified — will update when ShipStaticData arrives # MMSI Maritime Identification Digit (MID) → Country mapping # First 3 digits of MMSI (for 9-digit MMSIs) encode the flag state MID_COUNTRY = { - 201: "Albania", 202: "Andorra", 203: "Austria", 204: "Portugal", 205: "Belgium", - 206: "Belarus", 207: "Bulgaria", 208: "Vatican", 209: "Cyprus", 210: "Cyprus", - 211: "Germany", 212: "Cyprus", 213: "Georgia", 214: "Moldova", 215: "Malta", - 216: "Armenia", 218: "Germany", 219: "Denmark", 220: "Denmark", 224: "Spain", - 225: "Spain", 226: "France", 227: "France", 228: "France", 229: "Malta", - 230: "Finland", 231: "Faroe Islands", 232: "United Kingdom", 233: "United Kingdom", - 234: "United Kingdom", 235: "United Kingdom", 236: "Gibraltar", 237: "Greece", - 238: "Croatia", 239: "Greece", 240: "Greece", 241: "Greece", 242: "Morocco", - 243: "Hungary", 244: "Netherlands", 245: "Netherlands", 246: "Netherlands", - 247: "Italy", 248: "Malta", 249: "Malta", 250: "Ireland", 251: "Iceland", - 252: "Liechtenstein", 253: "Luxembourg", 254: "Monaco", 255: "Portugal", - 256: "Malta", 257: "Norway", 258: "Norway", 259: "Norway", 261: "Poland", - 263: "Portugal", 264: "Romania", 265: "Sweden", 266: "Sweden", 267: "Slovakia", - 268: "San Marino", 269: "Switzerland", 270: "Czech Republic", 271: "Turkey", - 272: "Ukraine", 273: "Russia", 274: "North Macedonia", 275: "Latvia", - 276: "Estonia", 277: "Lithuania", 278: "Slovenia", - 301: "Anguilla", 303: "Alaska", 304: "Antigua", 305: "Antigua", - 306: "Netherlands Antilles", 307: "Aruba", 308: "Bahamas", 309: "Bahamas", - 310: "Bermuda", 311: "Bahamas", 312: "Belize", 314: "Barbados", 316: "Canada", - 319: "Cayman Islands", 321: "Costa Rica", 323: "Cuba", 325: "Dominica", - 327: "Dominican Republic", 329: "Guadeloupe", 330: "Grenada", 331: "Greenland", - 332: "Guatemala", 334: "Honduras", 336: "Haiti", 338: "United States", - 339: "Jamaica", 341: "Saint Kitts", 343: "Saint Lucia", 345: "Mexico", - 347: "Martinique", 348: "Montserrat", 350: "Nicaragua", 351: "Panama", - 352: "Panama", 353: "Panama", 354: "Panama", 355: "Panama", - 356: "Panama", 357: "Panama", 358: "Puerto Rico", 359: "El Salvador", - 361: "Saint Pierre", 362: "Trinidad", 364: "Turks and Caicos", - 366: "United States", 367: "United States", 368: "United States", 369: "United States", - 370: "Panama", 371: "Panama", 372: "Panama", 373: "Panama", - 374: "Panama", 375: "Saint Vincent", 376: "Saint Vincent", 377: "Saint Vincent", - 378: "British Virgin Islands", 379: "US Virgin Islands", - 401: "Afghanistan", 403: "Saudi Arabia", 405: "Bangladesh", 408: "Bahrain", - 410: "Bhutan", 412: "China", 413: "China", 414: "China", - 416: "Taiwan", 417: "Sri Lanka", 419: "India", 422: "Iran", - 423: "Azerbaijan", 425: "Iraq", 428: "Israel", 431: "Japan", - 432: "Japan", 434: "Turkmenistan", 436: "Kazakhstan", 437: "Uzbekistan", - 438: "Jordan", 440: "South Korea", 441: "South Korea", 443: "Palestine", - 445: "North Korea", 447: "Kuwait", 450: "Lebanon", 451: "Kyrgyzstan", - 453: "Macao", 455: "Maldives", 457: "Mongolia", 459: "Nepal", - 461: "Oman", 463: "Pakistan", 466: "Qatar", 468: "Syria", - 470: "UAE", 472: "Tajikistan", 473: "Yemen", 475: "Tonga", - 477: "Hong Kong", 478: "Bosnia", - 501: "Antarctica", 503: "Australia", 506: "Myanmar", - 508: "Brunei", 510: "Micronesia", 511: "Palau", 512: "New Zealand", - 514: "Cambodia", 515: "Cambodia", 516: "Christmas Island", - 518: "Cook Islands", 520: "Fiji", 523: "Cocos Islands", - 525: "Indonesia", 529: "Kiribati", 531: "Laos", 533: "Malaysia", - 536: "Northern Mariana Islands", 538: "Marshall Islands", - 540: "New Caledonia", 542: "Niue", 544: "Nauru", 546: "French Polynesia", - 548: "Philippines", 553: "Papua New Guinea", 555: "Pitcairn", - 557: "Solomon Islands", 559: "American Samoa", 561: "Samoa", - 563: "Singapore", 564: "Singapore", 565: "Singapore", 566: "Singapore", - 567: "Thailand", 570: "Tonga", 572: "Tuvalu", 574: "Vietnam", - 576: "Vanuatu", 577: "Vanuatu", 578: "Wallis and Futuna", - 601: "South Africa", 603: "Angola", 605: "Algeria", 607: "Benin", - 609: "Botswana", 610: "Burundi", 611: "Cameroon", 612: "Cape Verde", - 613: "Central African Republic", 615: "Congo", 616: "Comoros", - 617: "DR Congo", 618: "Ivory Coast", 619: "Djibouti", - 620: "Egypt", 621: "Equatorial Guinea", 622: "Ethiopia", - 624: "Eritrea", 625: "Gabon", 626: "Gambia", 627: "Ghana", - 629: "Guinea", 630: "Guinea-Bissau", 631: "Kenya", 632: "Lesotho", - 633: "Liberia", 634: "Liberia", 635: "Liberia", 636: "Liberia", - 637: "Libya", 642: "Madagascar", 644: "Malawi", 645: "Mali", - 647: "Mauritania", 649: "Mauritius", 650: "Mozambique", - 654: "Namibia", 655: "Niger", 656: "Nigeria", 657: "Guinea", - 659: "Rwanda", 660: "Senegal", 661: "Sierra Leone", - 662: "Somalia", 663: "South Africa", 664: "Sudan", - 667: "Tanzania", 668: "Togo", 669: "Tunisia", 670: "Uganda", - 671: "Egypt", 672: "Tanzania", 674: "Zambia", 675: "Zimbabwe", - 676: "Comoros", 677: "Tanzania", + 201: "Albania", + 202: "Andorra", + 203: "Austria", + 204: "Portugal", + 205: "Belgium", + 206: "Belarus", + 207: "Bulgaria", + 208: "Vatican", + 209: "Cyprus", + 210: "Cyprus", + 211: "Germany", + 212: "Cyprus", + 213: "Georgia", + 214: "Moldova", + 215: "Malta", + 216: "Armenia", + 218: "Germany", + 219: "Denmark", + 220: "Denmark", + 224: "Spain", + 225: "Spain", + 226: "France", + 227: "France", + 228: "France", + 229: "Malta", + 230: "Finland", + 231: "Faroe Islands", + 232: "United Kingdom", + 233: "United Kingdom", + 234: "United Kingdom", + 235: "United Kingdom", + 236: "Gibraltar", + 237: "Greece", + 238: "Croatia", + 239: "Greece", + 240: "Greece", + 241: "Greece", + 242: "Morocco", + 243: "Hungary", + 244: "Netherlands", + 245: "Netherlands", + 246: "Netherlands", + 247: "Italy", + 248: "Malta", + 249: "Malta", + 250: "Ireland", + 251: "Iceland", + 252: "Liechtenstein", + 253: "Luxembourg", + 254: "Monaco", + 255: "Portugal", + 256: "Malta", + 257: "Norway", + 258: "Norway", + 259: "Norway", + 261: "Poland", + 263: "Portugal", + 264: "Romania", + 265: "Sweden", + 266: "Sweden", + 267: "Slovakia", + 268: "San Marino", + 269: "Switzerland", + 270: "Czech Republic", + 271: "Turkey", + 272: "Ukraine", + 273: "Russia", + 274: "North Macedonia", + 275: "Latvia", + 276: "Estonia", + 277: "Lithuania", + 278: "Slovenia", + 301: "Anguilla", + 303: "Alaska", + 304: "Antigua", + 305: "Antigua", + 306: "Netherlands Antilles", + 307: "Aruba", + 308: "Bahamas", + 309: "Bahamas", + 310: "Bermuda", + 311: "Bahamas", + 312: "Belize", + 314: "Barbados", + 316: "Canada", + 319: "Cayman Islands", + 321: "Costa Rica", + 323: "Cuba", + 325: "Dominica", + 327: "Dominican Republic", + 329: "Guadeloupe", + 330: "Grenada", + 331: "Greenland", + 332: "Guatemala", + 334: "Honduras", + 336: "Haiti", + 338: "United States", + 339: "Jamaica", + 341: "Saint Kitts", + 343: "Saint Lucia", + 345: "Mexico", + 347: "Martinique", + 348: "Montserrat", + 350: "Nicaragua", + 351: "Panama", + 352: "Panama", + 353: "Panama", + 354: "Panama", + 355: "Panama", + 356: "Panama", + 357: "Panama", + 358: "Puerto Rico", + 359: "El Salvador", + 361: "Saint Pierre", + 362: "Trinidad", + 364: "Turks and Caicos", + 366: "United States", + 367: "United States", + 368: "United States", + 369: "United States", + 370: "Panama", + 371: "Panama", + 372: "Panama", + 373: "Panama", + 374: "Panama", + 375: "Saint Vincent", + 376: "Saint Vincent", + 377: "Saint Vincent", + 378: "British Virgin Islands", + 379: "US Virgin Islands", + 401: "Afghanistan", + 403: "Saudi Arabia", + 405: "Bangladesh", + 408: "Bahrain", + 410: "Bhutan", + 412: "China", + 413: "China", + 414: "China", + 416: "Taiwan", + 417: "Sri Lanka", + 419: "India", + 422: "Iran", + 423: "Azerbaijan", + 425: "Iraq", + 428: "Israel", + 431: "Japan", + 432: "Japan", + 434: "Turkmenistan", + 436: "Kazakhstan", + 437: "Uzbekistan", + 438: "Jordan", + 440: "South Korea", + 441: "South Korea", + 443: "Palestine", + 445: "North Korea", + 447: "Kuwait", + 450: "Lebanon", + 451: "Kyrgyzstan", + 453: "Macao", + 455: "Maldives", + 457: "Mongolia", + 459: "Nepal", + 461: "Oman", + 463: "Pakistan", + 466: "Qatar", + 468: "Syria", + 470: "UAE", + 472: "Tajikistan", + 473: "Yemen", + 475: "Tonga", + 477: "Hong Kong", + 478: "Bosnia", + 501: "Antarctica", + 503: "Australia", + 506: "Myanmar", + 508: "Brunei", + 510: "Micronesia", + 511: "Palau", + 512: "New Zealand", + 514: "Cambodia", + 515: "Cambodia", + 516: "Christmas Island", + 518: "Cook Islands", + 520: "Fiji", + 523: "Cocos Islands", + 525: "Indonesia", + 529: "Kiribati", + 531: "Laos", + 533: "Malaysia", + 536: "Northern Mariana Islands", + 538: "Marshall Islands", + 540: "New Caledonia", + 542: "Niue", + 544: "Nauru", + 546: "French Polynesia", + 548: "Philippines", + 553: "Papua New Guinea", + 555: "Pitcairn", + 557: "Solomon Islands", + 559: "American Samoa", + 561: "Samoa", + 563: "Singapore", + 564: "Singapore", + 565: "Singapore", + 566: "Singapore", + 567: "Thailand", + 570: "Tonga", + 572: "Tuvalu", + 574: "Vietnam", + 576: "Vanuatu", + 577: "Vanuatu", + 578: "Wallis and Futuna", + 601: "South Africa", + 603: "Angola", + 605: "Algeria", + 607: "Benin", + 609: "Botswana", + 610: "Burundi", + 611: "Cameroon", + 612: "Cape Verde", + 613: "Central African Republic", + 615: "Congo", + 616: "Comoros", + 617: "DR Congo", + 618: "Ivory Coast", + 619: "Djibouti", + 620: "Egypt", + 621: "Equatorial Guinea", + 622: "Ethiopia", + 624: "Eritrea", + 625: "Gabon", + 626: "Gambia", + 627: "Ghana", + 629: "Guinea", + 630: "Guinea-Bissau", + 631: "Kenya", + 632: "Lesotho", + 633: "Liberia", + 634: "Liberia", + 635: "Liberia", + 636: "Liberia", + 637: "Libya", + 642: "Madagascar", + 644: "Malawi", + 645: "Mali", + 647: "Mauritania", + 649: "Mauritius", + 650: "Mozambique", + 654: "Namibia", + 655: "Niger", + 656: "Nigeria", + 657: "Guinea", + 659: "Rwanda", + 660: "Senegal", + 661: "Sierra Leone", + 662: "Somalia", + 663: "South Africa", + 664: "Sudan", + 667: "Tanzania", + 668: "Togo", + 669: "Tunisia", + 670: "Uganda", + 671: "Egypt", + 672: "Tanzania", + 674: "Zambia", + 675: "Zimbabwe", + 676: "Comoros", + 677: "Tanzania", } + def get_country_from_mmsi(mmsi: int) -> str: """Look up flag state from MMSI Maritime Identification Digit.""" mmsi_str = str(mmsi) @@ -130,8 +330,10 @@ def get_country_from_mmsi(mmsi: int) -> str: _vessels_lock = threading.Lock() _ws_thread: threading.Thread | None = None _ws_running = False +_proxy_process = None import os + CACHE_FILE = os.path.join(os.path.dirname(__file__), "ais_cache.json") @@ -141,7 +343,7 @@ def _save_cache(): with _vessels_lock: # Convert int keys to strings for JSON data = {str(k): v for k, v in _vessels.items()} - with open(CACHE_FILE, 'w') as f: + with open(CACHE_FILE, "w") as f: json.dump(data, f) logger.info(f"AIS cache saved: {len(data)} vessels") except (IOError, OSError) as e: @@ -154,7 +356,7 @@ def _load_cache(): if not os.path.exists(CACHE_FILE): return try: - with open(CACHE_FILE, 'r') as f: + with open(CACHE_FILE, "r") as f: data = json.load(f) now = time.time() stale_cutoff = now - 3600 # Accept vessels up to 1 hour old on restart @@ -169,41 +371,51 @@ def _load_cache(): logger.error(f"Failed to load AIS cache: {e}") -def get_ais_vessels() -> list[dict]: - """Return a snapshot of tracked AIS vessels, excluding 'other' type, pruning stale.""" +def prune_stale_vessels(): + """Remove vessels not updated in the last 15 minutes. Safe to call from a scheduler.""" now = time.time() - stale_cutoff = now - 900 # 15 minutes - + stale_cutoff = now - 900 with _vessels_lock: - # Prune stale vessels stale_keys = [k for k, v in _vessels.items() if v.get("_updated", 0) < stale_cutoff] for k in stale_keys: del _vessels[k] - + if stale_keys: + logger.info(f"AIS pruned {len(stale_keys)} stale vessels") + + +def get_ais_vessels() -> list[dict]: + """Return a snapshot of tracked AIS vessels, pruning stale.""" + prune_stale_vessels() + + with _vessels_lock: result = [] for mmsi, v in _vessels.items(): v_type = v.get("type", "unknown") - # Skip 'other' vessels (fishing, tug, pilot, etc.) to reduce load - if v_type == "other": - continue # Skip vessels without valid position if not v.get("lat") or not v.get("lng"): continue - - result.append({ - "mmsi": mmsi, - "name": v.get("name", "UNKNOWN"), - "type": v_type, - "lat": round(v.get("lat", 0), 5), - "lng": round(v.get("lng", 0), 5), - "heading": v.get("heading", 0), - "sog": round(v.get("sog", 0), 1), - "cog": round(v.get("cog", 0), 1), - "callsign": v.get("callsign", ""), - "destination": v.get("destination", "") or "UNKNOWN", - "imo": v.get("imo", 0), - "country": get_country_from_mmsi(mmsi), - }) + + # Sanitize speed: AIS 102.3 kn = "speed not available" + sog = v.get("sog", 0) + if sog >= 102.2: + sog = 0 + + result.append( + { + "mmsi": mmsi, + "name": v.get("name", "UNKNOWN"), + "type": v_type, + "lat": round(v.get("lat", 0), 5), + "lng": round(v.get("lng", 0), 5), + "heading": v.get("heading", 0), + "sog": round(sog, 1), + "cog": round(v.get("cog", 0), 1), + "callsign": v.get("callsign", ""), + "destination": v.get("destination", "") or "UNKNOWN", + "imo": v.get("imo", 0), + "country": get_country_from_mmsi(mmsi), + } + ) return result @@ -228,7 +440,9 @@ def ingest_ais_catcher(msgs: list[dict]) -> int: if lat is not None and lon is not None and lat != 91.0 and lon != 181.0: vessel["lat"] = lat vessel["lng"] = lon - vessel["sog"] = msg.get("speed", 0) + # AIS raw value 1023 (102.3 kn) = "speed not available" + raw_speed = msg.get("speed", 0) + vessel["sog"] = 0 if raw_speed >= 102.2 else raw_speed vessel["cog"] = msg.get("course", 0) heading = msg.get("heading", 511) vessel["heading"] = heading if heading != 511 else vessel.get("cog", 0) @@ -276,31 +490,37 @@ def _ais_stream_loop(): while _ws_running: try: logger.info("Starting Node.js AIS Stream Proxy...") + proxy_env = os.environ.copy() + proxy_env["AIS_API_KEY"] = API_KEY process = subprocess.Popen( - ['node', proxy_script, API_KEY], + ["node", proxy_script], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - bufsize=1 + bufsize=1, + env=proxy_env, ) - _proxy_process = process - + with _vessels_lock: + _proxy_process = process + # Drain stderr in a background thread to prevent deadlock import threading + def _drain_stderr(): - for errline in iter(process.stderr.readline, ''): + for errline in iter(process.stderr.readline, ""): errline = errline.strip() if errline: logger.warning(f"AIS proxy stderr: {errline}") + threading.Thread(target=_drain_stderr, daemon=True).start() - + logger.info("AIS Stream proxy started — receiving vessel data") - + msg_count = 0 ok_streak = 0 # Track consecutive successful messages for backoff reset last_log_time = time.time() - for raw_msg in iter(process.stdout.readline, ''): + for raw_msg in iter(process.stdout.readline, ""): if not _ws_running: process.terminate() break @@ -346,14 +566,18 @@ def _drain_stderr(): with _vessels_lock: vessel["lat"] = lat vessel["lng"] = lng - vessel["sog"] = report.get("Sog", 0) + # AIS raw value 1023 (102.3 kn) = "speed not available" + raw_sog = report.get("Sog", 0) + vessel["sog"] = 0 if raw_sog >= 102.2 else raw_sog vessel["cog"] = report.get("Cog", 0) heading = report.get("TrueHeading", 511) vessel["heading"] = heading if heading != 511 else report.get("Cog", 0) vessel["_updated"] = time.time() # Use metadata name if we don't have one yet if not vessel.get("name") or vessel["name"] == "UNKNOWN": - vessel["name"] = metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN" + vessel["name"] = ( + metadata.get("ShipName", "UNKNOWN").strip() or "UNKNOWN" + ) # Update static data from ShipStaticData elif msg_type == "ShipStaticData": @@ -361,10 +585,14 @@ def _drain_stderr(): ais_type = static.get("Type", 0) with _vessels_lock: - vessel["name"] = (static.get("Name", "") or metadata.get("ShipName", "UNKNOWN")).strip() or "UNKNOWN" + vessel["name"] = ( + static.get("Name", "") or metadata.get("ShipName", "UNKNOWN") + ).strip() or "UNKNOWN" vessel["callsign"] = (static.get("CallSign", "") or "").strip() vessel["imo"] = static.get("ImoNumber", 0) - vessel["destination"] = (static.get("Destination", "") or "").strip().replace("@", "") + vessel["destination"] = ( + (static.get("Destination", "") or "").strip().replace("@", "") + ) vessel["ais_type_code"] = ais_type vessel["type"] = classify_vessel(ais_type, mmsi) vessel["_updated"] = time.time() @@ -382,7 +610,9 @@ def _drain_stderr(): if now - last_log_time >= 60: with _vessels_lock: count = len(_vessels) - logger.info(f"AIS Stream: processed {msg_count} messages, tracking {count} vessels") + logger.info( + f"AIS Stream: processed {msg_count} messages, tracking {count} vessels" + ) _save_cache() last_log_time = now @@ -397,23 +627,34 @@ def _drain_stderr(): def _run_ais_loop(): """Thread target: run the AIS loop.""" + global _ws_running, _ws_thread, _proxy_process try: _ais_stream_loop() except Exception as e: logger.error(f"AIS Stream thread crashed: {e}") + finally: + with _vessels_lock: + _ws_running = False + _ws_thread = None + _proxy_process = None def start_ais_stream(): """Start the AIS WebSocket stream in a background thread.""" global _ws_thread, _ws_running - if _ws_thread and _ws_thread.is_alive(): + with _vessels_lock: + if _ws_running: + logger.info("AIS Stream already running") + return + _ws_running = True + existing_thread = _ws_thread + if existing_thread and existing_thread.is_alive(): logger.info("AIS Stream already running") return - + # Load cached vessel data from disk _load_cache() - - _ws_running = True + _ws_thread = threading.Thread(target=_run_ais_loop, daemon=True, name="ais-stream") _ws_thread.start() logger.info("AIS Stream background thread started") @@ -421,31 +662,36 @@ def start_ais_stream(): def stop_ais_stream(): """Stop the AIS WebSocket stream and save cache.""" - global _ws_running, _proxy_process - _ws_running = False - - if _proxy_process and _proxy_process.stdin: + global _ws_running, _ws_thread, _proxy_process + with _vessels_lock: + _ws_running = False + _ws_thread = None + proc = _proxy_process + _proxy_process = None + + if proc and proc.stdin: try: - _proxy_process.stdin.close() + proc.stdin.close() except Exception: pass - + _save_cache() # Save on shutdown logger.info("AIS Stream stopping...") + def update_ais_bbox(south: float, west: float, north: float, east: float): """Dynamically update the AIS stream bounding box via proxy stdin.""" - global _proxy_process - if not _proxy_process or not _proxy_process.stdin: + with _vessels_lock: + proc = _proxy_process + if not proc or not proc.stdin: return - + try: - cmd = json.dumps({ - "type": "update_bbox", - "bboxes": [[[south, west], [north, east]]] - }) - _proxy_process.stdin.write(cmd + "\n") - _proxy_process.stdin.flush() - logger.info(f"Updated AIS bounding box to: S:{south:.2f} W:{west:.2f} N:{north:.2f} E:{east:.2f}") + cmd = json.dumps({"type": "update_bbox", "bboxes": [[[south, west], [north, east]]]}) + proc.stdin.write(cmd + "\n") + proc.stdin.flush() + logger.info( + f"Updated AIS bounding box to: S:{south:.2f} W:{west:.2f} N:{north:.2f} E:{east:.2f}" + ) except Exception as e: logger.error(f"Failed to update AIS bbox: {e}") diff --git a/backend/services/api_settings.py b/backend/services/api_settings.py index 141defc3..9a9718f5 100644 --- a/backend/services/api_settings.py +++ b/backend/services/api_settings.py @@ -2,6 +2,7 @@ API Settings management — serves the API key registry and allows updates. Keys are stored in the backend .env file and loaded via python-dotenv. """ + import os import re from pathlib import Path @@ -121,6 +122,24 @@ "url": "https://openmhz.com/", "required": False, }, + { + "id": "shodan_api_key", + "env_key": "SHODAN_API_KEY", + "name": "Shodan — Operator API Key", + "description": "Paid Shodan API key for local operator-driven searches and temporary map overlays. Results are attributed to Shodan and are not merged into ShadowBroker core feeds.", + "category": "Reconnaissance", + "url": "https://account.shodan.io/billing", + "required": False, + }, + { + "id": "finnhub_api_key", + "env_key": "FINNHUB_API_KEY", + "name": "Finnhub — API Key", + "description": "Free market data API. Defense stock quotes, congressional trading disclosures, and insider transactions. 60 calls/min free tier.", + "category": "Financial", + "url": "https://finnhub.io/register", + "required": False, + }, ] @@ -160,7 +179,7 @@ def update_api_key(env_key: str, new_value: str) -> bool: valid_keys = {api["env_key"] for api in API_REGISTRY if api.get("env_key")} if env_key not in valid_keys: return False - + if not isinstance(new_value, str): return False if "\n" in new_value or "\r" in new_value: diff --git a/backend/services/carrier_tracker.py b/backend/services/carrier_tracker.py index 8780cb52..f89a539e 100644 --- a/backend/services/carrier_tracker.py +++ b/backend/services/carrier_tracker.py @@ -15,6 +15,7 @@ import time import logging import threading +import random from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional @@ -34,108 +35,127 @@ "name": "USS Nimitz (CVN-68)", "wiki": "https://en.wikipedia.org/wiki/USS_Nimitz", "homeport": "Bremerton, WA", - "homeport_lat": 47.5535, "homeport_lng": -122.6400, - "fallback_lat": 47.5535, "fallback_lng": -122.6400, + "homeport_lat": 47.5535, + "homeport_lng": -122.6400, + "fallback_lat": 47.5535, + "fallback_lng": -122.6400, "fallback_heading": 90, - "fallback_desc": "Bremerton, WA (Maintenance)" + "fallback_desc": "Bremerton, WA (Maintenance)", }, "CVN-76": { "name": "USS Ronald Reagan (CVN-76)", "wiki": "https://en.wikipedia.org/wiki/USS_Ronald_Reagan", "homeport": "Bremerton, WA", - "homeport_lat": 47.5580, "homeport_lng": -122.6360, - "fallback_lat": 47.5580, "fallback_lng": -122.6360, + "homeport_lat": 47.5580, + "homeport_lng": -122.6360, + "fallback_lat": 47.5580, + "fallback_lng": -122.6360, "fallback_heading": 90, - "fallback_desc": "Bremerton, WA (Decommissioning)" + "fallback_desc": "Bremerton, WA (Decommissioning)", }, - # --- Norfolk, VA (Naval Station Norfolk) --- # Piers run N-S along Willoughby Bay; each carrier gets a distinct berth "CVN-69": { "name": "USS Dwight D. Eisenhower (CVN-69)", "wiki": "https://en.wikipedia.org/wiki/USS_Dwight_D._Eisenhower", "homeport": "Norfolk, VA", - "homeport_lat": 36.9465, "homeport_lng": -76.3265, - "fallback_lat": 36.9465, "fallback_lng": -76.3265, + "homeport_lat": 36.9465, + "homeport_lng": -76.3265, + "fallback_lat": 36.9465, + "fallback_lng": -76.3265, "fallback_heading": 0, - "fallback_desc": "Norfolk, VA (Post-deployment maintenance)" + "fallback_desc": "Norfolk, VA (Post-deployment maintenance)", }, "CVN-78": { "name": "USS Gerald R. Ford (CVN-78)", "wiki": "https://en.wikipedia.org/wiki/USS_Gerald_R._Ford", "homeport": "Norfolk, VA", - "homeport_lat": 36.9505, "homeport_lng": -76.3250, - "fallback_lat": 18.0, "fallback_lng": 39.5, + "homeport_lat": 36.9505, + "homeport_lng": -76.3250, + "fallback_lat": 18.0, + "fallback_lng": 39.5, "fallback_heading": 0, - "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)" + "fallback_desc": "Red Sea — Operation Epic Fury (USNI Mar 9)", }, "CVN-74": { "name": "USS John C. Stennis (CVN-74)", "wiki": "https://en.wikipedia.org/wiki/USS_John_C._Stennis", "homeport": "Norfolk, VA", - "homeport_lat": 36.9540, "homeport_lng": -76.3235, - "fallback_lat": 36.98, "fallback_lng": -76.43, + "homeport_lat": 36.9540, + "homeport_lng": -76.3235, + "fallback_lat": 36.98, + "fallback_lng": -76.43, "fallback_heading": 0, - "fallback_desc": "Newport News, VA (RCOH refueling overhaul)" + "fallback_desc": "Newport News, VA (RCOH refueling overhaul)", }, "CVN-75": { "name": "USS Harry S. Truman (CVN-75)", "wiki": "https://en.wikipedia.org/wiki/USS_Harry_S._Truman", "homeport": "Norfolk, VA", - "homeport_lat": 36.9580, "homeport_lng": -76.3220, - "fallback_lat": 36.0, "fallback_lng": 15.0, + "homeport_lat": 36.9580, + "homeport_lng": -76.3220, + "fallback_lat": 36.0, + "fallback_lng": 15.0, "fallback_heading": 0, - "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)" + "fallback_desc": "Mediterranean Sea deployment (USNI Mar 9)", }, "CVN-77": { "name": "USS George H.W. Bush (CVN-77)", "wiki": "https://en.wikipedia.org/wiki/USS_George_H.W._Bush", "homeport": "Norfolk, VA", - "homeport_lat": 36.9620, "homeport_lng": -76.3210, - "fallback_lat": 36.5, "fallback_lng": -74.0, + "homeport_lat": 36.9620, + "homeport_lng": -76.3210, + "fallback_lat": 36.5, + "fallback_lng": -74.0, "fallback_heading": 0, - "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)" + "fallback_desc": "Atlantic — Pre-deployment workups (USNI Mar 9)", }, - # --- San Diego, CA (Naval Base San Diego) --- # Carrier piers along the east shore of San Diego Bay, spread N-S "CVN-70": { "name": "USS Carl Vinson (CVN-70)", "wiki": "https://en.wikipedia.org/wiki/USS_Carl_Vinson", "homeport": "San Diego, CA", - "homeport_lat": 32.6840, "homeport_lng": -117.1290, - "fallback_lat": 32.6840, "fallback_lng": -117.1290, + "homeport_lat": 32.6840, + "homeport_lng": -117.1290, + "fallback_lat": 32.6840, + "fallback_lng": -117.1290, "fallback_heading": 180, - "fallback_desc": "San Diego, CA (Homeport)" + "fallback_desc": "San Diego, CA (Homeport)", }, "CVN-71": { "name": "USS Theodore Roosevelt (CVN-71)", "wiki": "https://en.wikipedia.org/wiki/USS_Theodore_Roosevelt_(CVN-71)", "homeport": "San Diego, CA", - "homeport_lat": 32.6885, "homeport_lng": -117.1280, - "fallback_lat": 32.6885, "fallback_lng": -117.1280, + "homeport_lat": 32.6885, + "homeport_lng": -117.1280, + "fallback_lat": 32.6885, + "fallback_lng": -117.1280, "fallback_heading": 180, - "fallback_desc": "San Diego, CA (Maintenance)" + "fallback_desc": "San Diego, CA (Maintenance)", }, "CVN-72": { "name": "USS Abraham Lincoln (CVN-72)", "wiki": "https://en.wikipedia.org/wiki/USS_Abraham_Lincoln_(CVN-72)", "homeport": "San Diego, CA", - "homeport_lat": 32.6925, "homeport_lng": -117.1275, - "fallback_lat": 20.0, "fallback_lng": 64.0, + "homeport_lat": 32.6925, + "homeport_lng": -117.1275, + "fallback_lat": 20.0, + "fallback_lng": 64.0, "fallback_heading": 0, - "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)" + "fallback_desc": "Arabian Sea — Operation Epic Fury (USNI Mar 9)", }, - # --- Yokosuka, Japan (CFAY) --- "CVN-73": { "name": "USS George Washington (CVN-73)", "wiki": "https://en.wikipedia.org/wiki/USS_George_Washington_(CVN-73)", "homeport": "Yokosuka, Japan", - "homeport_lat": 35.2830, "homeport_lng": 139.6700, - "fallback_lat": 35.2830, "fallback_lng": 139.6700, + "homeport_lat": 35.2830, + "homeport_lng": 139.6700, + "fallback_lat": 35.2830, + "fallback_lng": 139.6700, "fallback_heading": 180, - "fallback_desc": "Yokosuka, Japan (Forward deployed)" + "fallback_desc": "Yokosuka, Japan (Forward deployed)", }, } @@ -175,7 +195,6 @@ "coral sea": (-18.0, 155.0), "gulf of mexico": (25.0, -90.0), "caribbean": (15.0, -75.0), - # Specific bases / ports "norfolk": (36.95, -76.33), "san diego": (32.68, -117.15), @@ -188,7 +207,6 @@ "bremerton": (47.56, -122.63), "puget sound": (47.56, -122.63), "newport news": (36.98, -76.43), - # Areas of operation "centcom": (25.0, 55.0), "indopacom": (20.0, 130.0), @@ -209,6 +227,11 @@ _carrier_positions: Dict[str, dict] = {} _positions_lock = threading.Lock() _last_update: Optional[datetime] = None +_last_gdelt_fetch_at = 0.0 +_cached_gdelt_articles: List[dict] = [] +_GDELT_FETCH_INTERVAL_SECONDS = 1800 +_GDELT_REQUEST_DELAY_SECONDS = 1.25 +_GDELT_REQUEST_JITTER_SECONDS = 0.35 def _load_cache() -> Dict[str, dict]: @@ -260,22 +283,41 @@ def _match_carrier(text: str) -> Optional[str]: def _fetch_gdelt_carrier_news() -> List[dict]: """Search GDELT for recent carrier movement news.""" + global _last_gdelt_fetch_at, _cached_gdelt_articles + + now = time.time() + if _cached_gdelt_articles and (now - _last_gdelt_fetch_at) < _GDELT_FETCH_INTERVAL_SECONDS: + logger.info("Carrier OSINT: using cached GDELT article set to avoid startup bursts") + return list(_cached_gdelt_articles) + results = [] search_terms = [ "aircraft+carrier+deployed", "carrier+strike+group+navy", - "USS+Nimitz+carrier", "USS+Ford+carrier", "USS+Eisenhower+carrier", - "USS+Vinson+carrier", "USS+Roosevelt+carrier+navy", - "USS+Lincoln+carrier", "USS+Truman+carrier", - "USS+Reagan+carrier", "USS+Washington+carrier+navy", - "USS+Bush+carrier", "USS+Stennis+carrier", + "USS+Nimitz+carrier", + "USS+Ford+carrier", + "USS+Eisenhower+carrier", + "USS+Vinson+carrier", + "USS+Roosevelt+carrier+navy", + "USS+Lincoln+carrier", + "USS+Truman+carrier", + "USS+Reagan+carrier", + "USS+Washington+carrier+navy", + "USS+Bush+carrier", + "USS+Stennis+carrier", ] - for term in search_terms: + for idx, term in enumerate(search_terms): try: url = f"https://api.gdeltproject.org/api/v2/doc/doc?query={term}&mode=artlist&maxrecords=5&format=json×pan=14d" raw = fetch_with_curl(url, timeout=8) - if not raw or not hasattr(raw, 'text'): + if getattr(raw, "status_code", 500) == 429: + logger.warning( + "GDELT returned 429 for '%s'; preserving cached carrier OSINT results", + term, + ) + continue + if not raw or not hasattr(raw, "text"): continue data = raw.json() articles = data.get("articles", []) @@ -286,7 +328,14 @@ def _fetch_gdelt_carrier_news() -> List[dict]: except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: logger.debug(f"GDELT search failed for '{term}': {e}") continue - + if idx < len(search_terms) - 1: + time.sleep( + _GDELT_REQUEST_DELAY_SECONDS + + random.uniform(0.0, _GDELT_REQUEST_JITTER_SECONDS) + ) + + _cached_gdelt_articles = list(results) + _last_gdelt_fetch_at = time.time() logger.info(f"Carrier OSINT: found {len(results)} GDELT articles") return results @@ -316,9 +365,11 @@ def _parse_carrier_positions_from_news(articles: List[dict]) -> Dict[str, dict]: "desc": title[:100], "source": "GDELT News API", "source_url": article.get("url", "https://api.gdeltproject.org"), - "updated": datetime.now(timezone.utc).isoformat() + "updated": datetime.now(timezone.utc).isoformat(), } - logger.info(f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})") + logger.info( + f"Carrier update: {CARRIER_REGISTRY[hull]['name']} → {coords} (from: {title[:80]})" + ) return updates @@ -336,21 +387,25 @@ def _load_carrier_fallbacks() -> Dict[str, dict]: "wiki": info["wiki"], "source": "USNI News Fleet & Marine Tracker", "source_url": "https://news.usni.org/category/fleet-tracker", - "updated": datetime.now(timezone.utc).isoformat() + "updated": datetime.now(timezone.utc).isoformat(), } # Overlay cached positions from previous runs (may have GDELT data) cached = _load_cache() for hull, cached_pos in cached.items(): if hull in positions: - if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get("source", "").startswith("News"): - positions[hull].update({ - "lat": cached_pos["lat"], - "lng": cached_pos["lng"], - "desc": cached_pos.get("desc", positions[hull]["desc"]), - "source": cached_pos.get("source", "Cached OSINT"), - "updated": cached_pos.get("updated", "") - }) + if cached_pos.get("source", "").startswith("GDELT") or cached_pos.get( + "source", "" + ).startswith("News"): + positions[hull].update( + { + "lat": cached_pos["lat"], + "lng": cached_pos["lng"], + "desc": cached_pos.get("desc", positions[hull]["desc"]), + "source": cached_pos.get("source", "Cached OSINT"), + "updated": cached_pos.get("updated", ""), + } + ) return positions @@ -371,7 +426,9 @@ def update_carrier_positions(): if not _carrier_positions: _carrier_positions.update(positions) _last_update = datetime.now(timezone.utc) - logger.info(f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)") + logger.info( + f"Carrier tracker: {len(positions)} carriers loaded from fallback/cache (GDELT enrichment starting...)" + ) # --- Phase 2: slow GDELT enrichment --- try: @@ -408,6 +465,7 @@ def _deconflict_positions(result: List[dict]) -> List[dict]: """ # Group by rounded lat/lng (within ~0.01° ≈ 1km = same spot) from collections import defaultdict + groups: dict[str, list[int]] = defaultdict(list) for i, c in enumerate(result): key = f"{round(c['lat'], 2)},{round(c['lng'], 2)}" @@ -454,22 +512,26 @@ def get_carrier_positions() -> List[dict]: result = [] for hull, pos in _carrier_positions.items(): info = CARRIER_REGISTRY.get(hull, {}) - result.append({ - "name": pos.get("name", info.get("name", hull)), - "type": "carrier", - "lat": pos["lat"], - "lng": pos["lng"], - "heading": None, # Heading unknown for carriers — OSINT cannot determine true heading - "sog": 0, - "cog": 0, - "country": "United States", - "desc": pos.get("desc", ""), - "wiki": pos.get("wiki", info.get("wiki", "")), - "estimated": True, - "source": pos.get("source", "OSINT estimated position"), - "source_url": pos.get("source_url", "https://news.usni.org/category/fleet-tracker"), - "last_osint_update": pos.get("updated", "") - }) + result.append( + { + "name": pos.get("name", info.get("name", hull)), + "type": "carrier", + "lat": pos["lat"], + "lng": pos["lng"], + "heading": None, # Heading unknown for carriers — OSINT cannot determine true heading + "sog": 0, + "cog": 0, + "country": "United States", + "desc": pos.get("desc", ""), + "wiki": pos.get("wiki", info.get("wiki", "")), + "estimated": True, + "source": pos.get("source", "OSINT estimated position"), + "source_url": pos.get( + "source_url", "https://news.usni.org/category/fleet-tracker" + ), + "last_osint_update": pos.get("updated", ""), + } + ) return _deconflict_positions(result) @@ -500,10 +562,13 @@ def _scheduler_loop(): next_run = now.replace(hour=next_hour % 24, minute=0, second=0, microsecond=0) if next_hour == 24: from datetime import timedelta + next_run = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0) wait_seconds = (next_run - now).total_seconds() - logger.info(f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)") + logger.info( + f"Carrier tracker: next update at {next_run.isoformat()} ({wait_seconds/3600:.1f}h)" + ) # Wait until next scheduled time, or until stop event if _scheduler_stop.wait(timeout=wait_seconds): @@ -521,7 +586,9 @@ def start_carrier_tracker(): if _scheduler_thread and _scheduler_thread.is_alive(): return _scheduler_stop.clear() - _scheduler_thread = threading.Thread(target=_scheduler_loop, daemon=True, name="carrier-tracker") + _scheduler_thread = threading.Thread( + target=_scheduler_loop, daemon=True, name="carrier-tracker" + ) _scheduler_thread.start() logger.info("Carrier tracker started") diff --git a/backend/services/cctv_pipeline.py b/backend/services/cctv_pipeline.py index 9be09a8f..246cde26 100644 --- a/backend/services/cctv_pipeline.py +++ b/backend/services/cctv_pipeline.py @@ -1,42 +1,123 @@ -import logging -import re +import os import sqlite3 -import xml.etree.ElementTree as ET -from abc import ABC, abstractmethod +import requests +import re from pathlib import Path -from typing import Any, Dict, List - +from urllib.parse import urljoin, urlparse, urlunparse, quote from services.network_utils import fetch_with_curl +import logging +from abc import ABC, abstractmethod +from typing import List, Dict, Any logger = logging.getLogger(__name__) DB_PATH = Path(__file__).resolve().parent.parent / "data" / "cctv.db" - -def _connect() -> sqlite3.Connection: - DB_PATH.parent.mkdir(parents=True, exist_ok=True) - return sqlite3.connect(str(DB_PATH)) +_KNOWN_CCTV_MEDIA_HOST_ALIASES = { + # Trusted upstream occasionally publishes a typo for this Georgia camera + # host. Normalize it at ingest so the proxy and client stay consistent. + "navigatos-c2c.dot.ga.gov": "navigator-c2c.dot.ga.gov", +} + +_POINT_WKT_RE = re.compile( + r"POINT\s*\(\s*([-+]?\d+(?:\.\d+)?)\s+([-+]?\d+(?:\.\d+)?)\s*\)", + re.IGNORECASE, +) + + +def _normalize_cctv_media_url(raw_url: str) -> str: + candidate = str(raw_url or "").strip() + if not candidate: + return "" + parsed = urlparse(candidate) + host = str(parsed.hostname or "").strip().lower() + replacement = _KNOWN_CCTV_MEDIA_HOST_ALIASES.get(host) + if not replacement: + return candidate + netloc = replacement + if parsed.port: + netloc = f"{replacement}:{parsed.port}" + return urlunparse(parsed._replace(netloc=netloc)) + + +def _looks_like_direct_cctv_media_url(url: str) -> bool: + candidate = str(url or "").strip().lower() + if not candidate.startswith(("http://", "https://")): + return False + parsed = urlparse(candidate) + path = str(parsed.path or "").lower() + if any(path.endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".gif", ".webp", ".mp4", ".webm", ".m3u8", ".mjpg", ".mjpeg")): + return True + return any(token in candidate for token in ("snapshot", "/image/", "playlist.m3u8", "mjpg", "mjpeg")) + + +def _extract_direct_cctv_media_from_tags(tags: Dict[str, Any]) -> tuple[str, str]: + for key in ("camera:url", "camera:image", "image", "url", "website"): + raw = _normalize_cctv_media_url(str(tags.get(key) or "").strip()) + if not raw: + continue + if key in {"url", "website"} and not _looks_like_direct_cctv_media_url(raw): + continue + media_type = _detect_media_type(raw) + if key in {"camera:image", "image"} and media_type == "image": + return raw, "image" + if media_type in {"video", "hls", "mjpeg"} or _looks_like_direct_cctv_media_url(raw): + return raw, media_type or "image" + return "", "image" + + +def _media_url_reachable(url: str, *, timeout: int = 8, headers: Dict[str, str] | None = None) -> bool: + candidate = _normalize_cctv_media_url(str(url or "").strip()) + if not candidate: + return False + try: + resp = fetch_with_curl(candidate, timeout=timeout, headers=headers or {}) + except Exception as exc: + logger.debug(f"CCTV media probe failed for {candidate}: {exc}") + return False + return bool(resp and int(getattr(resp, "status_code", 500) or 500) < 400) + + +def _parse_wkt_point(raw_point: str) -> tuple[float | None, float | None]: + candidate = str(raw_point or "").strip() + if not candidate: + return None, None + match = _POINT_WKT_RE.search(candidate) + if not match: + return None, None + try: + lon = float(match.group(1)) + lat = float(match.group(2)) + except (TypeError, ValueError): + return None, None + return lat, lon def init_db(): - conn = _connect() - try: - cursor = conn.cursor() - cursor.execute(""" - CREATE TABLE IF NOT EXISTS cameras ( - id TEXT PRIMARY KEY, - source_agency TEXT, - lat REAL, - lon REAL, - direction_facing TEXT, - media_url TEXT, - refresh_rate_seconds INTEGER, - last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.commit() - finally: - conn.close() + DB_PATH.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(DB_PATH)) + cursor = conn.cursor() + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS cameras ( + id TEXT PRIMARY KEY, + source_agency TEXT, + lat REAL, + lon REAL, + direction_facing TEXT, + media_url TEXT, + media_type TEXT, + refresh_rate_seconds INTEGER, + last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + cursor.execute("PRAGMA table_info(cameras)") + columns = {str(row[1]) for row in cursor.fetchall()} + if "media_type" not in columns: + cursor.execute("ALTER TABLE cameras ADD COLUMN media_type TEXT") + conn.commit() + conn.close() class BaseCCTVIngestor(ABC): @@ -45,20 +126,41 @@ def fetch_data(self) -> List[Dict[str, Any]]: pass def ingest(self): - conn = None + conn = sqlite3.connect(str(DB_PATH)) try: - init_db() cameras = self.fetch_data() - conn = _connect() cursor = conn.cursor() + source_prefixes = { + str(cam.get("id") or "").split("-", 1)[0] + for cam in cameras + if str(cam.get("id") or "").strip() + } + if cameras and len(source_prefixes) == 1: + prefix = next(iter(source_prefixes)) + cursor.execute("DELETE FROM cameras WHERE id LIKE ?", (f"{prefix}-%",)) for cam in cameras: cursor.execute( """ INSERT INTO cameras - (id, source_agency, lat, lon, direction_facing, media_url, refresh_rate_seconds) - VALUES (?, ?, ?, ?, ?, ?, ?) + ( + id, + source_agency, + lat, + lon, + direction_facing, + media_url, + media_type, + refresh_rate_seconds + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET + source_agency=excluded.source_agency, + lat=excluded.lat, + lon=excluded.lon, + direction_facing=excluded.direction_facing, media_url=excluded.media_url, + media_type=excluded.media_type, + refresh_rate_seconds=excluded.refresh_rate_seconds, last_updated=CURRENT_TIMESTAMP """, ( @@ -68,6 +170,7 @@ def ingest(self): cam.get("lon"), cam.get("direction_facing", "Unknown"), cam.get("media_url"), + cam.get("media_type", _detect_media_type(cam.get("media_url", ""))), cam.get("refresh_rate_seconds", 60), ), ) @@ -77,14 +180,12 @@ def ingest(self): ) except Exception as e: try: - if conn is not None: - conn.rollback() + conn.rollback() except Exception: pass logger.error(f"Failed to ingest cameras in {self.__class__.__name__}: {e}") finally: - if conn is not None: - conn.close() + conn.close() class TFLJamCamIngestor(BaseCCTVIngestor): @@ -117,6 +218,7 @@ def fetch_data(self) -> List[Dict[str, Any]]: "lon": item.get("lon"), "direction_facing": item.get("commonName", "Unknown"), "media_url": media, + "media_type": "video" if vid_url else "image", "refresh_rate_seconds": 15, } ) @@ -144,6 +246,7 @@ def fetch_data(self) -> List[Dict[str, Any]]: "lon": loc.get("longitude"), "direction_facing": f"Camera {item.get('camera_id')}", "media_url": item.get("image"), + "media_type": "image", "refresh_rate_seconds": 60, } ) @@ -163,9 +266,15 @@ def fetch_data(self) -> List[Dict[str, Any]]: cam_id = item.get("camera_id") if not cam_id: continue + status = str(item.get("camera_status") or "").strip().upper() + if status and status != "TURNED_ON": + continue loc = item.get("location", {}) coords = loc.get("coordinates", []) + screenshot = _normalize_cctv_media_url(str(item.get("screenshot_address") or "").strip()) + if not screenshot: + screenshot = f"https://cctv.austinmobility.io/image/{cam_id}.jpg" # coords is usually [lon, lat] if len(coords) == 2: @@ -175,10 +284,9 @@ def fetch_data(self) -> List[Dict[str, Any]]: "source_agency": "Austin TxDOT", "lat": coords[1], "lon": coords[0], - "direction_facing": item.get( - "location_name", "Austin TX Camera" - ), - "media_url": f"https://cctv.austinmobility.io/image/{cam_id}.jpg", + "direction_facing": item.get("location_name", "Austin TX Camera"), + "media_url": screenshot, + "media_type": "image", "refresh_rate_seconds": 60, } ) @@ -209,390 +317,663 @@ def fetch_data(self) -> List[Dict[str, Any]]: "lon": lon, "direction_facing": item.get("name", "NYC Camera"), "media_url": f"https://webcams.nyctmc.org/api/cameras/{cam_id}/image", + "media_type": "image", "refresh_rate_seconds": 30, } ) return cameras -class GlobalOSMCrawlingIngestor(BaseCCTVIngestor): +class CaltransIngestor(BaseCCTVIngestor): + """Caltrans highway cameras across all 12 California districts.""" + + DISTRICTS = list(range(1, 13)) + BASE_URL = "https://cwwp2.dot.ca.gov/data/d{d}/cctv/cctvStatusD{d:02d}.json" + def fetch_data(self) -> List[Dict[str, Any]]: - # This will pull physical street surveillance cameras across all global hotspots - # using OpenStreetMap Overpass mapping their exact geospatial coordinates to Google Street View - regions = [ - ("35.6,139.6,35.8,139.8", "Tokyo"), - ("48.8,2.3,48.9,2.4", "Paris"), - ("40.6,-74.1,40.8,-73.9", "NYC Expanded"), - ("34.0,-118.4,34.2,-118.2", "Los Angeles"), - ("-33.9,151.1,-33.7,151.3", "Sydney"), - ("52.4,13.3,52.6,13.5", "Berlin"), - ("25.1,55.2,25.3,55.4", "Dubai"), - ("19.3,-99.2,19.5,-99.0", "Mexico City"), - ("-23.6,-46.7,-23.4,-46.5", "Sao Paulo"), - ("39.6,-105.1,39.9,-104.8", "Denver"), - ] - - query_parts = [ - f'node["man_made"="surveillance"]({bbox});' for bbox, city in regions - ] - query = "".join(query_parts) - url = f"https://overpass-api.de/api/interpreter?data=[out:json];({query});out%202000;" + cameras = [] + for district in self.DISTRICTS: + try: + url = self.BASE_URL.format(d=district) + resp = fetch_with_curl(url, timeout=15) + if not resp or resp.status_code != 200: + continue + data = resp.json() + entries = data.get("data", data) + if not isinstance(entries, list): + continue - try: - response = fetch_with_curl(url, timeout=15) - response.raise_for_status() - data = response.json() - - cameras = [] - for item in data.get("elements", []): - lat = item.get("lat") - lon = item.get("lon") - cam_id = item.get("id") - - if lat and lon: - # Find which city this belongs to - source_city = "Global OSINT" - for bbox, city in regions: - s, w, n, e = map(float, bbox.split(",")) - if s <= lat <= n and w <= lon <= e: - source_city = f"OSINT: {city}" - break - - # Attempt to parse camera direction for a cool realistic bearing angle if OSM mapped it - direction_str = item.get("tags", {}).get("camera:direction", "0") + for wrapper in entries: + entry = wrapper.get("cctv", wrapper) if isinstance(wrapper, dict) else None + if not isinstance(entry, dict): + continue + + loc = entry.get("location", {}) + lat_s = loc.get("latitude") + lon_s = loc.get("longitude") + if not lat_s or not lon_s: + continue try: - bearing = int(float(direction_str)) + lat, lon = float(lat_s), float(lon_s) except (ValueError, TypeError): - bearing = 0 - - mapbox_key = "YOUR_MAPBOX_TOKEN_HERE" - mapbox_url = f"https://api.mapbox.com/styles/v1/mapbox/satellite-streets-v12/static/{lon},{lat},18,{bearing},60/600x400?access_token={mapbox_key}" - + continue + if abs(lat) > 90 or abs(lon) > 180: + continue + + if entry.get("inService") == "false": + continue + + img_data = entry.get("imageData", {}) + streaming = str(img_data.get("streamingVideoURL") or "").strip() + streaming = urljoin(url, streaming) if streaming else "" + static_image = str(img_data.get("static", {}).get("currentImageURL") or "").strip() + static_image = urljoin(url, static_image) if static_image else "" + streaming_type = _detect_media_type(streaming) + if static_image: + media = static_image + media_type = "image" + elif streaming and streaming_type in {"video", "hls", "mjpeg"}: + media = streaming + media_type = streaming_type + else: + media = streaming + media_type = streaming_type or "image" + if not media: + continue + + idx = entry.get("index", len(cameras)) cameras.append( { - "id": f"OSM-{cam_id}", - "source_agency": source_city, + "id": f"CAL-D{district:02d}-{idx}", + "source_agency": f"Caltrans D{district:02d}", "lat": lat, "lon": lon, - "direction_facing": item.get("tags", {}).get( - "surveillance:type", "Street Level Camera" - ), - "media_url": mapbox_url, - "refresh_rate_seconds": 3600, + "direction_facing": ( + loc.get("locationName") + or loc.get("nearbyPlace") + or f"CA-{loc.get('route', '?')}" + )[:120], + "media_url": media, + "media_type": media_type, + "refresh_rate_seconds": 60, } ) - return cameras - except Exception: - return [] + except Exception as e: + logger.warning(f"Caltrans D{district:02d} fetch error: {e}") + return cameras -# --------------------------------------------------------------------------- -# Spain — DGT National Roads (DATEX2 XML, ~1,900 cameras) -# --------------------------------------------------------------------------- -class SpainDGTIngestor(BaseCCTVIngestor): - # Dirección General de Tráfico — national road cameras via DATEX2 v3 XML. - # No API key required. Covers all national roads (autopistas, autovías, N-roads) - # EXCEPT Basque Country and Catalonia. - # Published under Spain's open data framework (Ley 37/2007, EU PSI Directive 2019/1024). - DGT_URL = "https://nap.dgt.es/datex2/v3/dgt/DevicePublication/camaras_datex2_v36.xml" +class WSDOTIngestor(BaseCCTVIngestor): + """Washington State DOT cameras via ArcGIS REST (1,500+ cameras).""" - def fetch_data(self) -> List[Dict[str, Any]]: - try: - response = fetch_with_curl(self.DGT_URL, timeout=30) - response.raise_for_status() - except Exception as e: - logger.error(f"SpainDGTIngestor: failed to fetch DATEX2 XML: {e}") - return [] + URL = ( + "https://www.wsdot.wa.gov/arcgis/rest/services/Production/" + "WSDOTTrafficCameras/MapServer/0/query" + ) - try: - root = ET.fromstring(response.content) - except ET.ParseError as e: - logger.error(f"SpainDGTIngestor: failed to parse XML: {e}") + def fetch_data(self) -> List[Dict[str, Any]]: + resp = fetch_with_curl( + self.URL + "?where=1%3D1" + "&outFields=CameraID,CameraTitl,ImageURL,CameraOwne" + "&outSR=4326&f=json", + timeout=25, + ) + if not resp or resp.status_code != 200: + logger.error(f"WSDOT fetch failed: HTTP {resp.status_code if resp else 'no response'}") return [] - + data = resp.json() cameras = [] - # DGT DATEX2 v3 uses elements with typeOfDevice=camera. - # Namespace-agnostic: match local name "device". - for el in root.iter(): - local = el.tag.split("}")[-1] if "}" in el.tag else el.tag - if local != "device": + for feat in data.get("features", []): + attrs = feat.get("attributes", {}) + geom = feat.get("geometry", {}) + cam_id = attrs.get("CameraID") + lat = geom.get("y") + lon = geom.get("x") + img = attrs.get("ImageURL") + if not (cam_id and lat and lon and img): continue - try: - cam_id = el.get("id", "") - if not cam_id: - continue + lat, lon = float(lat), float(lon) + except (ValueError, TypeError): + continue + cameras.append( + { + "id": f"WSDOT-{cam_id}", + "source_agency": (attrs.get("CameraOwne") or "WSDOT")[:60], + "lat": lat, + "lon": lon, + "direction_facing": (attrs.get("CameraTitl") or "WA Camera")[:120], + "media_url": img, + "media_type": "image", + "refresh_rate_seconds": 120, + } + ) + return cameras + + +class GeorgiaDOTIngestor(BaseCCTVIngestor): + """Georgia cameras via the public 511GA list feed.""" + + URL = "https://511ga.org/List/GetData/Cameras" + BASE_URL = "https://511ga.org" + PAGE_SIZE = 500 - # Coordinates are nested: pointLocation > ... > pointCoordinates > latitude/longitude - lat = self._find_text(el, "latitude") - lon = self._find_text(el, "longitude") - if not lat or not lon: + def fetch_data(self) -> List[Dict[str, Any]]: + cameras = [] + start = 0 + draw = 1 + while True: + resp = fetch_with_curl( + self.URL, + method="POST", + json_data={"draw": draw, "start": start, "length": self.PAGE_SIZE}, + timeout=30, + headers={ + "Accept": "application/json", + "Referer": "https://511ga.org/cctv", + "Origin": "https://511ga.org", + }, + ) + if not resp or resp.status_code != 200: + logger.error( + "Georgia CCTV fetch failed: HTTP %s", + resp.status_code if resp else "no response", + ) + break + data = resp.json() + rows = data.get("data") or [] + if not rows: + break + for row in rows: + site_id = row.get("id") or row.get("DT_RowId") + location = row.get("location") or row.get("roadway") or "GA Camera" + lat_lng = row.get("latLng") or {} + geography = lat_lng.get("geography") if isinstance(lat_lng, dict) else {} + lat, lon = _parse_wkt_point(geography.get("wellKnownText") if isinstance(geography, dict) else "") + images = row.get("images") or [] + image = next( + ( + candidate + for candidate in images + if str(candidate.get("imageUrl") or "").strip() + and not bool(candidate.get("blocked")) + ), + None, + ) + if not (site_id and image and lat is not None and lon is not None): continue + media_url = _normalize_cctv_media_url( + urljoin(self.BASE_URL, str(image.get("imageUrl") or "").strip()) + ) + cameras.append( + { + "id": f"GDOT-{site_id}", + "source_agency": "Georgia DOT", + "lat": lat, + "lon": lon, + "direction_facing": str(location)[:120], + "media_url": media_url, + "media_type": "image", + "refresh_rate_seconds": 60, + } + ) + start += len(rows) + draw += 1 + total = int(data.get("recordsTotal") or 0) + if total and start >= total: + break + if not total and len(rows) < self.PAGE_SIZE: + break + return cameras - image_url = self._find_text(el, "deviceUrl") or f"https://infocar.dgt.es/etraffic/data/camaras/{cam_id}.jpg" - road_name = self._find_text(el, "roadName") or "" - road_dest = self._find_text(el, "roadDestination") or "" - description = f"{road_name} → {road_dest}".strip(" →") or f"DGT Camera {cam_id}" +class IllinoisDOTIngestor(BaseCCTVIngestor): + """Illinois DOT cameras via ArcGIS FeatureServer (3,400+ cameras).""" - cameras.append({ - "id": f"DGT-{cam_id}", - "source_agency": "DGT Spain", - "lat": float(lat), - "lon": float(lon), - "direction_facing": description, - "media_url": image_url, - "refresh_rate_seconds": 300, - }) - except (ValueError, TypeError) as e: - logger.debug(f"SpainDGTIngestor: skipping malformed record: {e}") - continue + URL = ( + "https://services2.arcgis.com/aIrBD8yn1TDTEXoz/arcgis/rest/services/" + "TrafficCamerasTM_Public/FeatureServer/0/query" + ) - logger.info(f"SpainDGTIngestor: parsed {len(cameras)} cameras") + def fetch_data(self) -> List[Dict[str, Any]]: + resp = fetch_with_curl( + self.URL + "?where=1%3D1" + "&outFields=CameraLocation,CameraDirection,SnapShot" + "&outSR=4326&f=json", + timeout=30, + ) + if not resp or resp.status_code != 200: + return [] + data = resp.json() + cameras = [] + for feat in data.get("features", []): + attrs = feat.get("attributes", {}) + geom = feat.get("geometry", {}) + lat = geom.get("y") + lon = geom.get("x") + img = attrs.get("SnapShot") or "" + if not (lat and lon and img): + continue + try: + lat, lon = float(lat), float(lon) + except (ValueError, TypeError): + continue + cameras.append({ + "id": f"IDOT-{len(cameras)}", + "source_agency": "Illinois DOT", + "lat": lat, "lon": lon, + "direction_facing": ( + attrs.get("CameraLocation") or attrs.get("CameraDirection") or "IL Camera" + )[:120], + "media_url": img, + "media_type": "image", + "refresh_rate_seconds": 120, + }) return cameras - @staticmethod - def _find_text(element: ET.Element, tag: str) -> str | None: - for child in element.iter(): - local = child.tag.split("}")[-1] if "}" in child.tag else child.tag - if local.lower() == tag.lower() and child.text: - return child.text.strip() - return None +class MichiganDOTIngestor(BaseCCTVIngestor): + """Michigan DOT cameras (775+ cameras). Parses HTML-embedded JSON.""" -# --------------------------------------------------------------------------- -# Spain — Madrid City Hall (KML, ~200 cameras) -# --------------------------------------------------------------------------- -class MadridCCTVIngestor(BaseCCTVIngestor): - # Madrid City Hall urban traffic cameras via open data KML. - # No API key required. Published on datos.madrid.es. - # Licence: Madrid Open Data (free reuse with attribution). - MADRID_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml" + URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list" def fetch_data(self) -> List[Dict[str, Any]]: - try: - response = fetch_with_curl(self.MADRID_URL, timeout=20) - response.raise_for_status() - except Exception as e: - logger.error(f"MadridCCTVIngestor: failed to fetch KML: {e}") + import re + resp = fetch_with_curl(self.URL, timeout=20) + if not resp or resp.status_code != 200: return [] + data = resp.json() + cameras = [] + for cam in data: + county = cam.get("county", "") + m = re.search(r"lat=([\d.-]+)&lon=([\d.-]+)", county) + if not m: + continue + try: + lat, lon = float(m.group(1)), float(m.group(2)) + except (ValueError, TypeError): + continue + img_m = re.search(r'src="([^"]+)"', cam.get("image", "")) + if not img_m: + continue + id_m = re.search(r"id=(\d+)", county) + cam_id = id_m.group(1) if id_m else str(len(cameras)) + media_url = urljoin(self.URL, img_m.group(1)) + cameras.append({ + "id": f"MDOT-{cam_id}", + "source_agency": "Michigan DOT", + "lat": lat, "lon": lon, + "direction_facing": ( + f"{cam.get('route', '')} {cam.get('location', '')}".strip() or "MI Camera" + )[:120], + "media_url": media_url, + "media_type": "image", + "refresh_rate_seconds": 120, + }) + return cameras - try: - root = ET.fromstring(response.content) - except ET.ParseError as e: - logger.error(f"MadridCCTVIngestor: failed to parse KML: {e}") + +class WindyWebcamsIngestor(BaseCCTVIngestor): + """Windy Webcams API v3 — global cameras. Requires WINDY_API_KEY env var.""" + + BASE = "https://api.windy.com/webcams/api/v3/webcams" + + def fetch_data(self) -> List[Dict[str, Any]]: + api_key = os.environ.get("WINDY_API_KEY", "") + if not api_key: return [] cameras = [] - # KML namespace varies — try both common ones, then fall back to tag-name search - placemarks = root.findall(".//{http://www.opengis.net/kml/2.2}Placemark") - if not placemarks: - placemarks = root.findall(".//{http://earth.google.com/kml/2.2}Placemark") - if not placemarks: - placemarks = [el for el in root.iter() if el.tag.endswith("Placemark")] + offset = 0 + limit = 50 + max_cameras = 1000 # Free tier offset cap - for i, pm in enumerate(placemarks): + while offset < max_cameras: try: - name = self._find_kml_text(pm, "name") or f"Madrid Camera {i}" - coords_text = self._find_kml_text(pm, "coordinates") - if not coords_text: - continue - - # KML coordinates: lon,lat,elevation - parts = coords_text.strip().split(",") - if len(parts) < 2: - continue - lon, lat = float(parts[0]), float(parts[1]) - - # Extract image URL from description CDATA - desc = self._find_kml_text(pm, "description") or "" - image_url = self._extract_img_src(desc) - if not image_url: - continue + resp = requests.get( + self.BASE, + params={"limit": limit, "offset": offset, "include": "location,images"}, + headers={ + "X-WINDY-API-KEY": api_key, + "Accept": "application/json", + }, + timeout=20, + ) + if resp.status_code != 200: + break + data = resp.json() + webcams = data.get("webcams", []) + if not webcams: + break + + for wc in webcams: + loc = wc.get("location", {}) + lat = loc.get("latitude") + lon = loc.get("longitude") + if lat is None or lon is None: + continue + try: + lat, lon = float(lat), float(lon) + except (ValueError, TypeError): + continue - cameras.append({ - "id": f"MAD-{i:04d}", - "source_agency": "Madrid City Hall", - "lat": lat, - "lon": lon, - "direction_facing": name, - "media_url": image_url, - "refresh_rate_seconds": 600, - }) - except (ValueError, TypeError, IndexError) as e: - logger.debug(f"MadridCCTVIngestor: skipping malformed placemark: {e}") - continue + images = wc.get("images", {}) + current = images.get("current", {}) + img_url = current.get("preview") or current.get("thumbnail") or "" - logger.info(f"MadridCCTVIngestor: parsed {len(cameras)} cameras") + city = loc.get("city") or loc.get("country") or "Global" + cameras.append( + { + "id": f"WINDY-{wc.get('webcamId', offset)}", + "source_agency": f"Windy: {city}"[:60], + "lat": lat, + "lon": lon, + "direction_facing": (wc.get("title") or "Webcam")[:120], + "media_url": img_url, + "media_type": "image", + "refresh_rate_seconds": 600, + } + ) + offset += limit + except Exception as e: + logger.warning(f"Windy webcams fetch error at offset {offset}: {e}") + break return cameras - @staticmethod - def _find_kml_text(element: ET.Element, tag: str) -> str | None: - for child in element.iter(): - local = child.tag.split("}")[-1] if "}" in child.tag else child.tag - if local == tag and child.text: - return child.text.strip() - return None - - @staticmethod - def _extract_img_src(html_fragment: str) -> str | None: - match = re.search(r'src=["\']([^"\']+)["\']', html_fragment, re.IGNORECASE) - if match: - return match.group(1) - match = re.search(r'https?://\S+\.jpg', html_fragment, re.IGNORECASE) - if match: - return match.group(0) - return None +class ColoradoDOTIngestor(BaseCCTVIngestor): + """Colorado DOT cameras via the official COtrip camera service.""" -# --------------------------------------------------------------------------- -# Spain — Málaga (GeoJSON, ~134 cameras) -# --------------------------------------------------------------------------- -class MalagaCCTVIngestor(BaseCCTVIngestor): - # Málaga open data — traffic cameras in EPSG:4326 GeoJSON. - # No API key required. Published on datosabiertos.malaga.eu. - MALAGA_URL = "https://datosabiertos.malaga.eu/recursos/transporte/trafico/da_camarasTrafico-4326.geojson" + URL = "https://cotg.carsprogram.org/cameras_v1/api/cameras" def fetch_data(self) -> List[Dict[str, Any]]: - try: - response = fetch_with_curl(self.MALAGA_URL, timeout=15) - response.raise_for_status() - data = response.json() - except Exception as e: - logger.error(f"MalagaCCTVIngestor: failed to fetch GeoJSON: {e}") + resp = fetch_with_curl( + self.URL, + timeout=25, + headers={"Accept": "application/json"}, + ) + if not resp or resp.status_code != 200: + logger.warning(f"Colorado DOT camera fetch failed: HTTP {resp.status_code if resp else 'no response'}") return [] - + data = resp.json() cameras = [] - for feature in data.get("features", []): + for item in data if isinstance(data, list) else []: + if item.get("public") is False or item.get("active") is False: + continue + loc = item.get("location", {}) + lat = loc.get("latitude") + lon = loc.get("longitude") + if lat is None or lon is None: + continue try: - props = feature.get("properties", {}) - geom = feature.get("geometry", {}) - coords = geom.get("coordinates", []) - if len(coords) < 2: - continue - - image_url = props.get("URLIMAGEN") or props.get("urlimagen") - if not image_url: - continue - - cam_id = props.get("NOMBRE") or props.get("nombre") or str(coords) - description = props.get("DESCRIPCION") or props.get("descripcion") or cam_id + lat, lon = float(lat), float(lon) + except (ValueError, TypeError): + continue - cameras.append({ - "id": f"MLG-{cam_id}", - "source_agency": "Málaga City", - "lat": float(coords[1]), - "lon": float(coords[0]), - "direction_facing": description, - "media_url": image_url, - "refresh_rate_seconds": 300, - }) - except (ValueError, TypeError, IndexError) as e: - logger.debug(f"MalagaCCTVIngestor: skipping malformed feature: {e}") + media_url = "" + media_type = "image" + for view in item.get("views") or []: + preview_url = _normalize_cctv_media_url(str(view.get("videoPreviewUrl") or "").strip()) + if preview_url: + media_url = preview_url + media_type = "image" + break + if not media_url: + for view in item.get("views") or []: + stream_url = _normalize_cctv_media_url(str(view.get("url") or "").strip()) + stream_type = _detect_media_type(stream_url) + if stream_url and stream_type in {"video", "hls", "mjpeg"}: + media_url = stream_url + media_type = stream_type + break + if not media_url: continue - logger.info(f"MalagaCCTVIngestor: parsed {len(cameras)} cameras") + owner = item.get("cameraOwner", {}) + cameras.append( + { + "id": f"CODOT-{item.get('id')}", + "source_agency": str(owner.get("name") or "Colorado DOT")[:60], + "lat": lat, + "lon": lon, + "direction_facing": str(item.get("name") or loc.get("routeId") or "Colorado Camera")[:120], + "media_url": media_url, + "media_type": media_type, + "refresh_rate_seconds": 30 if media_type in {"video", "hls"} else 60, + } + ) return cameras -# --------------------------------------------------------------------------- -# Spain — Vigo (GeoJSON, ~59 cameras) -# --------------------------------------------------------------------------- -class VigoCCTVIngestor(BaseCCTVIngestor): - # Vigo open data — traffic cameras in GeoJSON. - # No API key required. Published on datos.vigo.org. - VIGO_URL = "https://datos.vigo.org/data/trafico/camaras-trafico.geojson" +class OSMTrafficCameraIngestor(BaseCCTVIngestor): + """Traffic cameras from OpenStreetMap/Overpass with direct public media URLs.""" + + URL = "https://overpass-api.de/api/interpreter" + QUERY = """ +[out:json][timeout:30]; +( + node["camera:type"="traffic_monitoring"]["camera:url"]; + node["camera:type"="traffic_monitoring"]["camera:image"]; + node["camera:type"="traffic_monitoring"]["image"]; + node["camera:type"="traffic_monitoring"]["url"]; + node["surveillance:type"="traffic_monitoring"]["camera:url"]; + node["surveillance:type"="traffic_monitoring"]["camera:image"]; + node["surveillance:type"="traffic_monitoring"]["image"]; + node["surveillance:type"="traffic_monitoring"]["url"]; + node["man_made"="surveillance"]["camera:type"="traffic_monitoring"]["camera:url"]; + node["man_made"="surveillance"]["camera:type"="traffic_monitoring"]["camera:image"]; + node["man_made"="surveillance"]["camera:type"="traffic_monitoring"]["image"]; + node["man_made"="surveillance"]["camera:type"="traffic_monitoring"]["url"]; +); +out body; +""".strip() def fetch_data(self) -> List[Dict[str, Any]]: - try: - response = fetch_with_curl(self.VIGO_URL, timeout=15) - response.raise_for_status() - data = response.json() - except Exception as e: - logger.error(f"VigoCCTVIngestor: failed to fetch GeoJSON: {e}") + query = quote(self.QUERY, safe="") + resp = fetch_with_curl( + f"{self.URL}?data={query}", + timeout=35, + headers={"Accept": "application/json"}, + ) + if not resp or resp.status_code != 200: + logger.warning(f"OSM camera fetch failed: HTTP {resp.status_code if resp else 'no response'}") return [] - + data = resp.json() cameras = [] - for feature in data.get("features", []): + for item in data.get("elements", []) if isinstance(data, dict) else []: + lat = item.get("lat") + lon = item.get("lon") + tags = item.get("tags", {}) if isinstance(item.get("tags"), dict) else {} + if lat is None or lon is None: + continue try: - props = feature.get("properties", {}) - geom = feature.get("geometry", {}) - coords = geom.get("coordinates", []) - if len(coords) < 2: - continue - - # Vigo uses PHP image endpoints - image_url = props.get("urlimagen") or props.get("URLIMAGEN") or props.get("url") - if not image_url: - continue + lat, lon = float(lat), float(lon) + except (ValueError, TypeError): + continue - cam_id = props.get("id") or props.get("nombre") or str(coords) - description = props.get("nombre") or props.get("descripcion") or f"Vigo Camera {cam_id}" + media_url, media_type = _extract_direct_cctv_media_from_tags(tags) + if not media_url: + continue - cameras.append({ - "id": f"VGO-{cam_id}", - "source_agency": "Vigo City", - "lat": float(coords[1]), - "lon": float(coords[0]), - "direction_facing": description, - "media_url": image_url, + direction = ( + tags.get("camera:direction") + or tags.get("direction") + or tags.get("surveillance:direction") + or tags.get("name") + or "OSM Traffic Camera" + ) + operator = tags.get("operator") or tags.get("network") or tags.get("brand") or "OpenStreetMap" + cameras.append( + { + "id": f"OSM-{item.get('id')}", + "source_agency": str(operator)[:60], + "lat": lat, + "lon": lon, + "direction_facing": str(direction)[:120], + "media_url": media_url, + "media_type": media_type or "image", "refresh_rate_seconds": 300, - }) - except (ValueError, TypeError, IndexError) as e: - logger.debug(f"VigoCCTVIngestor: skipping malformed feature: {e}") - continue + } + ) + return cameras + + - logger.info(f"VigoCCTVIngestor: parsed {len(cameras)} cameras") +# --------------------------------------------------------------------------- +# DGT Spain — National Road Cameras +# --------------------------------------------------------------------------- +# Image URL pattern confirmed working: infocar.dgt.es/etraffic/data/camaras/{id}.jpg +# Source: DGT (Dirección General de Tráfico) — public open data (Ley 37/2007). +# Author credit: Alborz Nazari (github.com/AlborzNazari) — PR #91 + +class DGTNationalIngestor(BaseCCTVIngestor): + """DGT national road cameras — 20 seed cameras across Spanish motorways.""" + + KNOWN_CAMERAS = [ + (1398, 36.7213, -4.4214, "MA-19 Málaga"), + (1001, 40.4168, -3.7038, "A-6 Madrid"), + (1002, 40.4500, -3.6800, "A-2 Madrid"), + (1003, 40.3800, -3.7200, "A-4 Madrid"), + (1004, 40.4200, -3.8100, "A-5 Madrid"), + (1005, 40.4600, -3.6600, "M-30 Madrid"), + (1010, 41.3888, 2.1590, "AP-7 Barcelona"), + (1011, 41.4100, 2.1800, "A-2 Barcelona"), + (1020, 37.3891, -5.9845, "A-4 Sevilla"), + (1021, 37.4000, -6.0000, "A-49 Sevilla"), + (1030, 39.4699, -0.3763, "V-30 Valencia"), + (1031, 39.4800, -0.3900, "A-3 Valencia"), + (1040, 43.2630, -2.9350, "A-8 Bilbao"), + (1050, 42.8782, -8.5448, "AG-55 Santiago"), + (1060, 41.6488, -0.8891, "A-2 Zaragoza"), + (1070, 37.9922, -1.1307, "A-30 Murcia"), + (1080, 36.5271, -6.2886, "A-4 Cádiz"), + (1090, 43.3623, -8.4115, "A-6 A Coruña"), + (1100, 38.9942, -1.8585, "A-31 Albacete"), + (1110, 39.8628, -4.0273, "A-4 Toledo"), + ] + + def fetch_data(self) -> List[Dict[str, Any]]: + cameras = [] + probe_headers = { + "Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8", + "Referer": "https://infocar.dgt.es/", + } + for cam_id, lat, lon, description in self.KNOWN_CAMERAS: + media_url = f"https://infocar.dgt.es/etraffic/data/camaras/{cam_id}.jpg" + if not _media_url_reachable(media_url, timeout=6, headers=probe_headers): + continue + cameras.append({ + "id": f"DGT-{cam_id}", + "source_agency": "DGT Spain", + "lat": lat, + "lon": lon, + "direction_facing": description, + "media_url": media_url, + "media_type": "image", + "refresh_rate_seconds": 300, + }) + logger.info(f"DGTNationalIngestor: loaded {len(cameras)} cameras") return cameras # --------------------------------------------------------------------------- -# Spain — Vitoria-Gasteiz (GeoJSON, ~17 cameras) +# Madrid City Hall — KML open data (~357 cameras) # --------------------------------------------------------------------------- -class VitoriaGasteizCCTVIngestor(BaseCCTVIngestor): - # Vitoria-Gasteiz municipal traffic cameras in GeoJSON. - # No API key required. Published on vitoria-gasteiz.org. - VITORIA_URL = "https://www.vitoria-gasteiz.org/c11-01w/cameras?action=list&format=GEOJSON" +# Published on datos.madrid.es — free reuse with attribution, +# Licence: Madrid Open Data (EU PSI Directive 2019/1024). +# Author credit: Alborz Nazari (github.com/AlborzNazari) — PR #91 + +_KML_NS = {"kml": "http://www.opengis.net/kml/2.2"} + + +def _find_kml_element(element, tag): + """Find first descendant matching tag, ignoring XML namespace prefix.""" + import xml.etree.ElementTree as ET + el = element.find(f".//{tag}") + if el is not None: + return el + for child in element.iter(): + if child.tag.endswith(f"}}{tag}") or child.tag == tag: + return child + return None + + +def _extract_img_src(html_fragment: str): + """Extract src URL from an tag or bare .jpg URL in an HTML fragment.""" + import re + match = re.search(r'src=["\']([^"\']+)["\']', html_fragment, re.IGNORECASE) + if match: + return match.group(1) + match = re.search(r'https?://\S+\.jpg', html_fragment, re.IGNORECASE) + if match: + return match.group(0) + return None + + +class MadridCityIngestor(BaseCCTVIngestor): + """Madrid City Hall traffic cameras from datos.madrid.es KML feed.""" + + KML_URL = "http://datos.madrid.es/egob/catalogo/202088-0-trafico-camaras.kml" def fetch_data(self) -> List[Dict[str, Any]]: + import xml.etree.ElementTree as ET + try: - response = fetch_with_curl(self.VITORIA_URL, timeout=15) + response = fetch_with_curl(self.KML_URL, timeout=20) response.raise_for_status() - data = response.json() except Exception as e: - logger.error(f"VitoriaGasteizCCTVIngestor: failed to fetch GeoJSON: {e}") + logger.error(f"MadridCityIngestor: failed to fetch KML: {e}") + return [] + + try: + root = ET.fromstring(response.content) + except ET.ParseError as e: + logger.error(f"MadridCityIngestor: failed to parse KML: {e}") return [] cameras = [] - for feature in data.get("features", []): + placemarks = root.findall(".//kml:Placemark", _KML_NS) + if not placemarks: + placemarks = [el for el in root.iter() if el.tag.endswith("Placemark")] + + for i, placemark in enumerate(placemarks): try: - props = feature.get("properties", {}) - geom = feature.get("geometry", {}) - coords = geom.get("coordinates", []) - if len(coords) < 2: + name_el = _find_kml_element(placemark, "name") + name = name_el.text.strip() if name_el is not None and name_el.text else f"Madrid Camera {i}" + + coords_el = _find_kml_element(placemark, "coordinates") + if coords_el is None or not coords_el.text: continue - image_url = props.get("imagen") or props.get("url") - if not image_url: + parts = coords_el.text.strip().split(",") + if len(parts) < 2: continue + lon = float(parts[0]) + lat = float(parts[1]) - cam_id = props.get("id") or props.get("nombre") or str(coords) - description = props.get("nombre") or props.get("descripcion") or f"Vitoria Camera {cam_id}" + desc_el = _find_kml_element(placemark, "description") + image_url = None + if desc_el is not None and desc_el.text: + image_url = _extract_img_src(desc_el.text) + + if not image_url: + continue cameras.append({ - "id": f"VIT-{cam_id}", - "source_agency": "Vitoria-Gasteiz", - "lat": float(coords[1]), - "lon": float(coords[0]), - "direction_facing": description, + "id": f"MAD-{i:04d}", + "source_agency": "Madrid City Hall", + "lat": lat, + "lon": lon, + "direction_facing": name, "media_url": image_url, - "refresh_rate_seconds": 300, + "media_type": "image", + "refresh_rate_seconds": 600, }) except (ValueError, TypeError, IndexError) as e: - logger.debug(f"VitoriaGasteizCCTVIngestor: skipping malformed feature: {e}") + logger.debug(f"MadridCityIngestor: skipping malformed placemark: {e}") continue - logger.info(f"VitoriaGasteizCCTVIngestor: parsed {len(cameras)} cameras") + logger.info(f"MadridCityIngestor: parsed {len(cameras)} cameras") return cameras @@ -603,10 +984,7 @@ def _detect_media_type(url: str) -> str: url_lower = url.lower() if any(ext in url_lower for ext in [".mp4", ".webm", ".ogg"]): return "video" - if any( - kw in url_lower - for kw in [".mjpg", ".mjpeg", "mjpg", "axis-cgi/mjpg", "mode=motion"] - ): + if any(kw in url_lower for kw in [".mjpg", ".mjpeg", "mjpg", "axis-cgi/mjpg", "mode=motion"]): return "mjpeg" if ".m3u8" in url_lower or "hls" in url_lower: return "hls" @@ -617,9 +995,33 @@ def _detect_media_type(url: str) -> str: return "image" +def run_all_ingestors(): + """Run all CCTV ingestors synchronously. Used for first-run DB seeding.""" + ingestors = [ + TFLJamCamIngestor(), + LTASingaporeIngestor(), + AustinTXIngestor(), + NYCDOTIngestor(), + CaltransIngestor(), + ColoradoDOTIngestor(), + WSDOTIngestor(), + GeorgiaDOTIngestor(), + IllinoisDOTIngestor(), + MichiganDOTIngestor(), + WindyWebcamsIngestor(), + OSMTrafficCameraIngestor(), + DGTNationalIngestor(), + MadridCityIngestor(), + ] + for ing in ingestors: + try: + ing.ingest() + except Exception as e: + logger.warning(f"Ingestor {ing.__class__.__name__} failed during seed: {e}") + + def get_all_cameras() -> List[Dict[str, Any]]: - init_db() - conn = _connect() + conn = sqlite3.connect(str(DB_PATH)) conn.row_factory = sqlite3.Row cursor = conn.cursor() cursor.execute("SELECT * FROM cameras") @@ -628,6 +1030,6 @@ def get_all_cameras() -> List[Dict[str, Any]]: cameras = [] for row in rows: cam = dict(row) - cam["media_type"] = _detect_media_type(cam.get("media_url", "")) + cam["media_type"] = str(cam.get("media_type") or _detect_media_type(cam.get("media_url", "")) or "image") cameras.append(cam) return cameras diff --git a/backend/services/config.py b/backend/services/config.py new file mode 100644 index 00000000..b2772dff --- /dev/null +++ b/backend/services/config.py @@ -0,0 +1,122 @@ +"""Typed configuration via pydantic-settings.""" + +from functools import lru_cache +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + # Admin/security + ADMIN_KEY: str = "" + ALLOW_INSECURE_ADMIN: bool = False + PUBLIC_API_KEY: str = "" + + # Data sources + AIS_API_KEY: str = "" + OPENSKY_CLIENT_ID: str = "" + OPENSKY_CLIENT_SECRET: str = "" + LTA_ACCOUNT_KEY: str = "" + + # Runtime + CORS_ORIGINS: str = "" + FETCH_SLOW_THRESHOLD_S: float = 5.0 + MESH_STRICT_SIGNATURES: bool = True + MESH_DEBUG_MODE: bool = False + MESH_MQTT_EXTRA_ROOTS: str = "" + MESH_MQTT_EXTRA_TOPICS: str = "" + MESH_MQTT_INCLUDE_DEFAULT_ROOTS: bool = True + MESH_RNS_ENABLED: bool = False + MESH_ARTI_ENABLED: bool = False + MESH_ARTI_SOCKS_PORT: int = 9050 + MESH_RELAY_PEERS: str = "" + MESH_BOOTSTRAP_DISABLED: bool = False + MESH_BOOTSTRAP_MANIFEST_PATH: str = "data/bootstrap_peers.json" + MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY: str = "" + MESH_NODE_MODE: str = "participant" + MESH_SYNC_INTERVAL_S: int = 300 + MESH_SYNC_FAILURE_BACKOFF_S: int = 60 + MESH_RELAY_PUSH_TIMEOUT_S: int = 10 + MESH_RELAY_MAX_FAILURES: int = 3 + MESH_RELAY_FAILURE_COOLDOWN_S: int = 120 + MESH_PEER_PUSH_SECRET: str = "" + MESH_RNS_APP_NAME: str = "shadowbroker" + MESH_RNS_ASPECT: str = "infonet" + MESH_RNS_IDENTITY_PATH: str = "" + MESH_RNS_PEERS: str = "" + MESH_RNS_DANDELION_HOPS: int = 2 + MESH_RNS_DANDELION_DELAY_MS: int = 400 + MESH_RNS_CHURN_INTERVAL_S: int = 300 + MESH_RNS_MAX_PEERS: int = 32 + MESH_RNS_MAX_PAYLOAD: int = 8192 + MESH_RNS_PEER_BUCKET_PREFIX: int = 4 + MESH_RNS_MAX_PEERS_PER_BUCKET: int = 4 + MESH_RNS_PEER_FAIL_THRESHOLD: int = 3 + MESH_RNS_PEER_COOLDOWN_S: int = 300 + MESH_RNS_SHARD_ENABLED: bool = False + MESH_RNS_SHARD_DATA_SHARDS: int = 3 + MESH_RNS_SHARD_PARITY_SHARDS: int = 1 + MESH_RNS_SHARD_TTL_S: int = 30 + MESH_RNS_FEC_CODEC: str = "xor" # xor | rs + MESH_RNS_BATCH_MS: int = 200 + # Keep a low background cadence on private RNS links so quiet nodes are less + # trivially fingerprintable by silence alone. Set to 0 to disable explicitly. + MESH_RNS_COVER_INTERVAL_S: int = 30 + MESH_RNS_COVER_SIZE: int = 64 + MESH_RNS_IBF_WINDOW: int = 256 + MESH_RNS_IBF_TABLE_SIZE: int = 64 + MESH_RNS_IBF_MINHASH_SIZE: int = 16 + MESH_RNS_IBF_MINHASH_THRESHOLD: float = 0.25 + MESH_RNS_IBF_WINDOW_JITTER: int = 32 + MESH_RNS_IBF_INTERVAL_S: int = 120 + MESH_RNS_IBF_SYNC_PEERS: int = 3 + MESH_RNS_IBF_QUORUM_TIMEOUT_S: int = 6 + MESH_RNS_IBF_MAX_REQUEST_IDS: int = 64 + MESH_RNS_IBF_MAX_EVENTS: int = 64 + MESH_RNS_SESSION_ROTATE_S: int = 1800 + MESH_RNS_IBF_FAIL_THRESHOLD: int = 3 + MESH_RNS_IBF_COOLDOWN_S: int = 120 + MESH_VERIFY_INTERVAL_S: int = 600 + MESH_VERIFY_SIGNATURES: bool = True + MESH_DM_SECURE_MODE: bool = True + MESH_DM_TOKEN_PEPPER: str = "" + MESH_DM_ALLOW_LEGACY_GET: bool = False + MESH_DM_PERSIST_SPOOL: bool = False + MESH_DM_REQUIRE_SENDER_SEAL_SHARED: bool = True + MESH_DM_NONCE_TTL_S: int = 300 + MESH_DM_NONCE_CACHE_MAX: int = 4096 + MESH_DM_REQUEST_MAX_AGE_S: int = 300 + MESH_DM_REQUEST_MAILBOX_LIMIT: int = 12 + MESH_DM_SHARED_MAILBOX_LIMIT: int = 48 + MESH_DM_SELF_MAILBOX_LIMIT: int = 12 + MESH_DM_MAX_MSG_BYTES: int = 8192 + MESH_DM_ALLOW_SENDER_SEAL: bool = False + # TTL for DH key and prekey bundle registrations — stale entries are pruned. + MESH_DM_KEY_TTL_DAYS: int = 30 + # TTL for mailbox binding metadata — shorter = smaller metadata footprint on disk. + MESH_DM_BINDING_TTL_DAYS: int = 7 + # When False, mailbox bindings are memory-only (agents re-register on restart). + MESH_DM_METADATA_PERSIST: bool = True + MESH_SCOPED_TOKENS: str = "" + MESH_GATE_SESSION_ROTATE_MSGS: int = 50 + MESH_GATE_SESSION_ROTATE_S: int = 3600 + # Add a randomized grace window before anonymous gate-session auto-rotation + # so threshold-triggered identity swaps are less trivially correlated. + MESH_GATE_SESSION_ROTATE_JITTER_S: int = 180 + # Private gate APIs expose a backward-jittered timestamp view so observers + # cannot trivially align exact send times from response metadata alone. + MESH_GATE_TIMESTAMP_JITTER_S: int = 60 + MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK: bool = False + MESH_PRIVATE_LOG_TTL_S: int = 900 + # Clearnet fallback policy for private-tier messages. + # "block" (default) = refuse to send private messages over clearnet. + # "allow" = fall back to clearnet when Tor/RNS is unavailable (weaker privacy). + MESH_PRIVATE_CLEARNET_FALLBACK: str = "block" + # Meshtastic MQTT broker credentials (defaults match public firmware). + MESH_MQTT_USER: str = "meshdev" + MESH_MQTT_PASS: str = "large4cats" + + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/backend/services/constants.py b/backend/services/constants.py index 3a8ce52b..92ed26e3 100644 --- a/backend/services/constants.py +++ b/backend/services/constants.py @@ -2,32 +2,33 @@ # Centralized magic numbers. Import from here instead of hardcoding. # ─── Flight Trails ────────────────────────────────────────────────────────── -FLIGHT_TRAIL_MAX_TRACKED = 2000 # Max concurrent tracked trails before LRU eviction +FLIGHT_TRAIL_MAX_TRACKED = 2000 # Max concurrent tracked trails before LRU eviction FLIGHT_TRAIL_POINTS_PER_FLIGHT = 200 # Max trail points kept per aircraft -TRACKED_TRAIL_TTL_S = 1800 # 30 min - trail TTL for tracked flights -DEFAULT_TRAIL_TTL_S = 300 # 5 min - trail TTL for non-tracked flights +TRACKED_TRAIL_TTL_S = 1800 # 30 min - trail TTL for tracked flights +DEFAULT_TRAIL_TTL_S = 300 # 5 min - trail TTL for non-tracked flights # ─── Detection Thresholds ────────────────────────────────────────────────── -HOLD_PATTERN_DEGREES = 300 # Total heading change to flag holding pattern -GPS_JAMMING_NACP_THRESHOLD = 8 # NACp below this = degraded GPS signal -GPS_JAMMING_GRID_SIZE = 1.0 # 1 degree grid for aggregation -GPS_JAMMING_MIN_RATIO = 0.25 # 25% degraded aircraft to flag zone +HOLD_PATTERN_DEGREES = 300 # Total heading change to flag holding pattern +GPS_JAMMING_NACP_THRESHOLD = 8 # NACp below this = degraded GPS signal +GPS_JAMMING_GRID_SIZE = 1.0 # 1 degree grid for aggregation +GPS_JAMMING_MIN_RATIO = 0.30 # 30% degraded aircraft to flag zone +GPS_JAMMING_MIN_AIRCRAFT = 5 # Min aircraft in grid cell for statistical significance # ─── Network & Circuit Breaker ────────────────────────────────────────────── -CIRCUIT_BREAKER_TTL_S = 120 # Skip domain for 2 min after total failure -DOMAIN_FAIL_TTL_S = 300 # Skip requests.get for 5 min, go straight to curl -CONNECT_TIMEOUT_S = 3 # Short connect timeout for fast firewall-block detection +CIRCUIT_BREAKER_TTL_S = 120 # Skip domain for 2 min after total failure +DOMAIN_FAIL_TTL_S = 300 # Skip requests.get for 5 min, go straight to curl +CONNECT_TIMEOUT_S = 3 # Short connect timeout for fast firewall-block detection # ─── Data Fetcher Intervals ──────────────────────────────────────────────── -FAST_FETCH_INTERVAL_S = 60 # Flights, ships, satellites, military -SLOW_FETCH_INTERVAL_MIN = 30 # News, markets, space weather -CCTV_FETCH_INTERVAL_MIN = 1 # CCTV camera pipeline -LIVEUAMAP_FETCH_INTERVAL_HR = 12 # LiveUAMap scraper +FAST_FETCH_INTERVAL_S = 60 # Flights, ships, satellites, military +SLOW_FETCH_INTERVAL_MIN = 30 # News, markets, space weather +CCTV_FETCH_INTERVAL_MIN = 1 # CCTV camera pipeline +LIVEUAMAP_FETCH_INTERVAL_HR = 12 # LiveUAMap scraper # ─── External API ────────────────────────────────────────────────────────── -OPENSKY_RATE_LIMIT_S = 300 # Only re-fetch OpenSky every 5 minutes -OPENSKY_REQUEST_TIMEOUT_S = 15 # Timeout for OpenSky API calls -ROUTE_FETCH_TIMEOUT_S = 15 # Timeout for adsb.lol route lookups +OPENSKY_RATE_LIMIT_S = 300 # Only re-fetch OpenSky every 5 minutes +OPENSKY_REQUEST_TIMEOUT_S = 15 # Timeout for OpenSky API calls +ROUTE_FETCH_TIMEOUT_S = 15 # Timeout for adsb.lol route lookups # ─── Internet Outage Detection ───────────────────────────────────────────── -INTERNET_OUTAGE_MIN_SEVERITY = 0.10 # 10% drop minimum to show +INTERNET_OUTAGE_MIN_SEVERITY = 0.10 # 10% drop minimum to show diff --git a/backend/services/correlation_engine.py b/backend/services/correlation_engine.py new file mode 100644 index 00000000..7c11c607 --- /dev/null +++ b/backend/services/correlation_engine.py @@ -0,0 +1,342 @@ +""" +Emergent Intelligence — Cross-layer correlation engine. + +Scans co-located events across multiple data layers and emits composite +alerts that no single source could generate alone. + +Correlation types: + - RF Anomaly: GPS jamming + internet outage (both required) + - Military Buildup: Military flights + naval vessels + GDELT conflict events + - Infrastructure Cascade: Internet outage + KiwiSDR offline in same zone +""" + +import logging +from collections import defaultdict + +logger = logging.getLogger(__name__) + +# Grid cell size in degrees — 1° ≈ 111 km at equator. +# Tighter than the previous 2° to reduce false co-locations. +_CELL_SIZE = 1 + +# Quality gates for RF anomaly correlation — only high-confidence inputs. +# GPS jamming + internet outage overlap in a 111km cell is easily a coincidence +# (IODA returns ~100 regional outages; GPS NACp dips are common in busy airspace). +# Only fire when the evidence is strong enough to indicate deliberate RF interference. +_RF_CORR_MIN_GPS_RATIO = 0.60 # Need strong jamming signal, not marginal NACp dips +_RF_CORR_MIN_OUTAGE_PCT = 40 # Need a serious outage, not routine BGP fluctuation +_RF_CORR_MIN_INDICATORS = 3 # Require 3+ corroborating signals (not just GPS+outage) + + +def _cell_key(lat: float, lng: float) -> str: + """Convert lat/lng to a grid cell key.""" + clat = int(lat // _CELL_SIZE) * _CELL_SIZE + clng = int(lng // _CELL_SIZE) * _CELL_SIZE + return f"{clat},{clng}" + + +def _cell_center(key: str) -> tuple[float, float]: + """Get center lat/lng from a cell key.""" + parts = key.split(",") + return float(parts[0]) + _CELL_SIZE / 2, float(parts[1]) + _CELL_SIZE / 2 + + +def _severity(indicator_count: int) -> str: + if indicator_count >= 3: + return "high" + if indicator_count >= 2: + return "medium" + return "low" + + +def _severity_score(sev: str) -> float: + return {"high": 90, "medium": 60, "low": 30}.get(sev, 0) + + +def _outage_pct(outage: dict) -> float: + """Extract outage severity percentage from an outage dict.""" + return float(outage.get("severity", 0) or outage.get("severity_pct", 0) or 0) + + +# --------------------------------------------------------------------------- +# RF Anomaly: GPS jamming + internet outage (both must be present) +# --------------------------------------------------------------------------- + + +def _detect_rf_anomalies(data: dict) -> list[dict]: + gps_jamming = data.get("gps_jamming") or [] + internet_outages = data.get("internet_outages") or [] + + if not gps_jamming: + return [] # No GPS jamming → no RF anomalies possible + + # Build grid of indicators + cells: dict[str, dict] = defaultdict(lambda: { + "gps_jam": False, "gps_ratio": 0.0, + "outage": False, "outage_pct": 0.0, + }) + + for z in gps_jamming: + lat, lng = z.get("lat"), z.get("lng") + if lat is None or lng is None: + continue + ratio = z.get("ratio", 0) + if ratio < _RF_CORR_MIN_GPS_RATIO: + continue # Skip marginal jamming zones + key = _cell_key(lat, lng) + cells[key]["gps_jam"] = True + cells[key]["gps_ratio"] = max(cells[key]["gps_ratio"], ratio) + + for o in internet_outages: + lat = o.get("lat") or o.get("latitude") + lng = o.get("lng") or o.get("lon") or o.get("longitude") + if lat is None or lng is None: + continue + pct = _outage_pct(o) + if pct < _RF_CORR_MIN_OUTAGE_PCT: + continue # Skip minor outages (ISP maintenance noise) + key = _cell_key(float(lat), float(lng)) + cells[key]["outage"] = True + cells[key]["outage_pct"] = max(cells[key]["outage_pct"], pct) + + # PSK Reporter: presence = healthy RF. Only used as a bonus indicator, + # NOT as a standalone trigger (absence is normal in most cells). + psk_reporter = data.get("psk_reporter") or [] + psk_cells: set[str] = set() + for s in psk_reporter: + lat, lng = s.get("lat"), s.get("lon") + if lat is not None and lng is not None: + psk_cells.add(_cell_key(lat, lng)) + + # When PSK data is unavailable, we can't get a 3rd indicator, so require + # an even higher GPS jamming ratio to compensate (real EW shows 75%+). + psk_available = len(psk_reporter) > 0 + + alerts: list[dict] = [] + for key, c in cells.items(): + # GPS jamming is the anchor — required for every RF anomaly alert + if not c["gps_jam"]: + continue + if not c["outage"]: + continue # Both GPS jamming AND outage are always required + + indicators = 2 # GPS jamming + outage + drivers: list[str] = [f"GPS jamming {int(c['gps_ratio'] * 100)}%"] + pct = c["outage_pct"] + drivers.append(f"Internet outage{f' {pct:.0f}%' if pct else ''}") + + # PSK absence confirms RF environment is disrupted + if psk_available and key not in psk_cells: + indicators += 1 + drivers.append("No HF digital activity (PSK Reporter)") + + if indicators < _RF_CORR_MIN_INDICATORS: + # Without PSK data, only allow through if GPS ratio is extreme + # (75%+ indicates deliberate, sustained jamming — not noise) + if not psk_available and c["gps_ratio"] >= 0.75 and pct >= 50: + pass # Allow this high-confidence 2-indicator alert through + else: + continue + + lat, lng = _cell_center(key) + sev = _severity(indicators) + alerts.append({ + "lat": lat, + "lng": lng, + "type": "rf_anomaly", + "severity": sev, + "score": _severity_score(sev), + "drivers": drivers[:3], + "cell_size": _CELL_SIZE, + }) + + return alerts + + +# --------------------------------------------------------------------------- +# Military Buildup: flights + ships + GDELT conflict +# --------------------------------------------------------------------------- + + +def _detect_military_buildups(data: dict) -> list[dict]: + mil_flights = data.get("military_flights") or [] + ships = data.get("ships") or [] + gdelt = data.get("gdelt") or [] + + cells: dict[str, dict] = defaultdict(lambda: { + "mil_flights": 0, "mil_ships": 0, "gdelt_events": 0, + }) + + for f in mil_flights: + lat = f.get("lat") or f.get("latitude") + lng = f.get("lng") or f.get("lon") or f.get("longitude") + if lat is None or lng is None: + continue + try: + key = _cell_key(float(lat), float(lng)) + cells[key]["mil_flights"] += 1 + except (ValueError, TypeError): + continue + + mil_ship_types = {"military_vessel", "military", "warship", "patrol", "destroyer", + "frigate", "corvette", "carrier", "submarine", "cruiser"} + for s in ships: + stype = (s.get("type") or s.get("ship_type") or "").lower() + if not any(mt in stype for mt in mil_ship_types): + continue + lat = s.get("lat") or s.get("latitude") + lng = s.get("lng") or s.get("lon") or s.get("longitude") + if lat is None or lng is None: + continue + try: + key = _cell_key(float(lat), float(lng)) + cells[key]["mil_ships"] += 1 + except (ValueError, TypeError): + continue + + for g in gdelt: + lat = g.get("lat") or g.get("latitude") or g.get("actionGeo_Lat") + lng = g.get("lng") or g.get("lon") or g.get("longitude") or g.get("actionGeo_Long") + if lat is None or lng is None: + continue + try: + key = _cell_key(float(lat), float(lng)) + cells[key]["gdelt_events"] += 1 + except (ValueError, TypeError): + continue + + alerts: list[dict] = [] + for key, c in cells.items(): + mil_total = c["mil_flights"] + c["mil_ships"] + has_gdelt = c["gdelt_events"] > 0 + + # Need meaningful military presence AND a conflict indicator + if mil_total < 3 or not has_gdelt: + continue + + drivers: list[str] = [] + if c["mil_flights"]: + drivers.append(f"{c['mil_flights']} military aircraft") + if c["mil_ships"]: + drivers.append(f"{c['mil_ships']} military vessels") + if c["gdelt_events"]: + drivers.append(f"{c['gdelt_events']} conflict events") + + if mil_total >= 11: + sev = "high" + elif mil_total >= 6: + sev = "medium" + else: + sev = "low" + + lat, lng = _cell_center(key) + alerts.append({ + "lat": lat, + "lng": lng, + "type": "military_buildup", + "severity": sev, + "score": _severity_score(sev), + "drivers": drivers[:3], + "cell_size": _CELL_SIZE, + }) + + return alerts + + +# --------------------------------------------------------------------------- +# Infrastructure Cascade: outage + KiwiSDR co-location +# +# Power plants are removed from this detector — with 35K plants globally, +# virtually every 2° cell contains one, making every outage a false hit. +# KiwiSDR receivers (~300 worldwide) are sparse enough to be meaningful: +# an outage in the same cell as a KiwiSDR indicates real infrastructure +# disruption affecting radio monitoring capability. +# --------------------------------------------------------------------------- + + +def _detect_infra_cascades(data: dict) -> list[dict]: + internet_outages = data.get("internet_outages") or [] + kiwisdr = data.get("kiwisdr") or [] + + if not kiwisdr: + return [] + + # Build set of cells with KiwiSDR receivers + kiwi_cells: set[str] = set() + for k in kiwisdr: + lat, lng = k.get("lat"), k.get("lon") or k.get("lng") + if lat is not None and lng is not None: + try: + kiwi_cells.add(_cell_key(float(lat), float(lng))) + except (ValueError, TypeError): + pass + + if not kiwi_cells: + return [] + + alerts: list[dict] = [] + for o in internet_outages: + lat = o.get("lat") or o.get("latitude") + lng = o.get("lng") or o.get("lon") or o.get("longitude") + if lat is None or lng is None: + continue + try: + key = _cell_key(float(lat), float(lng)) + except (ValueError, TypeError): + continue + + if key not in kiwi_cells: + continue + + pct = _outage_pct(o) + drivers = [f"Internet outage{f' {pct:.0f}%' if pct else ''}", + "KiwiSDR receivers in affected zone"] + + lat_c, lng_c = _cell_center(key) + alerts.append({ + "lat": lat_c, + "lng": lng_c, + "type": "infra_cascade", + "severity": "medium", + "score": _severity_score("medium"), + "drivers": drivers, + "cell_size": _CELL_SIZE, + }) + + return alerts + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def compute_correlations(data: dict) -> list[dict]: + """Run all correlation detectors and return merged alert list.""" + alerts: list[dict] = [] + + try: + alerts.extend(_detect_rf_anomalies(data)) + except Exception as e: + logger.error("Correlation engine RF anomaly error: %s", e) + + try: + alerts.extend(_detect_military_buildups(data)) + except Exception as e: + logger.error("Correlation engine military buildup error: %s", e) + + try: + alerts.extend(_detect_infra_cascades(data)) + except Exception as e: + logger.error("Correlation engine infra cascade error: %s", e) + + rf = sum(1 for a in alerts if a["type"] == "rf_anomaly") + mil = sum(1 for a in alerts if a["type"] == "military_buildup") + infra = sum(1 for a in alerts if a["type"] == "infra_cascade") + if alerts: + logger.info( + "Correlations: %d alerts (%d rf, %d mil, %d infra)", + len(alerts), rf, mil, infra, + ) + + return alerts diff --git a/backend/services/data_fetcher.py b/backend/services/data_fetcher.py index bce5ff94..a0cc5c21 100644 --- a/backend/services/data_fetcher.py +++ b/backend/services/data_fetcher.py @@ -13,10 +13,14 @@ - infrastructure.py — internet outages, data centers, CCTV, KiwiSDR - geo.py — ships, airports, frontlines, GDELT, LiveUAMap """ + import logging import concurrent.futures -from datetime import datetime +import os +import time +from datetime import datetime, timedelta from dotenv import load_dotenv + load_dotenv() from apscheduler.schedulers.background import BackgroundScheduler @@ -25,7 +29,11 @@ # Shared state — all fetcher modules read/write through this from services.fetchers._store import ( - latest_data, source_timestamps, _mark_fresh, _data_lock, # noqa: F401 — re-exported for main.py + latest_data, + source_timestamps, + _mark_fresh, + _data_lock, # noqa: F401 — re-exported for main.py + get_latest_data_subset, ) # Domain-specific fetcher modules (already extracted) @@ -36,24 +44,109 @@ from services.fetchers.news import fetch_news # noqa: F401 # Newly extracted fetcher modules -from services.fetchers.financial import fetch_defense_stocks, fetch_oil_prices # noqa: F401 +from services.fetchers.financial import fetch_financial_markets # noqa: F401 +from services.fetchers.unusual_whales import fetch_unusual_whales # noqa: F401 from services.fetchers.earth_observation import ( # noqa: F401 - fetch_earthquakes, fetch_firms_fires, fetch_space_weather, fetch_weather, + fetch_earthquakes, + fetch_firms_fires, + fetch_firms_country_fires, + fetch_space_weather, + fetch_weather, + fetch_weather_alerts, + fetch_air_quality, + fetch_volcanoes, + fetch_viirs_change_nodes, ) from services.fetchers.infrastructure import ( # noqa: F401 - fetch_internet_outages, fetch_datacenters, fetch_military_bases, fetch_power_plants, - fetch_cctv, fetch_kiwisdr, + fetch_internet_outages, + fetch_ripe_atlas_probes, + fetch_datacenters, + fetch_military_bases, + fetch_power_plants, + fetch_cctv, + fetch_kiwisdr, + fetch_scanners, + fetch_satnogs, + fetch_tinygs, + fetch_psk_reporter, ) from services.fetchers.geo import ( # noqa: F401 - fetch_ships, fetch_airports, find_nearest_airport, cached_airports, - fetch_frontlines, fetch_gdelt, fetch_geopolitics, update_liveuamap, + fetch_ships, + fetch_airports, + find_nearest_airport, + cached_airports, + fetch_frontlines, + fetch_gdelt, + fetch_geopolitics, + update_liveuamap, + fetch_fishing_activity, ) +from services.fetchers.prediction_markets import fetch_prediction_markets # noqa: F401 +from services.fetchers.sigint import fetch_sigint # noqa: F401 +from services.fetchers.trains import fetch_trains # noqa: F401 +from services.fetchers.ukraine_alerts import fetch_ukraine_air_raid_alerts # noqa: F401 +from services.fetchers.meshtastic_map import ( + fetch_meshtastic_nodes, + load_meshtastic_cache_if_available, +) # noqa: F401 +from services.fetchers.fimi import fetch_fimi # noqa: F401 +from services.ais_stream import prune_stale_vessels # noqa: F401 logger = logging.getLogger(__name__) +_SLOW_FETCH_S = float(os.environ.get("FETCH_SLOW_THRESHOLD_S", "5")) + +# Shared thread pool — reused across all fetch cycles instead of creating/destroying per tick +_SHARED_EXECUTOR = concurrent.futures.ThreadPoolExecutor( + max_workers=20, thread_name_prefix="fetch" +) + # --------------------------------------------------------------------------- # Scheduler & Orchestration # --------------------------------------------------------------------------- +def _run_tasks(label: str, funcs: list): + """Run tasks concurrently and log any exceptions (do not fail silently).""" + if not funcs: + return + futures = {_SHARED_EXECUTOR.submit(func): (func.__name__, time.perf_counter()) for func in funcs} + for future in concurrent.futures.as_completed(futures): + name, start = futures[future] + try: + future.result() + duration = time.perf_counter() - start + from services.fetch_health import record_success + + record_success(name, duration_s=duration) + if duration > _SLOW_FETCH_S: + logger.warning(f"{label} task slow: {name} took {duration:.2f}s") + except Exception as e: + duration = time.perf_counter() - start + from services.fetch_health import record_failure + + record_failure(name, error=e, duration_s=duration) + logger.exception(f"{label} task failed: {name}") + + +def _run_task_with_health(func, name: str | None = None): + """Run a single task with health tracking.""" + task_name = name or getattr(func, "__name__", "task") + start = time.perf_counter() + try: + func() + duration = time.perf_counter() - start + from services.fetch_health import record_success + + record_success(task_name, duration_s=duration) + if duration > _SLOW_FETCH_S: + logger.warning(f"task slow: {task_name} took {duration:.2f}s") + except Exception as e: + duration = time.perf_counter() - start + from services.fetch_health import record_failure + + record_failure(task_name, error=e, duration_s=duration) + logger.exception(f"task failed: {task_name}") + + def update_fast_data(): """Fast-tier: moving entities that need frequent updates (every 60s).""" logger.info("Fast-tier data update starting...") @@ -62,50 +155,201 @@ def update_fast_data(): fetch_military_flights, fetch_ships, fetch_satellites, + fetch_sigint, + fetch_trains, + fetch_tinygs, ] - with concurrent.futures.ThreadPoolExecutor(max_workers=len(fast_funcs)) as executor: - futures = [executor.submit(func) for func in fast_funcs] - concurrent.futures.wait(futures) + _run_tasks("fast-tier", fast_funcs) with _data_lock: - latest_data['last_updated'] = datetime.utcnow().isoformat() + latest_data["last_updated"] = datetime.utcnow().isoformat() + from services.fetchers._store import bump_data_version + bump_data_version() logger.info("Fast-tier update complete.") + def update_slow_data(): """Slow-tier: contextual + enrichment data that refreshes less often (every 5–10 min).""" logger.info("Slow-tier data update starting...") slow_funcs = [ fetch_news, + fetch_prediction_markets, fetch_earthquakes, fetch_firms_fires, - fetch_defense_stocks, - fetch_oil_prices, + fetch_firms_country_fires, fetch_weather, fetch_space_weather, fetch_internet_outages, + fetch_ripe_atlas_probes, # runs after IODA to deduplicate fetch_cctv, fetch_kiwisdr, + fetch_satnogs, fetch_frontlines, - fetch_gdelt, fetch_datacenters, fetch_military_bases, + fetch_scanners, + fetch_psk_reporter, + fetch_weather_alerts, + fetch_air_quality, + fetch_fishing_activity, fetch_power_plants, + fetch_ukraine_air_raid_alerts, ] - with concurrent.futures.ThreadPoolExecutor(max_workers=len(slow_funcs)) as executor: - futures = [executor.submit(func) for func in slow_funcs] - concurrent.futures.wait(futures) + _run_tasks("slow-tier", slow_funcs) + # Run correlation engine after all data is fresh + try: + from services.correlation_engine import compute_correlations + with _data_lock: + snapshot = dict(latest_data) + correlations = compute_correlations(snapshot) + with _data_lock: + latest_data["correlations"] = correlations + except Exception as e: + logger.error("Correlation engine failed: %s", e) + from services.fetchers._store import bump_data_version + bump_data_version() logger.info("Slow-tier update complete.") -def update_all_data(): - """Full refresh — all tiers run IN PARALLEL for fastest startup.""" + +def update_all_data(*, startup_mode: bool = False): + """Full refresh. + + On startup we prefer cached/DB-backed data first, then let scheduled jobs + perform some heavy top-ups after the app is already responsive. + """ logger.info("Full data update starting (parallel)...") - with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool: - f0 = pool.submit(fetch_airports) - f1 = pool.submit(update_fast_data) - f2 = pool.submit(update_slow_data) - concurrent.futures.wait([f0, f1, f2]) + # Preload Meshtastic map cache immediately (instant, from disk) + load_meshtastic_cache_if_available() + with _data_lock: + meshtastic_seeded = bool(latest_data.get("meshtastic_map_nodes")) + futures = { + _SHARED_EXECUTOR.submit(fetch_airports): ("fetch_airports", time.perf_counter()), + _SHARED_EXECUTOR.submit(update_fast_data): ("update_fast_data", time.perf_counter()), + _SHARED_EXECUTOR.submit(update_slow_data): ("update_slow_data", time.perf_counter()), + _SHARED_EXECUTOR.submit(fetch_volcanoes): ("fetch_volcanoes", time.perf_counter()), + _SHARED_EXECUTOR.submit(fetch_viirs_change_nodes): ("fetch_viirs_change_nodes", time.perf_counter()), + _SHARED_EXECUTOR.submit(fetch_unusual_whales): ("fetch_unusual_whales", time.perf_counter()), + _SHARED_EXECUTOR.submit(fetch_fimi): ("fetch_fimi", time.perf_counter()), + _SHARED_EXECUTOR.submit(fetch_gdelt): ("fetch_gdelt", time.perf_counter()), + _SHARED_EXECUTOR.submit(update_liveuamap): ("update_liveuamap", time.perf_counter()), + } + if not startup_mode or not meshtastic_seeded: + futures[_SHARED_EXECUTOR.submit(fetch_meshtastic_nodes)] = ( + "fetch_meshtastic_nodes", + time.perf_counter(), + ) + else: + logger.info( + "Startup preload: Meshtastic cache already loaded, deferring remote map refresh to scheduled cadence" + ) + for future in concurrent.futures.as_completed(futures): + name, start = futures[future] + try: + future.result() + duration = time.perf_counter() - start + from services.fetch_health import record_success + + record_success(name, duration_s=duration) + if duration > _SLOW_FETCH_S: + logger.warning(f"full-refresh task slow: {name} took {duration:.2f}s") + except Exception as e: + duration = time.perf_counter() - start + from services.fetch_health import record_failure + + record_failure(name, error=e, duration_s=duration) + logger.exception(f"full-refresh task failed: {name}") logger.info("Full data update complete.") + _scheduler = None +_STARTUP_CCTV_INGEST_DELAY_S = 30 +_FINANCIAL_REFRESH_MINUTES = 30 + + +def _oracle_resolution_sweep(): + """Hourly sweep: check if any markets with active predictions have concluded. + + Resolution logic: + - If a market's end_date has passed AND it's no longer in the active API data → resolved + - For binary markets: final probability determines outcome (>50% = yes, <50% = no) + - For multi-outcome: the outcome with highest final probability wins + """ + try: + from services.mesh.mesh_oracle import oracle_ledger + + active_titles = oracle_ledger.get_active_markets() + if not active_titles: + return + + # Get current market data + with _data_lock: + markets = list(latest_data.get("prediction_markets", [])) + + # Build lookup of active API markets + api_titles = {m.get("title", "").lower(): m for m in markets} + + import time as _time + + now = _time.time() + resolved_count = 0 + + for title in active_titles: + api_market = api_titles.get(title.lower()) + + # If market still in API and end_date hasn't passed, skip + if api_market: + end_date = api_market.get("end_date") + if end_date: + try: + from datetime import datetime, timezone + + dt = datetime.fromisoformat(end_date.replace("Z", "+00:00")) + if dt.timestamp() > now: + continue # Market hasn't ended yet + except Exception: + continue + else: + continue # No end date, can't auto-resolve + + # Market has concluded (past end_date or dropped from API) + # Determine outcome from last known data + if api_market: + outcomes = api_market.get("outcomes", []) + if outcomes and len(outcomes) > 2: + # Multi-outcome: highest pct wins + best = max(outcomes, key=lambda o: o.get("pct", 0)) + outcome = best.get("name", "") + else: + # Binary: consensus > 50 = yes + pct = api_market.get("consensus_pct") or api_market.get("polymarket_pct") or 50 + outcome = "yes" if float(pct) > 50 else "no" + else: + # Market dropped from API entirely — can't determine outcome, skip + logger.warning( + f"Oracle sweep: market '{title}' no longer in API, cannot auto-resolve" + ) + continue + + if not outcome: + continue + + # Resolve both free predictions and market stakes + winners, losers = oracle_ledger.resolve_market(title, outcome) + stake_result = oracle_ledger.resolve_market_stakes(title, outcome) + resolved_count += 1 + logger.info( + f"Oracle sweep resolved '{title}' → {outcome}: " + f"{winners}W/{losers}L free, " + f"{stake_result.get('winners', 0)}W/{stake_result.get('losers', 0)}L staked" + ) + + if resolved_count: + logger.info(f"Oracle sweep complete: {resolved_count} markets resolved") + # Also clean up old data periodically + oracle_ledger.cleanup_old_data() + + except Exception as e: + logger.error(f"Oracle resolution sweep error: {e}") + def start_scheduler(): global _scheduler @@ -113,38 +357,207 @@ def start_scheduler(): _scheduler = BackgroundScheduler(daemon=True) # Fast tier — every 60 seconds - _scheduler.add_job(update_fast_data, 'interval', seconds=60, id='fast_tier', max_instances=1, misfire_grace_time=30) + _scheduler.add_job( + lambda: _run_task_with_health(update_fast_data, "update_fast_data"), + "interval", + seconds=60, + id="fast_tier", + max_instances=1, + misfire_grace_time=30, + ) # Slow tier — every 5 minutes - _scheduler.add_job(update_slow_data, 'interval', minutes=5, id='slow_tier', max_instances=1, misfire_grace_time=120) + _scheduler.add_job( + lambda: _run_task_with_health(update_slow_data, "update_slow_data"), + "interval", + minutes=5, + id="slow_tier", + max_instances=1, + misfire_grace_time=120, + ) + + # Weather alerts — every 5 minutes (time-critical, separate from slow tier) + _scheduler.add_job( + lambda: _run_task_with_health(fetch_weather_alerts, "fetch_weather_alerts"), + "interval", + minutes=5, + id="weather_alerts", + max_instances=1, + misfire_grace_time=60, + ) + + # Ukraine air raid alerts — every 2 minutes (time-critical) + _scheduler.add_job( + lambda: _run_task_with_health(fetch_ukraine_air_raid_alerts, "fetch_ukraine_air_raid_alerts"), + "interval", + minutes=2, + id="ukraine_alerts", + max_instances=1, + misfire_grace_time=60, + ) - # Very slow — every 15 minutes - _scheduler.add_job(fetch_gdelt, 'interval', minutes=15, id='gdelt', max_instances=1, misfire_grace_time=120) - _scheduler.add_job(update_liveuamap, 'interval', minutes=15, id='liveuamap', max_instances=1, misfire_grace_time=120) + # AIS vessel pruning — every 5 minutes (prevents unbounded memory growth) + _scheduler.add_job( + lambda: _run_task_with_health(prune_stale_vessels, "prune_stale_vessels"), + "interval", + minutes=5, + id="ais_prune", + max_instances=1, + misfire_grace_time=60, + ) - # CCTV pipeline refresh — every 10 minutes - # Instantiate once and reuse — avoids re-creating DB connections on every tick + # GDELT — every 30 minutes (downloads 32 ZIP files per call, avoid rate limits) + _scheduler.add_job( + lambda: _run_task_with_health(fetch_gdelt, "fetch_gdelt"), + "interval", + minutes=30, + id="gdelt", + max_instances=1, + misfire_grace_time=120, + ) + _scheduler.add_job( + lambda: _run_task_with_health(update_liveuamap, "update_liveuamap"), + "interval", + minutes=30, + id="liveuamap", + max_instances=1, + misfire_grace_time=120, + ) + + # CCTV pipeline refresh — runs all ingestors, then refreshes in-memory data. + # Delay the first run slightly so startup serves cached/DB-backed data first. from services.cctv_pipeline import ( - TFLJamCamIngestor, LTASingaporeIngestor, - AustinTXIngestor, NYCDOTIngestor, + TFLJamCamIngestor, + LTASingaporeIngestor, + AustinTXIngestor, + NYCDOTIngestor, + CaltransIngestor, + ColoradoDOTIngestor, + WSDOTIngestor, + GeorgiaDOTIngestor, + IllinoisDOTIngestor, + MichiganDOTIngestor, + WindyWebcamsIngestor, + DGTNationalIngestor, + MadridCityIngestor, + OSMTrafficCameraIngestor, + ) + + _cctv_ingestors = [ + (TFLJamCamIngestor(), "cctv_tfl"), + (LTASingaporeIngestor(), "cctv_lta"), + (AustinTXIngestor(), "cctv_atx"), + (NYCDOTIngestor(), "cctv_nyc"), + (CaltransIngestor(), "cctv_caltrans"), + (ColoradoDOTIngestor(), "cctv_codot"), + (WSDOTIngestor(), "cctv_wsdot"), + (GeorgiaDOTIngestor(), "cctv_gdot"), + (IllinoisDOTIngestor(), "cctv_idot"), + (MichiganDOTIngestor(), "cctv_mdot"), + (WindyWebcamsIngestor(), "cctv_windy"), + (DGTNationalIngestor(), "cctv_dgt"), + (MadridCityIngestor(), "cctv_madrid"), + (OSMTrafficCameraIngestor(), "cctv_osm"), + ] + + def _run_cctv_ingest_cycle(): + from services.fetchers._store import is_any_active + + if not is_any_active("cctv"): + return + for ingestor, name in _cctv_ingestors: + _run_task_with_health(ingestor.ingest, name) + # Refresh in-memory CCTV data immediately after ingest + try: + from services.cctv_pipeline import get_all_cameras + from services.fetchers.infrastructure import fetch_cctv + fetch_cctv() + logger.info(f"CCTV ingest cycle complete — {len(get_all_cameras())} cameras in DB") + except Exception as e: + logger.warning(f"CCTV post-ingest refresh failed: {e}") + + _scheduler.add_job( + _run_cctv_ingest_cycle, + "interval", + minutes=10, + id="cctv_ingest", + max_instances=1, + misfire_grace_time=120, + next_run_time=datetime.utcnow() + timedelta(seconds=_STARTUP_CCTV_INGEST_DELAY_S), + ) + + # Financial tickers — every 30 minutes (Yahoo Finance rate-limits aggressively) + def _fetch_financial(): + _run_task_with_health(fetch_financial_markets, "fetch_financial_markets") + + _scheduler.add_job( + _fetch_financial, + "interval", + minutes=_FINANCIAL_REFRESH_MINUTES, + id="financial_tickers", + max_instances=1, + misfire_grace_time=120, + next_run_time=datetime.utcnow() + timedelta(minutes=_FINANCIAL_REFRESH_MINUTES), + ) + + # Unusual Whales — every 15 minutes (congress trades, dark pool, flow alerts) + _scheduler.add_job( + lambda: _run_task_with_health(fetch_unusual_whales, "fetch_unusual_whales"), + "interval", + minutes=15, + id="unusual_whales", + max_instances=1, + misfire_grace_time=120, + ) + + # Meshtastic map API — every 4 hours, fetch global node positions + _scheduler.add_job( + lambda: _run_task_with_health(fetch_meshtastic_nodes, "fetch_meshtastic_nodes"), + "interval", + hours=4, + id="meshtastic_map", + max_instances=1, + misfire_grace_time=600, + ) + + # Oracle resolution sweep — every hour, check if any markets with predictions have concluded + _scheduler.add_job( + lambda: _run_task_with_health(_oracle_resolution_sweep, "oracle_sweep"), + "interval", + hours=1, + id="oracle_sweep", + max_instances=1, + misfire_grace_time=300, + ) + + # VIIRS change detection — every 12 hours (monthly composites, no rush) + _scheduler.add_job( + lambda: _run_task_with_health(fetch_viirs_change_nodes, "fetch_viirs_change_nodes"), + "interval", + hours=12, + id="viirs_change", + max_instances=1, + misfire_grace_time=600, + ) + + # FIMI disinformation index — every 12 hours (weekly editorial feed) + _scheduler.add_job( + lambda: _run_task_with_health(fetch_fimi, "fetch_fimi"), + "interval", + hours=12, + id="fimi", + max_instances=1, + misfire_grace_time=600, ) - _cctv_tfl = TFLJamCamIngestor() - _cctv_lta = LTASingaporeIngestor() - _cctv_atx = AustinTXIngestor() - _cctv_nyc = NYCDOTIngestor() - _now = datetime.now() - _scheduler.add_job(_cctv_tfl.ingest, 'interval', minutes=10, id='cctv_tfl', max_instances=1, misfire_grace_time=120, next_run_time=_now) - _scheduler.add_job(_cctv_lta.ingest, 'interval', minutes=10, id='cctv_lta', max_instances=1, misfire_grace_time=120, next_run_time=_now) - _scheduler.add_job(_cctv_atx.ingest, 'interval', minutes=10, id='cctv_atx', max_instances=1, misfire_grace_time=120, next_run_time=_now) - _scheduler.add_job(_cctv_nyc.ingest, 'interval', minutes=10, id='cctv_nyc', max_instances=1, misfire_grace_time=120, next_run_time=_now) _scheduler.start() logger.info("Scheduler started.") + def stop_scheduler(): if _scheduler: _scheduler.shutdown(wait=False) + def get_latest_data(): - with _data_lock: - return dict(latest_data) + return get_latest_data_subset(*latest_data.keys()) diff --git a/backend/services/env_check.py b/backend/services/env_check.py index 7fd996f2..c8e10971 100644 --- a/backend/services/env_check.py +++ b/backend/services/env_check.py @@ -2,10 +2,16 @@ Ensures required env vars are present before the scheduler starts. Logs warnings for optional keys that degrade functionality when missing. +Audits security-critical config for dangerous combinations. """ + import os +import secrets import sys +import time import logging +from pathlib import Path +from services.config import get_settings logger = logging.getLogger(__name__) @@ -23,9 +29,200 @@ "OPENSKY_CLIENT_ID": "OpenSky OAuth2 — gap-fill flights in Africa/Asia/LatAm", "OPENSKY_CLIENT_SECRET": "OpenSky OAuth2 — gap-fill flights in Africa/Asia/LatAm", "LTA_ACCOUNT_KEY": "Singapore LTA traffic cameras (CCTV layer)", + "PUBLIC_API_KEY": "Optional client auth for public endpoints (recommended for exposed deployments)", } +def _invalid_dm_token_pepper_reason(value: str) -> str: + raw = str(value or "").strip() + lowered = raw.lower() + if not raw: + return "empty" + if lowered in {"change-me", "changeme"}: + return "placeholder" + if len(raw) < 16: + return "too short" + return "" + + +def _invalid_peer_push_secret_reason(value: str) -> str: + raw = str(value or "").strip() + lowered = raw.lower() + if not raw: + return "empty" + if lowered in {"change-me", "changeme"}: + return "placeholder" + if len(raw) < 16: + return "too short" + return "" + + +_PEPPER_FILE = Path(__file__).resolve().parents[1] / "data" / "dm_token_pepper.key" + + +def _ensure_dm_token_pepper(settings) -> str: + token_pepper = str(getattr(settings, "MESH_DM_TOKEN_PEPPER", "") or "").strip() + pepper_reason = _invalid_dm_token_pepper_reason(token_pepper) + if not pepper_reason: + return token_pepper + + # Try loading a previously persisted pepper before generating a new one. + try: + from services.mesh.mesh_secure_storage import read_secure_json + + stored = read_secure_json(_PEPPER_FILE, lambda: {}) + stored_pepper = str(stored.get("pepper", "") or "").strip() + if stored_pepper and not _invalid_dm_token_pepper_reason(stored_pepper): + os.environ["MESH_DM_TOKEN_PEPPER"] = stored_pepper + get_settings.cache_clear() + logger.info("Loaded persisted DM token pepper from %s", _PEPPER_FILE.name) + return stored_pepper + except Exception: + pass + + generated = secrets.token_hex(32) + os.environ["MESH_DM_TOKEN_PEPPER"] = generated + get_settings.cache_clear() + log_fn = logger.warning if bool(getattr(settings, "MESH_DEBUG_MODE", False)) else logger.critical + log_fn( + "⚠️ SECURITY: MESH_DM_TOKEN_PEPPER is invalid (%s) — mailbox tokens " + "would be predictably derivable. Auto-generated a random pepper for " + "this session.", + pepper_reason, + ) + + # Persist so the same pepper survives restarts. + try: + from services.mesh.mesh_secure_storage import write_secure_json + + _PEPPER_FILE.parent.mkdir(parents=True, exist_ok=True) + write_secure_json(_PEPPER_FILE, {"pepper": generated, "generated_at": int(time.time())}) + logger.info("Persisted auto-generated DM token pepper to %s", _PEPPER_FILE.name) + except Exception: + logger.warning("Could not persist auto-generated DM token pepper to disk — will regenerate on next restart") + + return generated + + +def _peer_push_secret_required(settings) -> bool: + relay_peers = str(getattr(settings, "MESH_RELAY_PEERS", "") or "").strip() + rns_peers = str(getattr(settings, "MESH_RNS_PEERS", "") or "").strip() + return bool(getattr(settings, "MESH_RNS_ENABLED", False) or relay_peers or rns_peers) + + +def get_security_posture_warnings(settings=None) -> list[str]: + snapshot = settings or get_settings() + warnings: list[str] = [] + + admin_key = str(getattr(snapshot, "ADMIN_KEY", "") or "").strip() + allow_insecure = bool(getattr(snapshot, "ALLOW_INSECURE_ADMIN", False)) + if allow_insecure and not admin_key: + warnings.append( + "ALLOW_INSECURE_ADMIN=true with no ADMIN_KEY leaves admin and Wormhole endpoints unauthenticated." + ) + + if not bool(getattr(snapshot, "MESH_STRICT_SIGNATURES", True)): + warnings.append( + "MESH_STRICT_SIGNATURES=false is deprecated and ignored; signature enforcement remains mandatory." + ) + + peer_secret = str(getattr(snapshot, "MESH_PEER_PUSH_SECRET", "") or "").strip() + peer_secret_reason = _invalid_peer_push_secret_reason(peer_secret) + if _peer_push_secret_required(snapshot) and peer_secret_reason: + warnings.append( + "MESH_PEER_PUSH_SECRET is invalid " + f"({peer_secret_reason}) while relay or RNS peers are enabled; private peer authentication, opaque gate forwarding, and voter blinding are not secure-by-default." + ) + + if os.name != "nt" and bool(getattr(snapshot, "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK", False)): + warnings.append( + "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK=true stores Wormhole keys in raw local files on this platform." + ) + + if bool(getattr(snapshot, "MESH_RNS_ENABLED", False)) and int(getattr(snapshot, "MESH_RNS_COVER_INTERVAL_S", 0) or 0) <= 0: + warnings.append( + "MESH_RNS_COVER_INTERVAL_S<=0 disables RNS cover traffic outside high-privacy mode, making quiet-node traffic analysis easier." + ) + + fallback_policy = str(getattr(snapshot, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower() + if fallback_policy == "allow": + warnings.append( + "MESH_PRIVATE_CLEARNET_FALLBACK=allow — private-tier messages may fall back to clearnet relay when Tor/RNS is unavailable." + ) + + metadata_persist = bool(getattr(snapshot, "MESH_DM_METADATA_PERSIST", True)) + binding_ttl = int(getattr(snapshot, "MESH_DM_BINDING_TTL_DAYS", 7) or 7) + if metadata_persist and binding_ttl > 14: + warnings.append( + f"MESH_DM_BINDING_TTL_DAYS={binding_ttl} with MESH_DM_METADATA_PERSIST=true — long-lived mailbox binding metadata persists communication graph structure on disk." + ) + + return warnings + + +def _audit_security_config(settings) -> None: + """Audit security-critical config combinations and log loud warnings. + + This does not block startup (dev ergonomics), but makes dangerous + settings impossible to miss in the logs. + """ + # ── 1. ALLOW_INSECURE_ADMIN without ADMIN_KEY ───────────────────── + admin_key = (getattr(settings, "ADMIN_KEY", "") or "").strip() + allow_insecure = bool(getattr(settings, "ALLOW_INSECURE_ADMIN", False)) + if allow_insecure and not admin_key: + logger.critical( + "🚨 SECURITY: ALLOW_INSECURE_ADMIN=true with no ADMIN_KEY — " + "ALL admin/wormhole endpoints are completely unauthenticated. " + "This is acceptable ONLY for local development. " + "Set ADMIN_KEY for any networked or production deployment." + ) + + # ── 2. Signature enforcement ────────────────────────────────────── + mesh_strict = bool(getattr(settings, "MESH_STRICT_SIGNATURES", True)) + if not mesh_strict: + logger.warning( + "⚠️ CONFIG: MESH_STRICT_SIGNATURES=false is deprecated and ignored — " + "runtime signature enforcement remains mandatory." + ) + + # ── 3. Empty DM token pepper ────────────────────────────────────── + _ensure_dm_token_pepper(settings) + + # ── 4. Peer push secret / private-plane integrity ───────────────── + peer_secret = str(getattr(settings, "MESH_PEER_PUSH_SECRET", "") or "").strip() + peer_secret_reason = _invalid_peer_push_secret_reason(peer_secret) + if _peer_push_secret_required(settings) and peer_secret_reason: + log_fn = logger.warning if bool(getattr(settings, "MESH_DEBUG_MODE", False)) else logger.critical + log_fn( + "⚠️ SECURITY: MESH_PEER_PUSH_SECRET is invalid (%s) while relay or RNS peers are enabled — " + "private peer authentication, opaque gate forwarding, and voter blinding are not secure-by-default until it is set to a non-placeholder secret.", + peer_secret_reason, + ) + + # ── 5. Raw secure-storage fallback on non-Windows ──────────────── + if os.name != "nt" and bool(getattr(settings, "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK", False)): + log_fn = logger.warning if bool(getattr(settings, "MESH_DEBUG_MODE", False)) else logger.critical + log_fn( + "⚠️ SECURITY: MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK=true leaves Wormhole keys in raw local files. " + "Use this only for development/CI until a native keyring provider is available." + ) + + # ── 6. Disabled cover traffic outside forced high-privacy mode ───────── + if bool(getattr(settings, "MESH_RNS_ENABLED", False)) and int(getattr(settings, "MESH_RNS_COVER_INTERVAL_S", 0) or 0) <= 0: + logger.warning( + "⚠️ PRIVACY: MESH_RNS_COVER_INTERVAL_S<=0 disables background RNS cover traffic outside high-privacy mode. " + "Quiet nodes become easier to fingerprint by silence and burst timing." + ) + + # ── 7. Clearnet fallback policy ────────────────────────────────── + fallback_policy = str(getattr(settings, "MESH_PRIVATE_CLEARNET_FALLBACK", "block") or "block").strip().lower() + if fallback_policy == "allow": + logger.warning( + "⚠️ PRIVACY: MESH_PRIVATE_CLEARNET_FALLBACK=allow — private-tier messages will fall " + "back to clearnet relay when Tor/RNS is unavailable. Set to 'block' for safer defaults." + ) + + def validate_env(*, strict: bool = True) -> bool: """Validate environment variables at startup. @@ -38,14 +235,20 @@ def validate_env(*, strict: bool = True) -> bool: """ all_ok = True + settings = get_settings() + # Required keys — must be set for key, desc in _REQUIRED.items(): - value = os.environ.get(key, "").strip() + value = getattr(settings, key, "") + if isinstance(value, str): + value = value.strip() if not value: logger.error( "❌ REQUIRED env var %s is not set. %s\n" " Set it in .env or via Docker secrets (%s_FILE).", - key, desc, key, + key, + desc, + key, ) all_ok = False @@ -55,21 +258,32 @@ def validate_env(*, strict: bool = True) -> bool: # Critical-warn keys — app works but security/functionality is degraded for key, desc in _CRITICAL_WARN.items(): - value = os.environ.get(key, "").strip() + value = getattr(settings, key, "") + if isinstance(value, str): + value = value.strip() if not value: - logger.critical( - "🔓 CRITICAL: env var %s is not set — %s\n" - " This is safe for local dev but MUST be set in production.", - key, desc, + allow_insecure = bool(getattr(settings, "ALLOW_INSECURE_ADMIN", False)) + logger.warning( + "⚠️ ADMIN_KEY is not set%s — %s", + " and ALLOW_INSECURE_ADMIN=true" if allow_insecure else "", + desc, ) + if not allow_insecure: + logger.critical( + "🔓 CRITICAL: env var %s is not set — this MUST be set in production.", + key, + ) # Optional keys — warn if missing for key, desc in _OPTIONAL.items(): - value = os.environ.get(key, "").strip() + value = getattr(settings, key, "") + if isinstance(value, str): + value = value.strip() if not value: - logger.warning( - "⚠️ Optional env var %s is not set — %s", key, desc - ) + logger.warning("⚠️ Optional env var %s is not set — %s", key, desc) + + # ── Security posture audit ──────────────────────────────────────── + _audit_security_config(settings) if all_ok: logger.info("✅ Environment validation passed.") diff --git a/backend/services/fetch_health.py b/backend/services/fetch_health.py new file mode 100644 index 00000000..f342aa78 --- /dev/null +++ b/backend/services/fetch_health.py @@ -0,0 +1,94 @@ +"""Fetch health registry — tracks per-source success/failure counts and timings.""" + +import logging +import threading +from datetime import datetime +from typing import Any, Dict, Optional + +from services.fetchers._store import _data_lock, source_freshness + +logger = logging.getLogger(__name__) + +_health: Dict[str, Dict[str, Any]] = {} +_lock = threading.Lock() + + +def _now_iso() -> str: + return datetime.utcnow().isoformat() + + +def _update_source_freshness(source: str, *, ok: bool, error_msg: Optional[str] = None): + """Mirror health summary into shared store for visibility.""" + with _data_lock: + entry = source_freshness.get(source, {}) + if ok: + entry["last_ok"] = _now_iso() + else: + entry["last_error"] = _now_iso() + if error_msg: + entry["last_error_msg"] = error_msg[:200] + source_freshness[source] = entry + + +def record_success(source: str, duration_s: Optional[float] = None, count: Optional[int] = None): + """Record a successful fetch for a source.""" + now = _now_iso() + with _lock: + entry = _health.setdefault( + source, + { + "ok_count": 0, + "error_count": 0, + "last_ok": None, + "last_error": None, + "last_error_msg": None, + "last_duration_ms": None, + "avg_duration_ms": None, + "last_count": None, + }, + ) + entry["ok_count"] += 1 + entry["last_ok"] = now + if duration_s is not None: + dur_ms = round(duration_s * 1000, 1) + entry["last_duration_ms"] = dur_ms + prev_avg = entry["avg_duration_ms"] or 0.0 + n = entry["ok_count"] + entry["avg_duration_ms"] = round(((prev_avg * (n - 1)) + dur_ms) / n, 1) + if count is not None: + entry["last_count"] = count + + _update_source_freshness(source, ok=True) + + +def record_failure(source: str, error: Exception, duration_s: Optional[float] = None): + """Record a failed fetch for a source.""" + now = _now_iso() + err_msg = str(error) + with _lock: + entry = _health.setdefault( + source, + { + "ok_count": 0, + "error_count": 0, + "last_ok": None, + "last_error": None, + "last_error_msg": None, + "last_duration_ms": None, + "avg_duration_ms": None, + "last_count": None, + }, + ) + entry["error_count"] += 1 + entry["last_error"] = now + entry["last_error_msg"] = err_msg[:200] + if duration_s is not None: + entry["last_duration_ms"] = round(duration_s * 1000, 1) + + _update_source_freshness(source, ok=False, error_msg=err_msg) + + +def get_health_snapshot() -> Dict[str, Dict[str, Any]]: + """Return a snapshot of current fetch health state.""" + with _lock: + return {k: dict(v) for k, v in _health.items()} diff --git a/backend/services/fetchers/_store.py b/backend/services/fetchers/_store.py index cfbcc5ad..9a9b5e59 100644 --- a/backend/services/fetchers/_store.py +++ b/backend/services/fetchers/_store.py @@ -3,14 +3,68 @@ Central location for latest_data, source_timestamps, and the data lock. Every fetcher imports from here instead of maintaining its own copy. """ + import threading import logging from datetime import datetime +from typing import Any, Dict, List, Optional, TypedDict logger = logging.getLogger("services.data_fetcher") + +class DashboardData(TypedDict, total=False): + """Schema for the in-memory data store. Catches key typos at dev time.""" + + last_updated: Optional[str] + news: List[Dict[str, Any]] + stocks: Dict[str, Any] + oil: Dict[str, Any] + commercial_flights: List[Dict[str, Any]] + private_flights: List[Dict[str, Any]] + private_jets: List[Dict[str, Any]] + flights: List[Dict[str, Any]] + ships: List[Dict[str, Any]] + military_flights: List[Dict[str, Any]] + tracked_flights: List[Dict[str, Any]] + cctv: List[Dict[str, Any]] + weather: Optional[Dict[str, Any]] + earthquakes: List[Dict[str, Any]] + uavs: List[Dict[str, Any]] + frontlines: Optional[Any] + gdelt: List[Dict[str, Any]] + liveuamap: List[Dict[str, Any]] + kiwisdr: List[Dict[str, Any]] + space_weather: Optional[Dict[str, Any]] + internet_outages: List[Dict[str, Any]] + firms_fires: List[Dict[str, Any]] + datacenters: List[Dict[str, Any]] + airports: List[Dict[str, Any]] + gps_jamming: List[Dict[str, Any]] + satellites: List[Dict[str, Any]] + satellite_source: str + prediction_markets: List[Dict[str, Any]] + sigint: List[Dict[str, Any]] + sigint_totals: Dict[str, Any] + mesh_channel_stats: Dict[str, Any] + meshtastic_map_nodes: List[Dict[str, Any]] + meshtastic_map_fetched_at: Optional[float] + weather_alerts: List[Dict[str, Any]] + air_quality: List[Dict[str, Any]] + volcanoes: List[Dict[str, Any]] + fishing_activity: List[Dict[str, Any]] + satnogs_stations: List[Dict[str, Any]] + satnogs_observations: List[Dict[str, Any]] + tinygs_satellites: List[Dict[str, Any]] + ukraine_alerts: List[Dict[str, Any]] + power_plants: List[Dict[str, Any]] + viirs_change_nodes: List[Dict[str, Any]] + fimi: Dict[str, Any] + psk_reporter: List[Dict[str, Any]] + correlations: List[Dict[str, Any]] + + # In-memory store -latest_data = { +latest_data: DashboardData = { "last_updated": None, "news": [], "stocks": {}, @@ -32,17 +86,158 @@ "firms_fires": [], "datacenters": [], "military_bases": [], - "power_plants": [] + "prediction_markets": [], + "sigint": [], + "sigint_totals": {}, + "mesh_channel_stats": {}, + "meshtastic_map_nodes": [], + "meshtastic_map_fetched_at": None, + "weather_alerts": [], + "air_quality": [], + "volcanoes": [], + "fishing_activity": [], + "satnogs_stations": [], + "satnogs_observations": [], + "tinygs_satellites": [], + "ukraine_alerts": [], + "power_plants": [], + "viirs_change_nodes": [], + "fimi": {}, + "psk_reporter": [], + "correlations": [], } # Per-source freshness timestamps source_timestamps = {} +# Per-source health/freshness metadata (last ok/error) +source_freshness: dict[str, dict] = {} + + def _mark_fresh(*keys): """Record the current UTC time for one or more data source keys.""" now = datetime.utcnow().isoformat() - for k in keys: - source_timestamps[k] = now + with _data_lock: + for k in keys: + source_timestamps[k] = now + # Thread lock for safe reads/writes to latest_data _data_lock = threading.Lock() + +# Monotonic version counter — incremented on each data update cycle. +# Used for cheap ETag generation instead of MD5-hashing the full response. +_data_version: int = 0 + + +def bump_data_version() -> None: + """Increment the data version counter after a fetch cycle completes.""" + global _data_version + _data_version += 1 + + +def get_data_version() -> int: + """Return the current data version (for ETag generation).""" + return _data_version + + +_active_layers_version: int = 0 + + +def bump_active_layers_version() -> None: + """Increment the active-layer version when frontend toggles change response shape.""" + global _active_layers_version + _active_layers_version += 1 + + +def get_active_layers_version() -> int: + """Return the current active-layer version (for ETag generation).""" + return _active_layers_version + + +def get_latest_data_subset(*keys: str) -> DashboardData: + """Return a shallow snapshot of only the requested top-level keys. + + This avoids cloning the entire dashboard store for endpoints that only need + a small tier-specific subset. + """ + with _data_lock: + snap: DashboardData = {} + for key in keys: + value = latest_data.get(key) + if isinstance(value, list): + snap[key] = list(value) + elif isinstance(value, dict): + snap[key] = dict(value) + else: + snap[key] = value + return snap + + +def get_latest_data_subset_refs(*keys: str) -> DashboardData: + """Return direct top-level references for read-only hot paths. + + Writers replace top-level values under the lock instead of mutating them + in place, so readers can safely use these references after releasing the + lock as long as they do not modify them. + """ + with _data_lock: + snap: DashboardData = {} + for key in keys: + snap[key] = latest_data.get(key) + return snap + + +def get_source_timestamps_snapshot() -> dict[str, str]: + """Return a stable copy of per-source freshness timestamps.""" + with _data_lock: + return dict(source_timestamps) + + +# --------------------------------------------------------------------------- +# Active layers — frontend POSTs toggles, fetchers check before running. +# Keep these aligned with the dashboard's default layer state so startup does +# not fetch heavyweight feeds the UI starts with disabled. +# --------------------------------------------------------------------------- +active_layers: dict[str, bool] = { + "flights": True, + "private": True, + "jets": True, + "military": True, + "tracked": True, + "satellites": True, + "ships_military": True, + "ships_cargo": True, + "ships_civilian": True, + "ships_passenger": True, + "ships_tracked_yachts": True, + "earthquakes": True, + "cctv": True, + "ukraine_frontline": True, + "global_incidents": True, + "gps_jamming": True, + "kiwisdr": True, + "scanners": True, + "firms": True, + "internet_outages": True, + "datacenters": True, + "military_bases": True, + "sigint_meshtastic": True, + "sigint_aprs": True, + "weather_alerts": True, + "air_quality": True, + "volcanoes": True, + "fishing_activity": True, + "satnogs": True, + "tinygs": True, + "ukraine_alerts": True, + "power_plants": False, + "viirs_nightlights": False, + "psk_reporter": True, + "correlations": True, +} + + +def is_any_active(*layer_names: str) -> bool: + """Return True if any of the given layer names is currently active.""" + return any(active_layers.get(name, True) for name in layer_names) diff --git a/backend/services/fetchers/earth_observation.py b/backend/services/fetchers/earth_observation.py index a83bc31d..082e70a9 100644 --- a/backend/services/fetchers/earth_observation.py +++ b/backend/services/fetchers/earth_observation.py @@ -1,8 +1,15 @@ -"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar.""" +"""Earth-observation fetchers — earthquakes, FIRMS fires, space weather, weather radar, +severe weather alerts, air quality, volcanoes.""" + import csv import io +import json import logging +import os +import time import heapq +from datetime import datetime +from pathlib import Path from services.network_utils import fetch_with_curl from services.fetchers._store import latest_data, _data_lock, _mark_fresh from services.fetchers.retry import with_retry @@ -15,6 +22,10 @@ # --------------------------------------------------------------------------- @with_retry(max_retries=1, base_delay=1) def fetch_earthquakes(): + from services.fetchers._store import is_any_active + + if not is_any_active("earthquakes"): + return quakes = [] try: url = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/2.5_day.geojson" @@ -24,12 +35,16 @@ def fetch_earthquakes(): for f in features[:50]: mag = f["properties"]["mag"] lng, lat, depth = f["geometry"]["coordinates"] - quakes.append({ - "id": f["id"], "mag": mag, - "lat": lat, "lng": lng, - "place": f["properties"]["place"] - }) - except Exception as e: + quakes.append( + { + "id": f["id"], + "mag": mag, + "lat": lat, + "lng": lng, + "place": f["properties"]["place"], + } + ) + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching earthquakes: {e}") with _data_lock: latest_data["earthquakes"] = quakes @@ -43,6 +58,10 @@ def fetch_earthquakes(): @with_retry(max_retries=1, base_delay=2) def fetch_firms_fires(): """Fetch global fire/thermal anomalies from NASA FIRMS (NOAA-20 VIIRS, 24h, no key needed).""" + from services.fetchers._store import is_any_active + + if not is_any_active("firms"): + return fires = [] try: url = "https://firms.modaps.eosdis.nasa.gov/data/active_fire/noaa-20-viirs-c2/csv/J1_VIIRS_C2_Global_24h.csv" @@ -58,18 +77,23 @@ def fetch_firms_fires(): conf = row.get("confidence", "nominal") daynight = row.get("daynight", "") bright = float(row.get("bright_ti4", 0)) - all_rows.append({ - "lat": lat, "lng": lng, "frp": frp, - "brightness": bright, "confidence": conf, - "daynight": daynight, - "acq_date": row.get("acq_date", ""), - "acq_time": row.get("acq_time", ""), - }) + all_rows.append( + { + "lat": lat, + "lng": lng, + "frp": frp, + "brightness": bright, + "confidence": conf, + "daynight": daynight, + "acq_date": row.get("acq_date", ""), + "acq_time": row.get("acq_time", ""), + } + ) except (ValueError, TypeError): continue fires = heapq.nlargest(5000, all_rows, key=lambda x: x["frp"]) logger.info(f"FIRMS fires: {len(fires)} hotspots (from {response.status_code})") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching FIRMS fires: {e}") with _data_lock: latest_data["firms_fires"] = fires @@ -77,6 +101,90 @@ def fetch_firms_fires(): _mark_fresh("firms_fires") +# --------------------------------------------------------------------------- +# NASA FIRMS Country-Scoped Fires (enriches global CSV with conflict zones) +# --------------------------------------------------------------------------- +# Conflict-zone countries of interest for higher-detail fire/thermal data +_FIRMS_COUNTRIES = ["ISR", "IRN", "IRQ", "LBN", "SYR", "YEM", "SAU", "UKR", "RUS", "TUR"] + + +@with_retry(max_retries=1, base_delay=2) +def fetch_firms_country_fires(): + """Fetch country-scoped fire hotspots from NASA FIRMS MAP_KEY API. + + Supplements the global CSV feed with more granular data for conflict zones. + Merges results into the existing firms_fires data store (no new frontend key). + Requires FIRMS_MAP_KEY env var (free from NASA Earthdata). Skips if not set. + """ + from services.fetchers._store import is_any_active + + if not is_any_active("firms"): + return + + map_key = os.environ.get("FIRMS_MAP_KEY", "") + if not map_key: + logger.debug("FIRMS_MAP_KEY not set, skipping country-scoped FIRMS fetch") + return + + # Build a set of existing (lat, lng) rounded to 0.01° for dedup + with _data_lock: + existing = set() + for f in latest_data.get("firms_fires", []): + existing.add((round(f["lat"], 2), round(f["lng"], 2))) + + new_fires = [] + for country in _FIRMS_COUNTRIES: + try: + url = ( + f"https://firms.modaps.eosdis.nasa.gov/api/country/csv/" + f"{map_key}/VIIRS_NOAA20_NRT/{country}/1" + ) + response = fetch_with_curl(url, timeout=15) + if response.status_code != 200: + logger.debug(f"FIRMS country {country}: HTTP {response.status_code}") + continue + + reader = csv.DictReader(io.StringIO(response.text)) + for row in reader: + try: + lat = float(row.get("latitude", 0)) + lng = float(row.get("longitude", 0)) + key = (round(lat, 2), round(lng, 2)) + if key in existing: + continue # Already in global data + existing.add(key) + + frp = float(row.get("frp", 0)) + new_fires.append({ + "lat": lat, + "lng": lng, + "frp": frp, + "brightness": float(row.get("bright_ti4", 0)), + "confidence": row.get("confidence", "nominal"), + "daynight": row.get("daynight", ""), + "acq_date": row.get("acq_date", ""), + "acq_time": row.get("acq_time", ""), + }) + except (ValueError, TypeError): + continue + + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: + logger.debug(f"FIRMS country {country} failed: {e}") + + if new_fires: + with _data_lock: + current = latest_data.get("firms_fires", []) + merged = current + new_fires + # Keep top 6000 by FRP (slightly more than global-only cap of 5000) + if len(merged) > 6000: + merged = heapq.nlargest(6000, merged, key=lambda x: x["frp"]) + latest_data["firms_fires"] = merged + logger.info(f"FIRMS country enrichment: +{len(new_fires)} fires from {len(_FIRMS_COUNTRIES)} countries") + _mark_fresh("firms_fires") + else: + logger.debug("FIRMS country enrichment: no new fires found") + + # --------------------------------------------------------------------------- # Space Weather (NOAA SWPC) # --------------------------------------------------------------------------- @@ -84,7 +192,9 @@ def fetch_firms_fires(): def fetch_space_weather(): """Fetch NOAA SWPC Kp index and recent solar events.""" try: - kp_resp = fetch_with_curl("https://services.swpc.noaa.gov/json/planetary_k_index_1m.json", timeout=10) + kp_resp = fetch_with_curl( + "https://services.swpc.noaa.gov/json/planetary_k_index_1m.json", timeout=10 + ) kp_value = None kp_text = "QUIET" if kp_resp.status_code == 200: @@ -102,16 +212,20 @@ def fetch_space_weather(): kp_text = "UNSETTLED" events = [] - ev_resp = fetch_with_curl("https://services.swpc.noaa.gov/json/edited_events.json", timeout=10) + ev_resp = fetch_with_curl( + "https://services.swpc.noaa.gov/json/edited_events.json", timeout=10 + ) if ev_resp.status_code == 200: all_events = ev_resp.json() for ev in all_events[-10:]: - events.append({ - "type": ev.get("type", ""), - "begin": ev.get("begin", ""), - "end": ev.get("end", ""), - "classtype": ev.get("classtype", ""), - }) + events.append( + { + "type": ev.get("type", ""), + "begin": ev.get("begin", ""), + "end": ev.get("end", ""), + "classtype": ev.get("classtype", ""), + } + ) with _data_lock: latest_data["space_weather"] = { @@ -121,7 +235,7 @@ def fetch_space_weather(): } _mark_fresh("space_weather") logger.info(f"Space weather: Kp={kp_value} ({kp_text}), {len(events)} events") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching space weather: {e}") @@ -138,7 +252,347 @@ def fetch_weather(): if "radar" in data and "past" in data["radar"]: latest_time = data["radar"]["past"][-1]["time"] with _data_lock: - latest_data["weather"] = {"time": latest_time, "host": data.get("host", "https://tilecache.rainviewer.com")} + latest_data["weather"] = { + "time": latest_time, + "host": data.get("host", "https://tilecache.rainviewer.com"), + } _mark_fresh("weather") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching weather: {e}") + + +# --------------------------------------------------------------------------- +# NOAA/NWS Severe Weather Alerts +# --------------------------------------------------------------------------- +@with_retry(max_retries=1, base_delay=2) +def fetch_weather_alerts(): + """Fetch active severe weather alerts from NOAA/NWS (US coverage, GeoJSON polygons).""" + from services.fetchers._store import is_any_active + + if not is_any_active("weather_alerts"): + return + alerts = [] + try: + url = "https://api.weather.gov/alerts/active?status=actual" + headers = { + "User-Agent": "(ShadowBroker OSINT Dashboard, github.com/BigBodyCobain/Shadowbroker)", + "Accept": "application/geo+json", + } + response = fetch_with_curl(url, timeout=15, headers=headers) + if response.status_code == 200: + features = response.json().get("features", []) + for f in features: + props = f.get("properties", {}) + geom = f.get("geometry") + if not geom: + continue # skip zone-only alerts with no polygon + alerts.append( + { + "id": props.get("id", ""), + "event": props.get("event", ""), + "severity": props.get("severity", "Unknown"), + "certainty": props.get("certainty", ""), + "urgency": props.get("urgency", ""), + "headline": props.get("headline", ""), + "description": (props.get("description", "") or "")[:300], + "expires": props.get("expires", ""), + "geometry": geom, + } + ) + logger.info(f"Weather alerts: {len(alerts)} active (with polygons)") + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: + logger.error(f"Error fetching weather alerts: {e}") + with _data_lock: + latest_data["weather_alerts"] = alerts + if alerts: + _mark_fresh("weather_alerts") + + +# --------------------------------------------------------------------------- +# Air Quality (OpenAQ v3) +# --------------------------------------------------------------------------- +def _pm25_to_aqi(pm25: float) -> int: + """Convert PM2.5 concentration (µg/m³) to US EPA AQI.""" + breakpoints = [ + (0, 12.0, 0, 50), + (12.1, 35.4, 51, 100), + (35.5, 55.4, 101, 150), + (55.5, 150.4, 151, 200), + (150.5, 250.4, 201, 300), + (250.5, 500.4, 301, 500), + ] + for c_lo, c_hi, i_lo, i_hi in breakpoints: + if pm25 <= c_hi: + return round(((i_hi - i_lo) / (c_hi - c_lo)) * (pm25 - c_lo) + i_lo) + return 500 + + +@with_retry(max_retries=1, base_delay=2) +def fetch_air_quality(): + """Fetch global air quality stations with PM2.5 data from OpenAQ.""" + from services.fetchers._store import is_any_active + + if not is_any_active("air_quality"): + return + stations = [] + api_key = os.environ.get("OPENAQ_API_KEY", "") + if not api_key: + logger.debug("OPENAQ_API_KEY not set, skipping air quality fetch") + return + try: + url = "https://api.openaq.org/v3/locations?limit=5000¶meter_id=2&order_by=datetime&sort_order=desc" + headers = {"X-API-Key": api_key} + response = fetch_with_curl(url, timeout=30, headers=headers) + if response.status_code == 200: + results = response.json().get("results", []) + for loc in results: + coords = loc.get("coordinates", {}) + lat = coords.get("latitude") + lng = coords.get("longitude") + if lat is None or lng is None: + continue + pm25 = None + for p in loc.get("parameters", []): + if p.get("id") == 2: + pm25 = p.get("lastValue") + break + if pm25 is None: + continue + pm25_val = float(pm25) + if pm25_val < 0: + continue + stations.append( + { + "id": loc.get("id"), + "name": loc.get("name", "Unknown"), + "lat": lat, + "lng": lng, + "pm25": round(pm25_val, 1), + "aqi": _pm25_to_aqi(pm25_val), + "country": loc.get("country", {}).get("code", ""), + } + ) + logger.info(f"Air quality: {len(stations)} stations") + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: + logger.error(f"Error fetching air quality: {e}") + with _data_lock: + latest_data["air_quality"] = stations + if stations: + _mark_fresh("air_quality") + + +# --------------------------------------------------------------------------- +# Volcanoes (Smithsonian Global Volcanism Program) +# --------------------------------------------------------------------------- +@with_retry(max_retries=2, base_delay=5) +def fetch_volcanoes(): + """Fetch Holocene volcanoes from Smithsonian GVP WFS (static reference data).""" + from services.fetchers._store import is_any_active + + if not is_any_active("volcanoes"): + return + volcanoes = [] + try: + url = ( + "https://webservices.volcano.si.edu/geoserver/GVP-VOTW/wfs" + "?service=WFS&version=2.0.0&request=GetFeature" + "&typeName=GVP-VOTW:E3WebApp_HoloceneVolcanoes" + "&outputFormat=application/json" + ) + response = fetch_with_curl(url, timeout=30) + if response.status_code == 200: + features = response.json().get("features", []) + for f in features: + props = f.get("properties", {}) + geom = f.get("geometry", {}) + coords = geom.get("coordinates", [None, None]) + if coords[0] is None: + continue + last_eruption = props.get("LastEruption") + last_eruption_year = None + if last_eruption is not None: + try: + last_eruption_year = int(last_eruption) + except (ValueError, TypeError): + pass + volcanoes.append( + { + "name": props.get("VolcanoName", "Unknown"), + "type": props.get("VolcanoType", ""), + "country": props.get("Country", ""), + "region": props.get("TectonicSetting", ""), + "elevation": props.get("Elevation", 0), + "last_eruption_year": last_eruption_year, + "lat": coords[1], + "lng": coords[0], + } + ) + logger.info(f"Volcanoes: {len(volcanoes)} Holocene volcanoes loaded") + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: + logger.error(f"Error fetching volcanoes: {e}") + with _data_lock: + latest_data["volcanoes"] = volcanoes + if volcanoes: + _mark_fresh("volcanoes") + + +# --------------------------------------------------------------------------- +# VIIRS Night Lights Change Detection (Google Earth Engine — optional) +# --------------------------------------------------------------------------- +_VIIRS_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "viirs_change_nodes.json" +_VIIRS_CACHE_MAX_AGE_S = 86400 # 24 hours + +# Conflict-zone AOIs: (name, south, west, north, east) +_VIIRS_AOIS = [ + ("Gaza Strip", 31.2, 34.2, 31.6, 34.6), + ("Kharkiv Oblast", 48.5, 35.0, 50.5, 38.5), + ("Donetsk Oblast", 47.0, 36.5, 49.0, 39.5), + ("Zaporizhzhia Oblast", 46.5, 34.5, 48.5, 37.0), + ("Aleppo", 35.8, 36.5, 36.5, 37.5), + ("Khartoum", 15.2, 32.2, 15.9, 32.9), + ("Sana'a", 14.9, 43.8, 15.6, 44.5), + ("Mosul", 36.0, 42.8, 36.7, 43.5), + ("Mariupol", 46.9, 37.2, 47.3, 37.8), + ("Southern Lebanon", 33.0, 35.0, 33.5, 36.0), +] + +_VIIRS_SEVERITY_THRESHOLDS = [ + (-100, -70, "severe"), + (-70, -50, "high"), + (-50, -30, "moderate"), + (30, 100, "growth"), + (100, 500, "rapid_growth"), +] + + +def _classify_viirs_severity(pct_change: float): + for lo, hi, label in _VIIRS_SEVERITY_THRESHOLDS: + if lo <= pct_change <= hi: + return label + return None + + +def _load_viirs_stale_cache(): + """Load stale cache if available (when GEE is not configured).""" + if _VIIRS_CACHE_PATH.exists(): + try: + cached = json.loads(_VIIRS_CACHE_PATH.read_text(encoding="utf-8")) + with _data_lock: + latest_data["viirs_change_nodes"] = cached + _mark_fresh("viirs_change_nodes") + logger.info(f"VIIRS change nodes: loaded {len(cached)} from stale cache") + except Exception: + pass + + +@with_retry(max_retries=1, base_delay=5) +def fetch_viirs_change_nodes(): + """Compute VIIRS nighttime radiance change nodes via GEE (optional).""" + from services.fetchers._store import is_any_active + + if not is_any_active("viirs_nightlights"): + return + + # Check cache freshness first + if _VIIRS_CACHE_PATH.exists(): + age = time.time() - _VIIRS_CACHE_PATH.stat().st_mtime + if age < _VIIRS_CACHE_MAX_AGE_S: + try: + cached = json.loads(_VIIRS_CACHE_PATH.read_text(encoding="utf-8")) + with _data_lock: + latest_data["viirs_change_nodes"] = cached + _mark_fresh("viirs_change_nodes") + logger.info(f"VIIRS change nodes: loaded {len(cached)} from cache (age {age:.0f}s)") + return + except Exception as e: + logger.warning(f"VIIRS cache read failed: {e}") + + # Try importing earthengine-api (optional dependency) + try: + import ee + except ImportError: + logger.debug("earthengine-api not installed, skipping VIIRS change detection") + _load_viirs_stale_cache() + return + + # Authenticate with service account + sa_key_path = os.environ.get("GEE_SERVICE_ACCOUNT_KEY", "") + if not sa_key_path: + logger.debug("GEE_SERVICE_ACCOUNT_KEY not set, skipping VIIRS change detection") + _load_viirs_stale_cache() + return + + try: + credentials = ee.ServiceAccountCredentials(None, key_file=sa_key_path) + ee.Initialize(credentials) + except Exception as e: + logger.error(f"GEE authentication failed: {e}") + _load_viirs_stale_cache() + return + + # Compute change nodes for each AOI + nodes = [] + viirs = ee.ImageCollection("NOAA/VIIRS/DNB/MONTHLY_V1/VCMCFG").select("avg_rad") + + for aoi_name, s_lat, w_lng, n_lat, e_lng in _VIIRS_AOIS: + try: + aoi = ee.Geometry.Rectangle([w_lng, s_lat, e_lng, n_lat]) + + # Most recent available date + now = ee.Date(datetime.utcnow().isoformat()[:10]) + + # Current: 12-month rolling mean ending now + current = viirs.filterDate(now.advance(-12, "month"), now).mean().clip(aoi) + + # Baseline: 12-month mean ending 12 months ago + baseline = viirs.filterDate( + now.advance(-24, "month"), now.advance(-12, "month") + ).mean().clip(aoi) + + # Floor baseline at 0.5 nW/cm²/sr to avoid div-by-zero in dark areas + baseline_safe = baseline.max(0.5) + + # Percentage change + change = current.subtract(baseline).divide(baseline_safe).multiply(100) + + # Only keep pixels with >30% absolute change + sig_mask = change.abs().gt(30) + change_masked = change.updateMask(sig_mask) + + # Sample up to 200 points per AOI + samples = change_masked.sample( + region=aoi, scale=500, numPixels=200, geometries=True + ) + sample_list = samples.getInfo() + + for feat in sample_list.get("features", []): + coords = feat["geometry"]["coordinates"] + pct = feat["properties"].get("avg_rad", 0) + severity = _classify_viirs_severity(pct) + if severity is None: + continue + nodes.append({ + "lat": round(coords[1], 4), + "lng": round(coords[0], 4), + "mean_change_pct": round(pct, 1), + "severity": severity, + "aoi_name": aoi_name, + }) + except Exception as e: + logger.warning(f"VIIRS change detection failed for {aoi_name}: {e}") + continue + + # Save to cache + try: + _VIIRS_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) + _VIIRS_CACHE_PATH.write_text( + json.dumps(nodes, separators=(",", ":")), encoding="utf-8" + ) + except Exception as e: + logger.warning(f"Failed to write VIIRS cache: {e}") + + with _data_lock: + latest_data["viirs_change_nodes"] = nodes + if nodes: + _mark_fresh("viirs_change_nodes") + logger.info(f"VIIRS change nodes: {len(nodes)} nodes from {len(_VIIRS_AOIS)} AOIs") diff --git a/backend/services/fetchers/emissions.py b/backend/services/fetchers/emissions.py new file mode 100644 index 00000000..de63fedb --- /dev/null +++ b/backend/services/fetchers/emissions.py @@ -0,0 +1,131 @@ +""" +Fuel burn & CO2 emissions estimator for private jets. +Based on manufacturer-published cruise fuel burn rates (GPH at long-range cruise). +1 US gallon of Jet-A produces ~21.1 lbs (9.57 kg) of CO2. +""" + +JET_A_CO2_KG_PER_GALLON = 9.57 + +# ICAO type code -> gallons per hour at long-range cruise +FUEL_BURN_GPH: dict[str, int] = { + # Gulfstream + "GLF6": 430, # G650/G650ER + "G700": 480, # G700 + "GLF5": 390, # G550 + "GVSP": 400, # GV-SP + "GLF4": 330, # G-IV + # Bombardier + "GL7T": 490, # Global 7500 + "GLEX": 430, # Global Express/6000/6500 + "GL5T": 420, # Global 5000/5500 + "CL35": 220, # Challenger 350 + "CL60": 310, # Challenger 604/605 + "CL30": 200, # Challenger 300 + "CL65": 320, # Challenger 650 + # Dassault + "F7X": 350, # Falcon 7X + "F8X": 370, # Falcon 8X + "F900": 285, # Falcon 900/900EX/900LX + "F2TH": 230, # Falcon 2000 + "FA50": 240, # Falcon 50 + # Cessna + "CITX": 280, # Citation X + "C68A": 195, # Citation Latitude + "C700": 230, # Citation Longitude + "C680": 220, # Citation Sovereign + "C560": 190, # Citation Excel/XLS + "C510": 75, # Citation Mustang + "CJ3": 120, # CJ3 + "CJ4": 135, # CJ4 + # Boeing + "B737": 850, # BBJ (737) + "B738": 920, # BBJ2 (737-800) + "B752": 1100, # 757-200 + "B762": 1400, # 767-200 + "B788": 1200, # 787-8 + # Airbus + "A318": 780, # ACJ318 + "A319": 850, # ACJ319 + "A320": 900, # ACJ320 + "A343": 1800, # A340-300 + "A346": 2100, # A340-600 + # Pilatus + "PC24": 115, # PC-24 + "PC12": 60, # PC-12 + # Embraer + "E55P": 185, # Legacy 500 + "E135": 300, # Legacy 600/650 + "E50P": 135, # Phenom 300 + "E500": 80, # Phenom 100 + # Learjet + "LJ60": 195, # Learjet 60 + "LJ75": 185, # Learjet 75 + "LJ45": 175, # Learjet 45 + # Hawker + "H25B": 210, # Hawker 800/800XP + "H25C": 215, # Hawker 900XP + # Beechcraft + "B350": 100, # King Air 350 + "B200": 80, # King Air 200/250 +} + +# Common string names -> ICAO type code +_ALIASES: dict[str, str] = { + "Gulfstream G650": "GLF6", "Gulfstream G650ER": "GLF6", "G650": "GLF6", "G650ER": "GLF6", + "Gulfstream G700": "G700", + "Gulfstream G550": "GLF5", "G550": "GLF5", "G500": "GLF5", + "Gulfstream GV": "GVSP", "Gulfstream G-V": "GVSP", "GV": "GVSP", + "Gulfstream G-IV": "GLF4", "Gulfstream GIV": "GLF4", "G450": "GLF4", + "Global 7500": "GL7T", "Bombardier Global 7500": "GL7T", + "Global 6000": "GLEX", "Global Express": "GLEX", "Bombardier Global 6000": "GLEX", + "Global 5000": "GL5T", + "Challenger 350": "CL35", "Challenger 300": "CL30", + "Challenger 604": "CL60", "Challenger 605": "CL60", "Challenger 650": "CL65", + "Falcon 7X": "F7X", "Dassault Falcon 7X": "F7X", + "Falcon 8X": "F8X", "Dassault Falcon 8X": "F8X", + "Falcon 900": "F900", "Falcon 900LX": "F900", "Falcon 900EX": "F900", + "Falcon 2000": "F2TH", + "Citation X": "CITX", "Citation Latitude": "C68A", "Citation Longitude": "C700", + "Boeing 757-200": "B752", "757-200": "B752", "Boeing 757": "B752", + "Boeing 767-200": "B762", "767-200": "B762", "Boeing 767": "B762", + "Boeing 787-8": "B788", "Boeing 787": "B788", + "Boeing 737": "B737", "737 BBJ": "B737", "BBJ": "B737", + "Airbus A340-300": "A343", "A340-300": "A343", "A340": "A343", + "Airbus A318": "A318", + "Pilatus PC-24": "PC24", "PC-24": "PC24", + "Legacy 500": "E55P", "Legacy 600": "E135", "Phenom 300": "E50P", + "Learjet 60": "LJ60", "Learjet 75": "LJ75", + "Hawker 800": "H25B", "Hawker 900XP": "H25C", + "King Air 350": "B350", "King Air 200": "B200", +} + + +def get_emissions_info(model: str) -> dict | None: + """ + Given an aircraft model string (ICAO type code or common name), + return emissions info dict or None if unknown. + """ + if not model: + return None + model_clean = model.strip() + # Try direct ICAO code match first + gph = FUEL_BURN_GPH.get(model_clean.upper()) + if gph is None: + # Try alias lookup + code = _ALIASES.get(model_clean) + if code: + gph = FUEL_BURN_GPH.get(code) + if gph is None: + # Fuzzy: check if any alias is a substring + model_lower = model_clean.lower() + for alias, code in _ALIASES.items(): + if alias.lower() in model_lower or model_lower in alias.lower(): + gph = FUEL_BURN_GPH.get(code) + if gph: + break + if gph is None: + return None + return { + "fuel_gph": gph, + "co2_kg_per_hour": round(gph * JET_A_CO2_KG_PER_GALLON, 1), + } diff --git a/backend/services/fetchers/fimi.py b/backend/services/fetchers/fimi.py new file mode 100644 index 00000000..c81ff4ba --- /dev/null +++ b/backend/services/fetchers/fimi.py @@ -0,0 +1,274 @@ +"""EUvsDisinfo FIMI (Foreign Information Manipulation & Interference) fetcher. + +Parses the EUvsDisinfo RSS feed to extract disinformation narratives, +debunked claims, threat actor mentions, and target country references. +Refreshes every 12 hours (FIMI data updates weekly). +""" + +import re +import logging +from datetime import datetime, timezone + +import feedparser +from services.network_utils import fetch_with_curl +from services.fetchers._store import latest_data, _data_lock, _mark_fresh +from services.fetchers.retry import with_retry + +logger = logging.getLogger("services.data_fetcher") + +_FIMI_FEED_URL = "https://euvsdisinfo.eu/feed/" + +# ── Threat actor keywords ────────────────────────────────────────────────── +# Map of keyword → canonical actor name. Checked case-insensitively. +_THREAT_ACTORS: dict[str, str] = { + "russia": "Russia", + "russian": "Russia", + "kremlin": "Russia", + "pro-kremlin": "Russia", + "moscow": "Russia", + "china": "China", + "chinese": "China", + "beijing": "China", + "iran": "Iran", + "iranian": "Iran", + "tehran": "Iran", + "north korea": "North Korea", + "pyongyang": "North Korea", + "dprk": "North Korea", + "belarus": "Belarus", + "belarusian": "Belarus", + "minsk": "Belarus", +} + +# ── Target country/region keywords ───────────────────────────────────────── +_TARGET_KEYWORDS: dict[str, str] = { + "ukraine": "Ukraine", + "kyiv": "Ukraine", + "moldova": "Moldova", + "georgia": "Georgia", + "tbilisi": "Georgia", + "eu": "EU", + "european union": "EU", + "europe": "Europe", + "nato": "NATO", + "united states": "United States", + "usa": "United States", + "germany": "Germany", + "france": "France", + "poland": "Poland", + "baltic": "Baltics", + "lithuania": "Baltics", + "latvia": "Baltics", + "estonia": "Baltics", + "romania": "Romania", + "czech": "Czech Republic", + "slovakia": "Slovakia", + "armenia": "Armenia", + "africa": "Africa", + "middle east": "Middle East", + "syria": "Syria", + "israel": "Israel", + "serbia": "Serbia", + "india": "India", + "brazil": "Brazil", +} + +# ── Disinformation topic keywords (for cross-referencing news) ───────────── +_DISINFO_TOPICS = [ + "sanctions", + "energy crisis", + "gas supply", + "nuclear threat", + "nato expansion", + "biolab", + "biological weapon", + "provocation", + "false flag", + "staged", + "nazi", + "genocide", + "referendum", + "regime change", + "coup", + "puppet government", + "election interference", + "election meddling", + "voter fraud", + "migrant invasion", + "refugee crisis", + "civil war", + "food crisis", + "grain deal", +] + +# Regex for extracting debunked report URLs from feed HTML +_REPORT_URL_RE = re.compile( + r'https?://euvsdisinfo\.eu/report/[a-z0-9\-]+/?', + re.IGNORECASE, +) + +# Regex for extracting the claim title from a report URL slug +_SLUG_RE = re.compile(r'/report/([a-z0-9\-]+)/?$', re.IGNORECASE) + + +def _slug_to_title(url: str) -> str: + """Convert a report URL slug to a human-readable title.""" + m = _SLUG_RE.search(url) + if not m: + return url + return m.group(1).replace("-", " ").title() + + +def _count_mentions(text: str, keywords: dict[str, str]) -> dict[str, int]: + """Count keyword mentions, mapping to canonical names.""" + counts: dict[str, int] = {} + text_lower = text.lower() + for kw, canonical in keywords.items(): + # Word-boundary match, case-insensitive + pattern = r'\b' + re.escape(kw) + r'\b' + matches = re.findall(pattern, text_lower) + if matches: + counts[canonical] = counts.get(canonical, 0) + len(matches) + return counts + + +def _extract_disinfo_keywords(text: str) -> list[str]: + """Return which disinformation topic keywords appear in the text.""" + text_lower = text.lower() + found = [] + for topic in _DISINFO_TOPICS: + if topic in text_lower: + found.append(topic) + return found + + +def _is_major_wave(narratives: list[dict], targets: dict[str, int]) -> bool: + """Heuristic: detect a 'major disinformation wave'. + + Triggers when: + - 3+ narratives in the feed mention the same target, OR + - A single target has 10+ total mentions across all narratives, OR + - 5+ distinct debunked claims extracted in one fetch + """ + if not narratives: + return False + + # Check per-target narrative count + target_narrative_counts: dict[str, int] = {} + total_claims = 0 + for n in narratives: + for t in n.get("targets", []): + target_narrative_counts[t] = target_narrative_counts.get(t, 0) + 1 + total_claims += len(n.get("claims", [])) + + if any(c >= 3 for c in target_narrative_counts.values()): + return True + if any(c >= 10 for c in targets.values()): + return True + if total_claims >= 5: + return True + return False + + +@with_retry(max_retries=1, base_delay=5) +def fetch_fimi(): + """Fetch and parse the EUvsDisinfo RSS feed.""" + try: + resp = fetch_with_curl(_FIMI_FEED_URL, timeout=15) + feed = feedparser.parse(resp.text) + except Exception as e: + logger.warning(f"FIMI feed fetch failed: {e}") + return + + if not feed.entries: + logger.warning("FIMI feed: no entries found") + return + + narratives = [] + all_claims: list[dict] = [] + agg_actors: dict[str, int] = {} + agg_targets: dict[str, int] = {} + all_disinfo_kw: set[str] = set() + + for entry in feed.entries[:15]: # Cap at 15 entries + title = entry.get("title", "") + link = entry.get("link", "") + published = entry.get("published", "") + summary_html = entry.get("summary", "") or entry.get("description", "") + + # Strip HTML tags for text analysis + summary_text = re.sub(r"<[^>]+>", " ", summary_html) + summary_text = re.sub(r"\s+", " ", summary_text).strip() + full_text = f"{title} {summary_text}" + + # Extract debunked report URLs + report_urls = list(set(_REPORT_URL_RE.findall(summary_html))) + claims = [{"url": url, "title": _slug_to_title(url)} for url in report_urls] + all_claims.extend(claims) + + # Count threat actors + actors = _count_mentions(full_text, _THREAT_ACTORS) + for actor, count in actors.items(): + agg_actors[actor] = agg_actors.get(actor, 0) + count + + # Count target countries + targets = _count_mentions(full_text, _TARGET_KEYWORDS) + for target, count in targets.items(): + agg_targets[target] = agg_targets.get(target, 0) + count + + # Extract disinfo topic keywords + disinfo_kw = _extract_disinfo_keywords(full_text) + all_disinfo_kw.update(disinfo_kw) + + # Truncate summary for storage + snippet = summary_text[:300] + ("..." if len(summary_text) > 300 else "") + + narratives.append({ + "title": title, + "link": link, + "published": published, + "snippet": snippet, + "claims": claims, + "actors": list(actors.keys()), + "targets": list(targets.keys()), + "disinfo_keywords": disinfo_kw, + }) + + # Sort actors and targets by count (descending) + sorted_actors = dict(sorted(agg_actors.items(), key=lambda x: x[1], reverse=True)) + sorted_targets = dict(sorted(agg_targets.items(), key=lambda x: x[1], reverse=True)) + + # Deduplicate claims + seen_urls: set[str] = set() + unique_claims = [] + for c in all_claims: + if c["url"] not in seen_urls: + seen_urls.add(c["url"]) + unique_claims.append(c) + + major_wave = _is_major_wave(narratives, sorted_targets) + + fimi_data = { + "narratives": narratives, + "claims": unique_claims, + "threat_actors": sorted_actors, + "targets": sorted_targets, + "disinfo_keywords": sorted(all_disinfo_kw), + "major_wave": major_wave, + "major_wave_target": ( + max(sorted_targets, key=sorted_targets.get) if major_wave and sorted_targets else None + ), + "last_fetched": datetime.now(timezone.utc).isoformat(), + "source": "EUvsDisinfo", + "source_url": "https://euvsdisinfo.eu", + } + + with _data_lock: + latest_data["fimi"] = fimi_data + _mark_fresh("fimi") + logger.info( + f"FIMI fetch complete: {len(narratives)} narratives, " + f"{len(unique_claims)} claims, " + f"{len(sorted_actors)} actors, " + f"major_wave={major_wave}" + ) diff --git a/backend/services/fetchers/financial.py b/backend/services/fetchers/financial.py index f2079302..54068ee1 100644 --- a/backend/services/fetchers/financial.py +++ b/backend/services/fetchers/financial.py @@ -1,97 +1,161 @@ -"""Financial data fetchers — defense stocks and oil prices. - -Uses yfinance batch download to minimise Yahoo Finance requests and avoid rate limiting. -""" import logging -import yfinance as yf +import math +import random +import time +import os +import urllib.request +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone from services.fetchers._store import latest_data, _data_lock, _mark_fresh from services.fetchers.retry import with_retry logger = logging.getLogger(__name__) +_YFINANCE_REQUEST_DELAY_SECONDS = 0.5 +_YFINANCE_REQUEST_JITTER_SECONDS = 0.2 -def _batch_fetch(symbols: list[str], period: str = "5d") -> dict: - """Fetch multiple tickers in a single yfinance request. Returns {symbol: {price, change_percent, up}}.""" - try: - hist = yf.download(symbols, period=period, auto_adjust=True, progress=False) - if hist.empty: - return {} - close = hist["Close"] - result = {} - for sym in symbols: - try: - col = close[sym] if len(symbols) > 1 else close - col = col.dropna() - if len(col) < 1: - continue - current = float(col.iloc[-1]) - prev = float(col.iloc[0]) if len(col) > 1 else current - change = ((current - prev) / prev * 100) if prev else 0 - result[sym] = { - "price": round(current, 2), - "change_percent": round(change, 2), - "up": bool(change >= 0), - } - except Exception as e: - logger.warning(f"Could not parse {sym}: {e}") - return result - except Exception as e: - logger.warning(f"Batch fetch failed: {e}") - return {} - +TICKERS_DEFENSE = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"] +TICKERS_TECH = ["NVDA", "AMD", "TSM", "INTC", "GOOGL", "AMZN", "MSFT", "AAPL", "TSLA", "META", "NFLX", "SMCI", "ARM", "ASML"] +TICKERS_CRYPTO = [ + ("BTC", "BINANCE:BTCUSDT", "BTC-USD"), + ("ETH", "BINANCE:ETHUSDT", "ETH-USD"), + ("SOL", "BINANCE:SOLUSDT", "SOL-USD"), + ("XRP", "BINANCE:XRPUSDT", "XRP-USD"), + ("ADA", "BINANCE:ADAUSDT", "ADA-USD"), +] -_STOCK_TICKERS = ["RTX", "LMT", "NOC", "GD", "BA", "PLTR"] -_OIL_MAP = {"WTI Crude": "CL=F", "Brent Crude": "BZ=F"} -_ALL_TICKERS = _STOCK_TICKERS + list(_OIL_MAP.values()) +# Ticker priority for high-frequency updates (we update these every tick) +PRIORITY_SYMBOLS = ["BTC", "ETH", "NVDA", "PLTR"] -_MARKET_COOLDOWN_SECONDS = 1800 # fetch at most once every 30 minutes -_last_market_fetch: float = 0.0 +# Persistence for state between short-lived scheduler ticks +_last_fetch_results = {} +_last_fetch_time = 0.0 +_rotating_index = 0 +_executor = ThreadPoolExecutor(max_workers=10) -def _fetch_all_market_data(): - """Single yfinance download for all market tickers to avoid rate limiting.""" - raw = _batch_fetch(_ALL_TICKERS, period="5d") - stocks = {sym: raw[sym] for sym in _STOCK_TICKERS if sym in raw} - oil = {name: raw[sym] for name, sym in _OIL_MAP.items() if sym in raw} - return stocks, oil +def _fetch_finnhub_quote(symbol: str, api_key: str): + """Fetch from Finnhub. Returns (symbol, data) or (symbol, None).""" + url = f"https://finnhub.io/api/v1/quote?symbol={symbol}&token={api_key}" + try: + req = urllib.request.Request(url) + with urllib.request.urlopen(req, timeout=5) as response: + data = json.loads(response.read().decode()) + if "c" not in data or data["c"] == 0: + return symbol, None + current = float(data["c"]) + change_p = float(data.get("dp", 0.0) or 0.0) + return symbol, { + "price": round(current, 2), + "change_percent": round(change_p, 2), + "up": bool(change_p >= 0), + } + except Exception as e: + logger.debug(f"Finnhub error for {symbol}: {e}") + return symbol, None -@with_retry(max_retries=2, base_delay=10) -def fetch_defense_stocks(): - global _last_market_fetch - import time - if time.time() - _last_market_fetch < _MARKET_COOLDOWN_SECONDS: - return +def _fetch_yfinance_single(symbol: str, period: str = "2d"): + """Fetch from yfinance. Returns (symbol, data) or (symbol, None).""" try: - stocks, oil = _fetch_all_market_data() - if stocks: - _last_market_fetch = time.time() - with _data_lock: - latest_data['stocks'] = stocks - if oil: - latest_data['oil'] = oil - _mark_fresh("stocks") - if oil: - _mark_fresh("oil") - logger.info(f"Markets: {len(stocks)} stocks, {len(oil)} oil tickers") - else: - logger.warning("Markets: empty result from yfinance (rate limited?)") + import yfinance as yf + ticker = yf.Ticker(symbol) + hist = ticker.history(period=period) + if len(hist) >= 1: + current_price = hist["Close"].iloc[-1] + prev_close = hist["Close"].iloc[0] if len(hist) > 1 else current_price + change_percent = ((current_price - prev_close) / prev_close) * 100 if prev_close else 0 + current_price_f = float(current_price) + change_percent_f = float(change_percent) + if not math.isfinite(current_price_f) or not math.isfinite(change_percent_f): + return symbol, None + return symbol, { + "price": round(current_price_f, 2), + "change_percent": round(change_percent_f, 2), + "up": bool(change_percent_f >= 0), + } except Exception as e: - logger.error(f"Error fetching market data: {e}") + logger.debug(f"Yfinance error for {symbol}: {e}") + return symbol, None + + +@with_retry(max_retries=1, base_delay=1) +def fetch_financial_markets(): + """Fetches full market list with smart throttling (3s for Finnhub, 60s for yfinance).""" + global _last_fetch_time, _last_fetch_results, _rotating_index + + finnhub_key = os.getenv("FINNHUB_API_KEY", "").strip() + use_finnhub = bool(finnhub_key) + + now = time.time() + # Throttle logic: 3s for Finnhub, 60s for yfinance fallback + throttle_s = 3.0 if use_finnhub else 60.0 + + if now - _last_fetch_time < throttle_s and _last_fetch_results: + return # Skip if too frequent + _last_fetch_time = now + + # Prepare symbol lists + all_crypto = {label: (f_sym, y_sym) for label, f_sym, y_sym in TICKERS_CRYPTO} + all_stocks = TICKERS_TECH + TICKERS_DEFENSE + + subset_to_fetch = [] + + if use_finnhub: + # Finnhub Free Limit: 60/min. + # Ticking every 3s = 20 ticks/min. + # To stay safe, we fetch only ~3 items per tick. + # Priority items (BTC, ETH) + 1 rotating item. + subset_to_fetch = ["BINANCE:BTCUSDT", "BINANCE:ETHUSDT"] + + # Determine rotating ticker + all_other_symbols = [] + for sym in all_stocks: + all_other_symbols.append(sym) + for label, (f_sym, y_sym) in all_crypto.items(): + if label not in ["BTC", "ETH"]: + all_other_symbols.append(f_sym) + + if all_other_symbols: + rotated = all_other_symbols[_rotating_index % len(all_other_symbols)] + subset_to_fetch.append(rotated) + _rotating_index += 1 + + # Concurrently fetch + futures = [_executor.submit(_fetch_finnhub_quote, s, finnhub_key) for s in subset_to_fetch] + for f in futures: + sym, data = f.result() + if data: + # Map back to readable label if it was crypto + label = sym + for l, (fs, ys) in all_crypto.items(): + if fs == sym: + label = l + break + _last_fetch_results[label] = data + else: + # Yahoo Finance Fallback - fetch all (once per minute) + logger.info("Finnhub key missing, using Yahoo Finance 60s update cycle.") + to_fetch = all_stocks + [y_sym for l, (fs, y_sym) in all_crypto.items()] + futures = [_executor.submit(_fetch_yfinance_single, s) for s in to_fetch] + for f in futures: + sym, data = f.result() + if data: + # Map back to readable label if it was crypto + label = sym + for l, (fs, ys) in all_crypto.items(): + if ys == sym: + label = l + break + _last_fetch_results[label] = data -@with_retry(max_retries=1, base_delay=10) -def fetch_oil_prices(): - # Oil is now fetched together with stocks in fetch_defense_stocks to use a single request. - # This function is kept for scheduler compatibility but is a no-op if stocks already ran. + if not _last_fetch_results: + return + with _data_lock: - if latest_data.get('oil'): - return # Already populated by fetch_defense_stocks - try: - _, oil = _fetch_all_market_data() - if oil: - with _data_lock: - latest_data['oil'] = oil - _mark_fresh("oil") - except Exception as e: - logger.error(f"Error fetching oil: {e}") + latest_data["stocks"] = dict(_last_fetch_results) + latest_data["financial_source"] = "finnhub" if use_finnhub else "yfinance" + _mark_fresh("stocks") diff --git a/backend/services/fetchers/flights.py b/backend/services/fetchers/flights.py index e41646fd..bd6a72db 100644 --- a/backend/services/fetchers/flights.py +++ b/backend/services/fetchers/flights.py @@ -1,5 +1,7 @@ """Commercial flight fetching — ADS-B, OpenSky, supplemental sources, routes, trail accumulation, GPS jamming detection, and holding pattern detection.""" + +import copy import re import os import time @@ -8,19 +10,23 @@ import logging import threading import concurrent.futures +import random import requests from datetime import datetime from cachetools import TTLCache from services.network_utils import fetch_with_curl from services.fetchers._store import latest_data, _data_lock, _mark_fresh from services.fetchers.plane_alert import enrich_with_plane_alert, enrich_with_tracked_names +from services.fetchers.emissions import get_emissions_info from services.fetchers.retry import with_retry +from services.constants import GPS_JAMMING_NACP_THRESHOLD, GPS_JAMMING_MIN_RATIO, GPS_JAMMING_MIN_AIRCRAFT logger = logging.getLogger("services.data_fetcher") # Pre-compiled regex patterns for airline code extraction (used in hot loop) -_RE_AIRLINE_CODE_1 = re.compile(r'^([A-Z]{3})\d') -_RE_AIRLINE_CODE_2 = re.compile(r'^([A-Z]{3})[A-Z\d]') +_RE_AIRLINE_CODE_1 = re.compile(r"^([A-Z]{3})\d") +_RE_AIRLINE_CODE_2 = re.compile(r"^([A-Z]{3})[A-Z\d]") + # --------------------------------------------------------------------------- # OpenSky Network API Client (OAuth2) @@ -39,7 +45,7 @@ def get_token(self): data = { "grant_type": "client_credentials", "client_id": self.client_id, - "client_secret": self.client_secret + "client_secret": self.client_secret, } try: r = requests.post(url, data=data, timeout=10) @@ -51,13 +57,20 @@ def get_token(self): return self.token else: logger.error(f"OpenSky Auth Failed: {r.status_code} {r.text}") - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + ) as e: logger.error(f"OpenSky Auth Exception: {e}") return None + opensky_client = OpenSkyClient( client_id=os.environ.get("OPENSKY_CLIENT_ID", ""), - client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", "") + client_secret=os.environ.get("OPENSKY_CLIENT_SECRET", ""), ) # Throttling and caching for OpenSky (400 req/day limit) @@ -68,46 +81,173 @@ def get_token(self): # Supplemental ADS-B sources for blind-spot gap-filling # --------------------------------------------------------------------------- _BLIND_SPOT_REGIONS = [ - {"name": "Yekaterinburg", "lat": 56.8, "lon": 60.6, "radius_nm": 250}, - {"name": "Novosibirsk", "lat": 55.0, "lon": 82.9, "radius_nm": 250}, - {"name": "Krasnoyarsk", "lat": 56.0, "lon": 92.9, "radius_nm": 250}, - {"name": "Vladivostok", "lat": 43.1, "lon": 131.9, "radius_nm": 250}, - {"name": "Urumqi", "lat": 43.8, "lon": 87.6, "radius_nm": 250}, - {"name": "Chengdu", "lat": 30.6, "lon": 104.1, "radius_nm": 250}, - {"name": "Lagos-Accra", "lat": 6.5, "lon": 3.4, "radius_nm": 250}, - {"name": "Addis Ababa", "lat": 9.0, "lon": 38.7, "radius_nm": 250}, + {"name": "Yekaterinburg", "lat": 56.8, "lon": 60.6, "radius_nm": 250}, + {"name": "Novosibirsk", "lat": 55.0, "lon": 82.9, "radius_nm": 250}, + {"name": "Krasnoyarsk", "lat": 56.0, "lon": 92.9, "radius_nm": 250}, + {"name": "Vladivostok", "lat": 43.1, "lon": 131.9, "radius_nm": 250}, + {"name": "Urumqi", "lat": 43.8, "lon": 87.6, "radius_nm": 250}, + {"name": "Chengdu", "lat": 30.6, "lon": 104.1, "radius_nm": 250}, + {"name": "Lagos-Accra", "lat": 6.5, "lon": 3.4, "radius_nm": 250}, + {"name": "Addis Ababa", "lat": 9.0, "lon": 38.7, "radius_nm": 250}, ] -_SUPPLEMENTAL_FETCH_INTERVAL = 120 +# The blind-spot supplement previously burst several airplanes.live point +# queries in parallel and triggered repeated 429s in real startup logs, so we +# keep it on a long cache interval and pace each regional point query serially. +_SUPPLEMENTAL_FETCH_INTERVAL = 1800 +_AIRPLANES_LIVE_DELAY_SECONDS = 1.2 +_AIRPLANES_LIVE_DELAY_JITTER_SECONDS = 0.4 last_supplemental_fetch = 0 cached_supplemental_flights = [] # Helicopter type codes (backend classification) _HELI_TYPES_BACKEND = { - "R22", "R44", "R66", "B06", "B06T", "B204", "B205", "B206", "B212", "B222", "B230", - "B407", "B412", "B427", "B429", "B430", "B505", "B525", - "AS32", "AS35", "AS50", "AS55", "AS65", - "EC20", "EC25", "EC30", "EC35", "EC45", "EC55", "EC75", - "H125", "H130", "H135", "H145", "H155", "H160", "H175", "H215", "H225", - "S55", "S58", "S61", "S64", "S70", "S76", "S92", - "A109", "A119", "A139", "A169", "A189", "AW09", - "MD52", "MD60", "MDHI", "MD90", "NOTR", - "B47G", "HUEY", "GAMA", "CABR", "EXE", + "R22", + "R44", + "R66", + "B06", + "B06T", + "B204", + "B205", + "B206", + "B212", + "B222", + "B230", + "B407", + "B412", + "B427", + "B429", + "B430", + "B505", + "B525", + "AS32", + "AS35", + "AS50", + "AS55", + "AS65", + "EC20", + "EC25", + "EC30", + "EC35", + "EC45", + "EC55", + "EC75", + "H125", + "H130", + "H135", + "H145", + "H155", + "H160", + "H175", + "H215", + "H225", + "S55", + "S58", + "S61", + "S64", + "S70", + "S76", + "S92", + "A109", + "A119", + "A139", + "A169", + "A189", + "AW09", + "MD52", + "MD60", + "MDHI", + "MD90", + "NOTR", + "B47G", + "HUEY", + "GAMA", + "CABR", + "EXE", } # Private jet ICAO type designator codes PRIVATE_JET_TYPES = { - "G150", "G200", "G280", "GLEX", "G500", "G550", "G600", "G650", "G700", - "GLF2", "GLF3", "GLF4", "GLF5", "GLF6", "GL5T", "GL7T", "GV", "GIV", - "CL30", "CL35", "CL60", "BD70", "BD10", "GL5T", "GL7T", - "CRJ1", "CRJ2", - "C25A", "C25B", "C25C", "C500", "C501", "C510", "C525", "C526", - "C550", "C560", "C56X", "C680", "C68A", "C700", "C750", - "FA10", "FA20", "FA50", "FA7X", "FA8X", "F900", "F2TH", "ASTR", - "E35L", "E545", "E550", "E55P", "LEGA", "PH10", "PH30", - "LJ23", "LJ24", "LJ25", "LJ28", "LJ31", "LJ35", "LJ36", - "LJ40", "LJ45", "LJ55", "LJ60", "LJ70", "LJ75", - "H25A", "H25B", "H25C", "HA4T", "BE40", "PRM1", - "HDJT", "PC24", "EA50", "SF50", "GALX", + "G150", + "G200", + "G280", + "GLEX", + "G500", + "G550", + "G600", + "G650", + "G700", + "GLF2", + "GLF3", + "GLF4", + "GLF5", + "GLF6", + "GL5T", + "GL7T", + "GV", + "GIV", + "CL30", + "CL35", + "CL60", + "BD70", + "BD10", + "GL5T", + "GL7T", + "CRJ1", + "CRJ2", + "C25A", + "C25B", + "C25C", + "C500", + "C501", + "C510", + "C525", + "C526", + "C550", + "C560", + "C56X", + "C680", + "C68A", + "C700", + "C750", + "FA10", + "FA20", + "FA50", + "FA7X", + "FA8X", + "F900", + "F2TH", + "ASTR", + "E35L", + "E545", + "E550", + "E55P", + "LEGA", + "PH10", + "PH30", + "LJ23", + "LJ24", + "LJ25", + "LJ28", + "LJ31", + "LJ35", + "LJ36", + "LJ40", + "LJ45", + "LJ55", + "LJ60", + "LJ70", + "LJ75", + "H25A", + "H25B", + "H25C", + "HA4T", + "BE40", + "PRM1", + "HDJT", + "PC24", + "EA50", + "SF50", + "GALX", } # Flight trails state @@ -127,35 +267,59 @@ def _fetch_supplemental_sources(seen_hex: set) -> list: now = time.time() if now - last_supplemental_fetch < _SUPPLEMENTAL_FETCH_INTERVAL: - return [f for f in cached_supplemental_flights - if f.get("hex", "").lower().strip() not in seen_hex] + return [ + f + for f in cached_supplemental_flights + if f.get("hex", "").lower().strip() not in seen_hex + ] new_supplemental = [] supplemental_hex = set() def _fetch_airplaneslive(region): try: - url = (f"https://api.airplanes.live/v2/point/" - f"{region['lat']}/{region['lon']}/{region['radius_nm']}") + url = ( + f"https://api.airplanes.live/v2/point/" + f"{region['lat']}/{region['lon']}/{region['radius_nm']}" + ) res = fetch_with_curl(url, timeout=10) if res.status_code == 200: data = res.json() return data.get("ac", []) - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.debug(f"airplanes.live {region['name']} failed: {e}") return [] try: - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - results = list(pool.map(_fetch_airplaneslive, _BLIND_SPOT_REGIONS)) - for region_flights in results: + for idx, region in enumerate(_BLIND_SPOT_REGIONS): + region_flights = _fetch_airplaneslive(region) for f in region_flights: h = f.get("hex", "").lower().strip() if h and h not in seen_hex and h not in supplemental_hex: f["supplemental_source"] = "airplanes.live" new_supplemental.append(f) supplemental_hex.add(h) - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: + if idx < len(_BLIND_SPOT_REGIONS) - 1: + time.sleep( + _AIRPLANES_LIVE_DELAY_SECONDS + + random.uniform(0.0, _AIRPLANES_LIVE_DELAY_JITTER_SECONDS) + ) + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + OSError, + ) as e: logger.warning(f"airplanes.live supplemental fetch failed: {e}") ap_count = len(new_supplemental) @@ -163,8 +327,10 @@ def _fetch_airplaneslive(region): try: for region in _BLIND_SPOT_REGIONS: try: - url = (f"https://opendata.adsb.fi/api/v3/lat/" - f"{region['lat']}/lon/{region['lon']}/dist/{region['radius_nm']}") + url = ( + f"https://opendata.adsb.fi/api/v3/lat/" + f"{region['lat']}/lon/{region['lon']}/dist/{region['radius_nm']}" + ) res = fetch_with_curl(url, timeout=10) if res.status_code == 200: data = res.json() @@ -174,10 +340,25 @@ def _fetch_airplaneslive(region): f["supplemental_source"] = "adsb.fi" new_supplemental.append(f) supplemental_hex.add(h) - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.debug(f"adsb.fi {region['name']} failed: {e}") time.sleep(1.1) - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + OSError, + ) as e: logger.warning(f"adsb.fi supplemental fetch failed: {e}") fi_count = len(new_supplemental) - ap_count @@ -187,8 +368,10 @@ def _fetch_airplaneslive(region): if new_supplemental: _mark_fresh("supplemental_flights") - logger.info(f"Supplemental: +{len(new_supplemental)} new aircraft from blind-spot " - f"hotspots (airplanes.live: {ap_count}, adsb.fi: {fi_count})") + logger.info( + f"Supplemental: +{len(new_supplemental)} new aircraft from blind-spot " + f"hotspots (airplanes.live: {ap_count}, adsb.fi: {fi_count})" + ) return new_supplemental @@ -204,18 +387,24 @@ def fetch_routes_background(sampled): for f in sampled: c_sign = str(f.get("flight", "")).strip() if c_sign and c_sign != "UNKNOWN": - callsigns_to_query.append({ - "callsign": c_sign, - "lat": f.get("lat", 0), - "lng": f.get("lon", 0) - }) + callsigns_to_query.append( + {"callsign": c_sign, "lat": f.get("lat", 0), "lng": f.get("lon", 0)} + ) batch_size = 100 - batches = [callsigns_to_query[i:i+batch_size] for i in range(0, len(callsigns_to_query), batch_size)] + batches = [ + callsigns_to_query[i : i + batch_size] + for i in range(0, len(callsigns_to_query), batch_size) + ] for batch in batches: try: - r = fetch_with_curl("https://api.adsb.lol/api/0/routeset", method="POST", json_data={"planes": batch}, timeout=15) + r = fetch_with_curl( + "https://api.adsb.lol/api/0/routeset", + method="POST", + json_data={"planes": batch}, + timeout=15, + ) if r.status_code == 200: route_data = r.json() route_list = [] @@ -238,7 +427,15 @@ def fetch_routes_background(sampled): "dest_loc": [dest_apt.get("lon", 0), dest_apt.get("lat", 0)], } time.sleep(0.25) - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.debug(f"Route batch request failed: {e}") finally: with _routes_lock: @@ -259,7 +456,9 @@ def _classify_and_publish(all_adsb_flights): with _routes_lock: already_running = routes_fetch_in_progress if not already_running: - threading.Thread(target=fetch_routes_background, args=(all_adsb_flights,), daemon=True).start() + threading.Thread( + target=fetch_routes_background, args=(all_adsb_flights,), daemon=True + ).start() for f in all_adsb_flights: try: @@ -308,27 +507,29 @@ def _classify_and_publish(all_adsb_flights): ac_category = "heli" if model_upper in _HELI_TYPES_BACKEND else "plane" - flights.append({ - "callsign": flight_str, - "country": f.get("r", "N/A"), - "lng": float(lng), - "lat": float(lat), - "alt": alt_value, - "heading": heading, - "type": "flight", - "origin_loc": origin_loc, - "dest_loc": dest_loc, - "origin_name": origin_name, - "dest_name": dest_name, - "registration": f.get("r", "N/A"), - "model": f.get("t", "Unknown"), - "icao24": f.get("hex", ""), - "speed_knots": speed_knots, - "squawk": f.get("squawk", ""), - "airline_code": airline_code, - "aircraft_category": ac_category, - "nac_p": f.get("nac_p") - }) + flights.append( + { + "callsign": flight_str, + "country": f.get("r", "N/A"), + "lng": float(lng), + "lat": float(lat), + "alt": alt_value, + "heading": heading, + "type": "flight", + "origin_loc": origin_loc, + "dest_loc": dest_loc, + "origin_name": origin_name, + "dest_name": dest_name, + "registration": f.get("r", "N/A"), + "model": f.get("t", "Unknown"), + "icao24": f.get("hex", ""), + "speed_knots": speed_knots, + "squawk": f.get("squawk", ""), + "airline_code": airline_code, + "aircraft_category": ac_category, + "nac_p": f.get("nac_p"), + } + ) except (ValueError, TypeError, KeyError, AttributeError) as loop_e: logger.error(f"Flight interpolation error: {loop_e}") continue @@ -342,80 +543,97 @@ def _classify_and_publish(all_adsb_flights): for f in flights: enrich_with_plane_alert(f) enrich_with_tracked_names(f) - - callsign = f.get('callsign', '').strip().upper() - is_commercial_format = bool(re.match(r'^[A-Z]{3}\d{1,4}[A-Z]{0,2}$', callsign)) - - if f.get('alert_category'): - f['type'] = 'tracked_flight' + # Attach fuel-burn / CO2 emissions estimate when model is known + model = f.get("model") + if model: + emi = get_emissions_info(model) + if emi: + f["emissions"] = emi + + callsign = f.get("callsign", "").strip().upper() + is_commercial_format = bool(re.match(r"^[A-Z]{3}\d{1,4}[A-Z]{0,2}$", callsign)) + + if f.get("alert_category"): + f["type"] = "tracked_flight" tracked.append(f) - elif f.get('airline_code') or is_commercial_format: - f['type'] = 'commercial_flight' + elif f.get("airline_code") or is_commercial_format: + f["type"] = "commercial_flight" commercial.append(f) - elif f.get('model', '').upper() in PRIVATE_JET_TYPES: - f['type'] = 'private_jet' + elif f.get("model", "").upper() in PRIVATE_JET_TYPES: + f["type"] = "private_jet" private_jets.append(f) else: - f['type'] = 'private_ga' + f["type"] = "private_ga" private_ga.append(f) # --- Smart merge: protect against partial API failures --- - prev_commercial_count = len(latest_data.get('commercial_flights', [])) - prev_total = prev_commercial_count + len(latest_data.get('private_jets', [])) + len(latest_data.get('private_flights', [])) + with _data_lock: + prev_commercial_count = len(latest_data.get("commercial_flights", [])) + prev_private_jets_count = len(latest_data.get("private_jets", [])) + prev_private_flights_count = len(latest_data.get("private_flights", [])) + prev_total = prev_commercial_count + prev_private_jets_count + prev_private_flights_count new_total = len(commercial) + len(private_jets) + len(private_ga) if new_total == 0: logger.warning("No civilian flights found! Skipping overwrite to prevent clearing the map.") elif prev_total > 100 and new_total < prev_total * 0.5: - logger.warning(f"Flight count dropped from {prev_total} to {new_total} (>50% loss). Keeping previous data to prevent flicker.") + logger.warning( + f"Flight count dropped from {prev_total} to {new_total} (>50% loss). Keeping previous data to prevent flicker." + ) else: _now = time.time() def _merge_category(new_list, old_list, max_stale_s=120): by_icao = {} for f in old_list: - icao = f.get('icao24', '') + icao = f.get("icao24", "") if icao: - f.setdefault('_seen_at', _now) - if (_now - f.get('_seen_at', _now)) < max_stale_s: + f.setdefault("_seen_at", _now) + if (_now - f.get("_seen_at", _now)) < max_stale_s: by_icao[icao] = f for f in new_list: - icao = f.get('icao24', '') + icao = f.get("icao24", "") if icao: - f['_seen_at'] = _now + f["_seen_at"] = _now by_icao[icao] = f else: continue return list(by_icao.values()) with _data_lock: - latest_data['commercial_flights'] = _merge_category(commercial, latest_data.get('commercial_flights', [])) - latest_data['private_jets'] = _merge_category(private_jets, latest_data.get('private_jets', [])) - latest_data['private_flights'] = _merge_category(private_ga, latest_data.get('private_flights', [])) + latest_data["commercial_flights"] = _merge_category( + commercial, latest_data.get("commercial_flights", []) + ) + latest_data["private_jets"] = _merge_category( + private_jets, latest_data.get("private_jets", []) + ) + latest_data["private_flights"] = _merge_category( + private_ga, latest_data.get("private_flights", []) + ) _mark_fresh("commercial_flights", "private_jets", "private_flights") with _data_lock: if flights: - latest_data['flights'] = flights + latest_data["flights"] = flights # Merge tracked civilian flights with tracked military flights with _data_lock: - existing_tracked = list(latest_data.get('tracked_flights', [])) + existing_tracked = copy.deepcopy(latest_data.get("tracked_flights", [])) fresh_tracked_map = {} for t in tracked: - icao = t.get('icao24', '').upper() + icao = t.get("icao24", "").upper() if icao: fresh_tracked_map[icao] = t merged_tracked = [] seen_icaos = set() for old_t in existing_tracked: - icao = old_t.get('icao24', '').upper() + icao = old_t.get("icao24", "").upper() if icao in fresh_tracked_map: fresh = fresh_tracked_map[icao] - for key in ('alert_category', 'alert_operator', 'alert_special', 'alert_flag'): + for key in ("alert_category", "alert_operator", "alert_special", "alert_flag"): if key in old_t and key not in fresh: fresh[key] = old_t[key] merged_tracked.append(fresh) @@ -429,36 +647,47 @@ def _merge_category(new_list, old_list, max_stale_s=120): merged_tracked.append(t) with _data_lock: - latest_data['tracked_flights'] = merged_tracked - logger.info(f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)") + latest_data["tracked_flights"] = merged_tracked + logger.info( + f"Tracked flights: {len(merged_tracked)} total ({len(fresh_tracked_map)} fresh from civilian)" + ) # --- Trail Accumulation --- def _accumulate_trail(f, now_ts, check_route=True): - hex_id = f.get('icao24', '').lower() + hex_id = f.get("icao24", "").lower() if not hex_id: return 0, None - if check_route and f.get('origin_name', 'UNKNOWN') != 'UNKNOWN': - f['trail'] = [] + if check_route and f.get("origin_name", "UNKNOWN") != "UNKNOWN": + f["trail"] = [] return 0, hex_id - lat, lng, alt = f.get('lat'), f.get('lng'), f.get('alt', 0) + lat, lng, alt = f.get("lat"), f.get("lng"), f.get("alt", 0) if lat is None or lng is None: - f['trail'] = flight_trails.get(hex_id, {}).get('points', []) + f["trail"] = flight_trails.get(hex_id, {}).get("points", []) return 0, hex_id point = [round(lat, 5), round(lng, 5), round(alt, 1), round(now_ts)] if hex_id not in flight_trails: - flight_trails[hex_id] = {'points': [], 'last_seen': now_ts} + flight_trails[hex_id] = {"points": [], "last_seen": now_ts} trail_data = flight_trails[hex_id] - if trail_data['points'] and trail_data['points'][-1][0] == point[0] and trail_data['points'][-1][1] == point[1]: - trail_data['last_seen'] = now_ts + if ( + trail_data["points"] + and trail_data["points"][-1][0] == point[0] + and trail_data["points"][-1][1] == point[1] + ): + trail_data["last_seen"] = now_ts else: - trail_data['points'].append(point) - trail_data['last_seen'] = now_ts - if len(trail_data['points']) > 200: - trail_data['points'] = trail_data['points'][-200:] - f['trail'] = trail_data['points'] + trail_data["points"].append(point) + trail_data["last_seen"] = now_ts + if len(trail_data["points"]) > 200: + trail_data["points"] = trail_data["points"][-200:] + f["trail"] = trail_data["points"] return 1, hex_id now_ts = datetime.utcnow().timestamp() + with _data_lock: + military_snapshot = copy.deepcopy(latest_data.get("military_flights", [])) + tracked_snapshot = copy.deepcopy(latest_data.get("tracked_flights", [])) + raw_flights_snapshot = list(latest_data.get("flights", [])) + all_lists = [commercial, private_jets, private_ga, existing_tracked] seen_hexes = set() trail_count = 0 @@ -470,97 +699,121 @@ def _accumulate_trail(f, now_ts, check_route=True): if hex_id: seen_hexes.add(hex_id) - for mf in latest_data.get('military_flights', []): + for mf in military_snapshot: count, hex_id = _accumulate_trail(mf, now_ts, check_route=False) trail_count += count if hex_id: seen_hexes.add(hex_id) - tracked_hexes = {t.get('icao24', '').lower() for t in latest_data.get('tracked_flights', [])} + tracked_hexes = {t.get("icao24", "").lower() for t in tracked_snapshot} stale_keys = [] for k, v in flight_trails.items(): cutoff = now_ts - 1800 if k in tracked_hexes else now_ts - 300 - if v['last_seen'] < cutoff: + if v["last_seen"] < cutoff: stale_keys.append(k) for k in stale_keys: del flight_trails[k] if len(flight_trails) > _MAX_TRACKED_TRAILS: - sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]['last_seen']) + sorted_keys = sorted(flight_trails.keys(), key=lambda k: flight_trails[k]["last_seen"]) evict_count = len(flight_trails) - _MAX_TRACKED_TRAILS for k in sorted_keys[:evict_count]: del flight_trails[k] - logger.info(f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total") + logger.info( + f"Trail accumulation: {trail_count} active trails, {len(stale_keys)} pruned, {len(flight_trails)} total" + ) # --- GPS Jamming Detection --- + # Uses NACp (Navigation Accuracy Category – Position) from ADS-B to infer + # GPS interference zones, similar to GPSJam.org / Flightradar24. + # NACp < 8 = position accuracy worse than the FAA-mandated 0.05 NM. + # + # Denoising (to suppress false positives from old GA transponders): + # 1. Skip nac_p == 0 ("unknown accuracy") — old transponders that never + # computed accuracy, NOT evidence of jamming. Real jamming shows 1-7. + # 2. Require minimum aircraft per grid cell for statistical validity. + # 3. Subtract 1 from degraded count per cell (GPSJam's technique) so a + # single quirky transponder can't flag an entire zone. + # 4. Require the adjusted ratio to exceed the threshold. try: jamming_grid = {} - raw_flights = latest_data.get('flights', []) + raw_flights = raw_flights_snapshot for rf in raw_flights: - rlat = rf.get('lat') - rlng = rf.get('lng') or rf.get('lon') + rlat = rf.get("lat") + rlng = rf.get("lng") or rf.get("lon") if rlat is None or rlng is None: continue - nacp = rf.get('nac_p') - if nacp is None: + nacp = rf.get("nac_p") + if nacp is None or nacp == 0: continue grid_key = f"{int(rlat)},{int(rlng)}" if grid_key not in jamming_grid: jamming_grid[grid_key] = {"degraded": 0, "total": 0} jamming_grid[grid_key]["total"] += 1 - if nacp < 8: + if nacp < GPS_JAMMING_NACP_THRESHOLD: jamming_grid[grid_key]["degraded"] += 1 jamming_zones = [] for gk, counts in jamming_grid.items(): - if counts["total"] < 3: + if counts["total"] < GPS_JAMMING_MIN_AIRCRAFT: + continue + adjusted_degraded = max(counts["degraded"] - 1, 0) + if adjusted_degraded == 0: continue - ratio = counts["degraded"] / counts["total"] - if ratio > 0.25: + ratio = adjusted_degraded / counts["total"] + if ratio > GPS_JAMMING_MIN_RATIO: lat_i, lng_i = gk.split(",") severity = "low" if ratio < 0.5 else "medium" if ratio < 0.75 else "high" - jamming_zones.append({ - "lat": int(lat_i) + 0.5, - "lng": int(lng_i) + 0.5, - "severity": severity, - "ratio": round(ratio, 2), - "degraded": counts["degraded"], - "total": counts["total"] - }) + jamming_zones.append( + { + "lat": int(lat_i) + 0.5, + "lng": int(lng_i) + 0.5, + "severity": severity, + "ratio": round(ratio, 2), + "degraded": counts["degraded"], + "total": counts["total"], + } + ) with _data_lock: - latest_data['gps_jamming'] = jamming_zones + latest_data["gps_jamming"] = jamming_zones if jamming_zones: logger.info(f"GPS Jamming: {len(jamming_zones)} interference zones detected") except (ValueError, TypeError, KeyError, ZeroDivisionError) as e: logger.error(f"GPS Jamming detection error: {e}") with _data_lock: - latest_data['gps_jamming'] = [] + latest_data["gps_jamming"] = [] # --- Holding Pattern Detection --- try: holding_count = 0 - all_flight_lists = [commercial, private_jets, private_ga, - latest_data.get('tracked_flights', []), - latest_data.get('military_flights', [])] + all_flight_lists = [ + commercial, + private_jets, + private_ga, + tracked_snapshot, + military_snapshot, + ] with _trails_lock: - trails_snapshot = {k: v.get('points', [])[:] for k, v in flight_trails.items()} + trails_snapshot = {k: v.get("points", [])[:] for k, v in flight_trails.items()} for flist in all_flight_lists: for f in flist: - hex_id = f.get('icao24', '').lower() + hex_id = f.get("icao24", "").lower() trail = trails_snapshot.get(hex_id, []) if len(trail) < 6: - f['holding'] = False + f["holding"] = False continue pts = trail[-8:] total_turn = 0.0 prev_bearing = 0.0 for i in range(1, len(pts)): - lat1, lng1 = math.radians(pts[i-1][0]), math.radians(pts[i-1][1]) + lat1, lng1 = math.radians(pts[i - 1][0]), math.radians(pts[i - 1][1]) lat2, lng2 = math.radians(pts[i][0]), math.radians(pts[i][1]) dlng = lng2 - lng1 x = math.sin(dlng) * math.cos(lat2) - y = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(lat2) * math.cos(dlng) + y = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos( + lat2 + ) * math.cos(dlng) bearing = math.degrees(math.atan2(x, y)) % 360 if i > 1: delta = abs(bearing - prev_bearing) @@ -568,8 +821,8 @@ def _accumulate_trail(f, now_ts, check_route=True): delta = 360 - delta total_turn += delta prev_bearing = bearing - f['holding'] = total_turn > 300 - if f['holding']: + f["holding"] = total_turn > 300 + if f["holding"]: holding_count += 1 if holding_count: logger.info(f"Holding patterns: {holding_count} aircraft circling") @@ -577,7 +830,7 @@ def _accumulate_trail(f, now_ts, check_route=True): logger.error(f"Holding pattern detection error: {e}") with _data_lock: - latest_data['last_updated'] = datetime.utcnow().isoformat() + latest_data["last_updated"] = datetime.utcnow().isoformat() def _fetch_adsb_lol_regions(): @@ -588,7 +841,7 @@ def _fetch_adsb_lol_regions(): {"lat": 35.0, "lon": 105.0, "dist": 2000}, {"lat": -25.0, "lon": 133.0, "dist": 2000}, {"lat": 0.0, "lon": 20.0, "dist": 2500}, - {"lat": -15.0, "lon": -60.0, "dist": 2000} + {"lat": -15.0, "lon": -60.0, "dist": 2000}, ] def _fetch_region(r): @@ -598,7 +851,15 @@ def _fetch_region(r): if res.status_code == 200: data = res.json() return data.get("ac", []) - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.warning(f"Region fetch failed for lat={r['lat']}: {e}") return [] @@ -632,9 +893,18 @@ def _enrich_with_opensky_and_supplemental(adsb_flights): token = opensky_client.get_token() if token: opensky_regions = [ - {"name": "Africa", "bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0}}, - {"name": "Asia", "bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0}}, - {"name": "South America", "bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0}} + { + "name": "Africa", + "bbox": {"lamin": -35.0, "lomin": -20.0, "lamax": 38.0, "lomax": 55.0}, + }, + { + "name": "Asia", + "bbox": {"lamin": 0.0, "lomin": 30.0, "lamax": 75.0, "lomax": 150.0}, + }, + { + "name": "South America", + "bbox": {"lamin": -60.0, "lomin": -95.0, "lamax": 15.0, "lomax": -30.0}, + }, ] new_opensky_flights = [] @@ -648,24 +918,38 @@ def _enrich_with_opensky_and_supplemental(adsb_flights): if os_res.status_code == 200: os_data = os_res.json() states = os_data.get("states") or [] - logger.info(f"OpenSky: Fetched {len(states)} states for {os_reg['name']}") + logger.info( + f"OpenSky: Fetched {len(states)} states for {os_reg['name']}" + ) for s in states: - new_opensky_flights.append({ - "hex": s[0], - "flight": s[1].strip() if s[1] else "UNKNOWN", - "r": s[2], - "lon": s[5], - "lat": s[6], - "alt_baro": (s[7] * 3.28084) if s[7] else 0, - "track": s[10] or 0, - "gs": (s[9] * 1.94384) if s[9] else 0, - "t": "Unknown", - "is_opensky": True - }) + new_opensky_flights.append( + { + "hex": s[0], + "flight": s[1].strip() if s[1] else "UNKNOWN", + "r": s[2], + "lon": s[5], + "lat": s[6], + "alt_baro": (s[7] * 3.28084) if s[7] else 0, + "track": s[10] or 0, + "gs": (s[9] * 1.94384) if s[9] else 0, + "t": "Unknown", + "is_opensky": True, + } + ) else: - logger.warning(f"OpenSky API {os_reg['name']} failed: {os_res.status_code}") - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as ex: + logger.warning( + f"OpenSky API {os_reg['name']} failed: {os_res.status_code}" + ) + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as ex: logger.error(f"OpenSky fetching error for {os_reg['name']}: {ex}") cached_opensky_flights = new_opensky_flights @@ -688,12 +972,21 @@ def _enrich_with_opensky_and_supplemental(adsb_flights): seen_hex.add(h) if gap_fill: logger.info(f"Gap-fill: added {len(gap_fill)} aircraft to pipeline") - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + OSError, + ) as e: logger.warning(f"Supplemental source fetch failed (non-fatal): {e}") # Re-publish with enriched data if len(all_flights) > len(adsb_flights): - logger.info(f"Enrichment: {len(all_flights) - len(adsb_flights)} additional aircraft from OpenSky + supplemental") + logger.info( + f"Enrichment: {len(all_flights) - len(adsb_flights)} additional aircraft from OpenSky + supplemental" + ) _classify_and_publish(all_flights) except Exception as e: logger.error(f"OpenSky/supplemental enrichment error: {e}") @@ -705,6 +998,10 @@ def fetch_flights(): Phase 1 (fast): Fetch adsb.lol → classify → publish immediately (~3-5s) Phase 2 (background): Merge OpenSky + supplemental → re-publish (~15-30s) """ + from services.fetchers._store import is_any_active + + if not is_any_active("flights", "private", "jets", "tracked", "gps_jamming"): + return try: # Phase 1: adsb.lol — fast, parallel, publish immediately adsb_flights = _fetch_adsb_lol_regions() diff --git a/backend/services/fetchers/geo.py b/backend/services/fetchers/geo.py index cd044ba4..5f35e168 100644 --- a/backend/services/fetchers/geo.py +++ b/backend/services/fetchers/geo.py @@ -1,7 +1,9 @@ -"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap.""" +"""Ship and geopolitics fetchers — AIS vessels, carriers, frontlines, GDELT, LiveUAmap, fishing.""" + import csv import io import math +import os import logging from services.network_utils import fetch_with_curl from services.fetchers._store import latest_data, _data_lock, _mark_fresh @@ -16,6 +18,12 @@ @with_retry(max_retries=1, base_delay=1) def fetch_ships(): """Fetch real-time AIS vessel data and combine with OSINT carrier positions.""" + from services.fetchers._store import is_any_active + + if not is_any_active( + "ships_military", "ships_cargo", "ships_civilian", "ships_passenger", "ships_tracked_yachts" + ): + return from services.ais_stream import get_ais_vessels from services.carrier_tracker import get_carrier_positions @@ -23,19 +31,20 @@ def fetch_ships(): try: carriers = get_carrier_positions() ships.extend(carriers) - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Carrier tracker error (non-fatal): {e}") carriers = [] try: ais_vessels = get_ais_vessels() ships.extend(ais_vessels) - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"AIS stream error (non-fatal): {e}") ais_vessels = [] # Enrich ships with yacht alert data (tracked superyachts) from services.fetchers.yacht_alert import enrich_with_yacht_alert + for ship in ships: enrich_with_yacht_alert(ship) @@ -46,7 +55,7 @@ def fetch_ships(): logger.info(f"Ships: {len(carriers)} carriers + {len(ais_vessels)} AIS vessels") with _data_lock: - latest_data['ships'] = ships + latest_data["ships"] = ships _mark_fresh("ships") @@ -62,16 +71,19 @@ def find_nearest_airport(lat, lng, max_distance_nm=200): return None best = None - best_dist = float('inf') + best_dist = float("inf") lat_r = math.radians(lat) lng_r = math.radians(lng) for apt in cached_airports: - apt_lat_r = math.radians(apt['lat']) - apt_lng_r = math.radians(apt['lng']) + apt_lat_r = math.radians(apt["lat"]) + apt_lng_r = math.radians(apt["lng"]) dlat = apt_lat_r - lat_r dlng = apt_lng_r - lng_r - a = math.sin(dlat / 2) ** 2 + math.cos(lat_r) * math.cos(apt_lat_r) * math.sin(dlng / 2) ** 2 + a = ( + math.sin(dlat / 2) ** 2 + + math.cos(lat_r) * math.cos(apt_lat_r) * math.sin(dlng / 2) ** 2 + ) c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) dist_nm = 3440.065 * c @@ -81,9 +93,11 @@ def find_nearest_airport(lat, lng, max_distance_nm=200): if best and best_dist <= max_distance_nm: return { - "iata": best['iata'], "name": best['name'], - "lat": best['lat'], "lng": best['lng'], - "distance_nm": round(best_dist, 1) + "iata": best["iata"], + "name": best["name"], + "lat": best["lat"], + "lng": best["lng"], + "distance_nm": round(best_dist, 1), } return None @@ -99,21 +113,23 @@ def fetch_airports(): f = io.StringIO(response.text) reader = csv.DictReader(f) for row in reader: - if row['type'] == 'large_airport' and row['iata_code']: - cached_airports.append({ - "id": row['ident'], - "name": row['name'], - "iata": row['iata_code'], - "lat": float(row['latitude_deg']), - "lng": float(row['longitude_deg']), - "type": "airport" - }) + if row["type"] == "large_airport" and row["iata_code"]: + cached_airports.append( + { + "id": row["ident"], + "name": row["name"], + "iata": row["iata_code"], + "lat": float(row["latitude_deg"]), + "lng": float(row["longitude_deg"]), + "type": "airport", + } + ) logger.info(f"Loaded {len(cached_airports)} large airports into cache.") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching airports: {e}") with _data_lock: - latest_data['airports'] = cached_airports + latest_data["airports"] = cached_airports # --------------------------------------------------------------------------- @@ -122,28 +138,38 @@ def fetch_airports(): @with_retry(max_retries=1, base_delay=2) def fetch_frontlines(): """Fetch Ukraine frontline data (fast — single GitHub API call).""" + from services.fetchers._store import is_any_active + + if not is_any_active("ukraine_frontline"): + return try: from services.geopolitics import fetch_ukraine_frontlines + frontlines = fetch_ukraine_frontlines() if frontlines: with _data_lock: - latest_data['frontlines'] = frontlines + latest_data["frontlines"] = frontlines _mark_fresh("frontlines") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching frontlines: {e}") @with_retry(max_retries=1, base_delay=3) def fetch_gdelt(): """Fetch GDELT global military incidents (slow — downloads 32 ZIP files).""" + from services.fetchers._store import is_any_active + + if not is_any_active("global_incidents"): + return try: from services.geopolitics import fetch_global_military_incidents + gdelt = fetch_global_military_incidents() if gdelt is not None: with _data_lock: - latest_data['gdelt'] = gdelt + latest_data["gdelt"] = gdelt _mark_fresh("gdelt") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Error fetching GDELT: {e}") @@ -154,13 +180,72 @@ def fetch_geopolitics(): def update_liveuamap(): + from services.fetchers._store import is_any_active + + if not is_any_active("global_incidents"): + return logger.info("Running scheduled Liveuamap scraper...") try: from services.liveuamap_scraper import fetch_liveuamap + res = fetch_liveuamap() if res: with _data_lock: - latest_data['liveuamap'] = res + latest_data["liveuamap"] = res _mark_fresh("liveuamap") - except Exception as e: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: logger.error(f"Liveuamap scraper error: {e}") + + +# --------------------------------------------------------------------------- +# Fishing Activity (Global Fishing Watch) +# --------------------------------------------------------------------------- +@with_retry(max_retries=1, base_delay=5) +def fetch_fishing_activity(): + """Fetch recent fishing events from Global Fishing Watch (~5 day lag).""" + from services.fetchers._store import is_any_active + + if not is_any_active("fishing_activity"): + return + token = os.environ.get("GFW_API_TOKEN", "") + if not token: + logger.debug("GFW_API_TOKEN not set, skipping fishing activity fetch") + return + events = [] + try: + url = ( + "https://gateway.api.globalfishingwatch.org/v3/events" + "?datasets[0]=public-global-fishing-events:latest" + "&limit=500&sort=start&sort-direction=DESC" + ) + headers = {"Authorization": f"Bearer {token}"} + response = fetch_with_curl(url, timeout=30, headers=headers) + if response.status_code == 200: + entries = response.json().get("entries", []) + for e in entries: + pos = e.get("position", {}) + lat = pos.get("lat") + lng = pos.get("lon") + if lat is None or lng is None: + continue + dur = e.get("event", {}).get("duration", 0) or 0 + events.append( + { + "id": e.get("id", ""), + "type": e.get("type", "fishing"), + "lat": lat, + "lng": lng, + "start": e.get("start", ""), + "end": e.get("end", ""), + "vessel_name": (e.get("vessel") or {}).get("name", "Unknown"), + "vessel_flag": (e.get("vessel") or {}).get("flag", ""), + "duration_hrs": round(dur / 3600, 1), + } + ) + logger.info(f"Fishing activity: {len(events)} events") + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: + logger.error(f"Error fetching fishing activity: {e}") + with _data_lock: + latest_data["fishing_activity"] = events + if events: + _mark_fresh("fishing_activity") diff --git a/backend/services/fetchers/infrastructure.py b/backend/services/fetchers/infrastructure.py index 15dce652..a25d3fe6 100644 --- a/backend/services/fetchers/infrastructure.py +++ b/backend/services/fetchers/infrastructure.py @@ -1,4 +1,5 @@ """Infrastructure fetchers — internet outages (IODA), data centers, CCTV, KiwiSDR.""" + import json import time import heapq @@ -25,6 +26,7 @@ def _geocode_region(region_name: str, country_name: str) -> tuple: return _region_geocode_cache[cache_key] try: import urllib.parse + query = urllib.parse.quote(f"{region_name}, {country_name}") url = f"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1" response = fetch_with_curl(url, timeout=8, headers={"User-Agent": "ShadowBroker-OSINT/1.0"}) @@ -35,7 +37,7 @@ def _geocode_region(region_name: str, country_name: str) -> tuple: lon = float(results[0]["lon"]) _region_geocode_cache[cache_key] = (lat, lon) return (lat, lon) - except Exception: + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError): pass _region_geocode_cache[cache_key] = None return None @@ -44,6 +46,10 @@ def _geocode_region(region_name: str, country_name: str) -> tuple: @with_retry(max_retries=1, base_delay=1) def fetch_internet_outages(): """Fetch regional internet outage alerts from IODA (Georgia Tech).""" + from services.fetchers._store import is_any_active + + if not is_any_active("internet_outages"): + return RELIABLE_DATASOURCES = {"bgp", "ping-slash24"} outages = [] try: @@ -96,7 +102,15 @@ def fetch_internet_outages(): geocoded.append(r) outages = heapq.nlargest(100, geocoded, key=lambda x: x["severity"]) logger.info(f"Internet outages: {len(outages)} regions affected") - except Exception as e: + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: logger.error(f"Error fetching internet outages: {e}") with _data_lock: latest_data["internet_outages"] = outages @@ -104,6 +118,116 @@ def fetch_internet_outages(): _mark_fresh("internet_outages") +# --------------------------------------------------------------------------- +# RIPE Atlas — complement IODA with probe-level disconnection data +# --------------------------------------------------------------------------- + +@with_retry(max_retries=1, base_delay=3) +def fetch_ripe_atlas_probes(): + """Fetch disconnected RIPE Atlas probes and merge into internet_outages (complementing IODA).""" + from services.fetchers._store import is_any_active + + if not is_any_active("internet_outages"): + return + try: + # 1. Fetch disconnected probes (status=2) — ~2,000 probes, no auth needed + url_disc = "https://atlas.ripe.net/api/v2/probes/?status=2&page_size=500&format=json" + resp_disc = fetch_with_curl(url_disc, timeout=20) + if resp_disc.status_code != 200: + logger.warning(f"RIPE Atlas probes API returned {resp_disc.status_code}") + return + disc_data = resp_disc.json() + disconnected = disc_data.get("results", []) + + # 2. Fetch connected probe count (page_size=1 — we only need the count) + url_conn = "https://atlas.ripe.net/api/v2/probes/?status=1&page_size=1&format=json" + resp_conn = fetch_with_curl(url_conn, timeout=10) + total_connected = 0 + if resp_conn.status_code == 200: + total_connected = resp_conn.json().get("count", 0) + + # 3. Group disconnected probes by country + country_disc: dict = {} + for p in disconnected: + cc = p.get("country_code", "") + if not cc: + continue + if cc not in country_disc: + country_disc[cc] = [] + country_disc[cc].append(p) + + # 4. Get IODA-covered countries to avoid double-reporting + with _data_lock: + ioda_outages = list(latest_data.get("internet_outages", [])) + ioda_countries = { + o.get("country_code", "").upper() + for o in ioda_outages + if o.get("datasource") != "ripe-atlas" + } + + # 5. Build RIPE-only alerts for countries NOT already in IODA + ripe_alerts = [] + for cc, probes in country_disc.items(): + if cc.upper() in ioda_countries: + continue # IODA already covers this country + if len(probes) < 3: + continue # Too few probes to be meaningful + + # Use centroid of disconnected probes as marker location + lats = [ + p["geometry"]["coordinates"][1] + for p in probes + if p.get("geometry") and p["geometry"].get("coordinates") + ] + lngs = [ + p["geometry"]["coordinates"][0] + for p in probes + if p.get("geometry") and p["geometry"].get("coordinates") + ] + if not lats: + continue + + disc_count = len(probes) + # Severity: scale 10-80 based on disconnected probe count + severity = min(80, 10 + disc_count * 2) + + ripe_alerts.append({ + "region_code": f"RIPE-{cc}", + "region_name": f"{cc} (Atlas probes)", + "country_code": cc, + "country_name": cc, + "level": "critical" if disc_count >= 10 else "warning", + "datasource": "ripe-atlas", + "severity": severity, + "lat": sum(lats) / len(lats), + "lng": sum(lngs) / len(lngs), + "probe_count": disc_count, + }) + + # 6. Merge into internet_outages — keep IODA entries, replace old RIPE entries + with _data_lock: + current = latest_data.get("internet_outages", []) + ioda_only = [o for o in current if o.get("datasource") != "ripe-atlas"] + latest_data["internet_outages"] = ioda_only + ripe_alerts + + if ripe_alerts: + _mark_fresh("internet_outages") + logger.info( + f"RIPE Atlas: {len(ripe_alerts)} countries with probe disconnections " + f"(from {len(disconnected)} disconnected / ~{total_connected} connected probes)" + ) + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: + logger.error(f"Error fetching RIPE Atlas probes: {e}") + + # --------------------------------------------------------------------------- # Data Centers (local geocoded JSON) # --------------------------------------------------------------------------- @@ -112,6 +236,10 @@ def fetch_internet_outages(): def fetch_datacenters(): """Load geocoded data centers (5K+ street-level precise locations).""" + from services.fetchers._store import is_any_active + + if not is_any_active("datacenters"): + return dcs = [] try: if not _DC_GEOCODED_PATH.exists(): @@ -125,17 +253,28 @@ def fetch_datacenters(): continue if not (-90 <= lat <= 90 and -180 <= lng <= 180): continue - dcs.append({ - "name": entry.get("name", "Unknown"), - "company": entry.get("company", ""), - "street": entry.get("street", ""), - "city": entry.get("city", ""), - "country": entry.get("country", ""), - "zip": entry.get("zip", ""), - "lat": lat, "lng": lng, - }) + dcs.append( + { + "name": entry.get("name", "Unknown"), + "company": entry.get("company", ""), + "street": entry.get("street", ""), + "city": entry.get("city", ""), + "country": entry.get("country", ""), + "zip": entry.get("zip", ""), + "lat": lat, + "lng": lng, + } + ) logger.info(f"Data centers: {len(dcs)} geocoded locations loaded") - except Exception as e: + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: logger.error(f"Error loading data centers: {e}") with _data_lock: latest_data["datacenters"] = dcs @@ -222,16 +361,34 @@ def fetch_power_plants(): # CCTV Cameras # --------------------------------------------------------------------------- def fetch_cctv(): + from services.fetchers._store import is_any_active + + if not is_any_active("cctv"): + return try: from services.cctv_pipeline import get_all_cameras + cameras = get_all_cameras() + if len(cameras) < 500: + # Serve the current DB snapshot immediately and let the scheduled + # ingest cycle populate/refresh cameras asynchronously. + logger.info( + "CCTV DB currently has %d cameras — serving cached snapshot and waiting for scheduled ingest", + len(cameras), + ) with _data_lock: latest_data["cctv"] = cameras _mark_fresh("cctv") - except Exception as e: + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: logger.error(f"Error fetching cctv from DB: {e}") - with _data_lock: - latest_data["cctv"] = [] # --------------------------------------------------------------------------- @@ -239,13 +396,332 @@ def fetch_cctv(): # --------------------------------------------------------------------------- @with_retry(max_retries=2, base_delay=2) def fetch_kiwisdr(): + from services.fetchers._store import is_any_active + + if not is_any_active("kiwisdr"): + return try: from services.kiwisdr_fetcher import fetch_kiwisdr_nodes + nodes = fetch_kiwisdr_nodes() with _data_lock: latest_data["kiwisdr"] = nodes _mark_fresh("kiwisdr") - except Exception as e: + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: logger.error(f"Error fetching KiwiSDR nodes: {e}") with _data_lock: latest_data["kiwisdr"] = [] + + +# --------------------------------------------------------------------------- +# SatNOGS Ground Stations + Observations +# --------------------------------------------------------------------------- +@with_retry(max_retries=2, base_delay=2) +def fetch_satnogs(): + from services.fetchers._store import is_any_active + + if not is_any_active("satnogs"): + return + try: + from services.satnogs_fetcher import fetch_satnogs_stations, fetch_satnogs_observations + + stations = fetch_satnogs_stations() + obs = fetch_satnogs_observations() + with _data_lock: + latest_data["satnogs_stations"] = stations + latest_data["satnogs_observations"] = obs + _mark_fresh("satnogs_stations", "satnogs_observations") + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: + logger.error(f"Error fetching SatNOGS: {e}") + + +# --------------------------------------------------------------------------- +# PSK Reporter — HF Digital Mode Spots +# --------------------------------------------------------------------------- +@with_retry(max_retries=2, base_delay=2) +def fetch_psk_reporter(): + from services.fetchers._store import is_any_active + + if not is_any_active("psk_reporter"): + return + try: + from services.psk_reporter_fetcher import fetch_psk_reporter_spots + + spots = fetch_psk_reporter_spots() + with _data_lock: + latest_data["psk_reporter"] = spots + _mark_fresh("psk_reporter") + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: + logger.error(f"Error fetching PSK Reporter: {e}") + with _data_lock: + latest_data["psk_reporter"] = [] + + +# --------------------------------------------------------------------------- +# TinyGS LoRa Satellites +# --------------------------------------------------------------------------- +@with_retry(max_retries=2, base_delay=2) +def fetch_tinygs(): + from services.fetchers._store import is_any_active + + if not is_any_active("tinygs"): + return + try: + from services.tinygs_fetcher import fetch_tinygs_satellites + + sats = fetch_tinygs_satellites() + with _data_lock: + latest_data["tinygs_satellites"] = sats + _mark_fresh("tinygs_satellites") + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: + logger.error(f"Error fetching TinyGS: {e}") + + +# --------------------------------------------------------------------------- +# Police Scanners (OpenMHZ) — geocode city+state via local GeoNames DB +# --------------------------------------------------------------------------- +_scanner_geo_cache: dict = {} # city|state -> (lat, lng) — populated once from GeoNames + + +def _build_scanner_geo_lookup(): + """Build a US city/county→coords lookup from reverse_geocoder's bundled GeoNames CSV.""" + if _scanner_geo_cache: + return + try: + import csv, os, reverse_geocoder as rg + + geo_file = os.path.join(os.path.dirname(rg.__file__), "rg_cities1000.csv") + # US state abbreviation → admin1 name mapping + _abbr = { + "AL": "Alabama", + "AK": "Alaska", + "AZ": "Arizona", + "AR": "Arkansas", + "CA": "California", + "CO": "Colorado", + "CT": "Connecticut", + "DE": "Delaware", + "FL": "Florida", + "GA": "Georgia", + "HI": "Hawaii", + "ID": "Idaho", + "IL": "Illinois", + "IN": "Indiana", + "IA": "Iowa", + "KS": "Kansas", + "KY": "Kentucky", + "LA": "Louisiana", + "ME": "Maine", + "MD": "Maryland", + "MA": "Massachusetts", + "MI": "Michigan", + "MN": "Minnesota", + "MS": "Mississippi", + "MO": "Missouri", + "MT": "Montana", + "NE": "Nebraska", + "NV": "Nevada", + "NH": "New Hampshire", + "NJ": "New Jersey", + "NM": "New Mexico", + "NY": "New York", + "NC": "North Carolina", + "ND": "North Dakota", + "OH": "Ohio", + "OK": "Oklahoma", + "OR": "Oregon", + "PA": "Pennsylvania", + "RI": "Rhode Island", + "SC": "South Carolina", + "SD": "South Dakota", + "TN": "Tennessee", + "TX": "Texas", + "UT": "Utah", + "VT": "Vermont", + "VA": "Virginia", + "WA": "Washington", + "WV": "West Virginia", + "WI": "Wisconsin", + "WY": "Wyoming", + "DC": "Washington, D.C.", + } + state_full = {v.lower(): k for k, v in _abbr.items()} + state_full["washington, d.c."] = "DC" + + county_coords = {} # admin2(county)|state -> (lat, lon) — first city per county + with open(geo_file, "r", encoding="utf-8") as f: + reader = csv.reader(f) + next(reader, None) # skip header + for row in reader: + if len(row) < 6 or row[5] != "US": + continue + lat_s, lon_s, name, admin1, admin2 = row[0], row[1], row[2], row[3], row[4] + st = state_full.get(admin1.lower(), "") + if not st: + continue + coords = (float(lat_s), float(lon_s)) + # City name → coords + _scanner_geo_cache[f"{name.lower()}|{st}"] = coords + # County name → coords (keep first match per county, usually the largest city) + if admin2: + county_key = f"{admin2.lower()}|{st}" + if county_key not in county_coords: + county_coords[county_key] = coords + # Also strip " County" suffix for matching + stripped = admin2.lower().replace(" county", "").strip() + stripped_key = f"{stripped}|{st}" + if stripped_key not in county_coords: + county_coords[stripped_key] = coords + + # Merge county lookups (don't override city entries) + for k, v in county_coords.items(): + if k not in _scanner_geo_cache: + _scanner_geo_cache[k] = v + # Special case: DC + _scanner_geo_cache["washington|DC"] = (38.89511, -77.03637) + logger.info(f"Scanner geo lookup: {len(_scanner_geo_cache)} US entries loaded") + except Exception as e: + logger.warning(f"Failed to build scanner geo lookup: {e}") + + +def _geocode_scanner(city: str, state: str): + """Look up city+state coordinates from local GeoNames cache.""" + _build_scanner_geo_lookup() + if not city or not state: + return None + st = state.upper() + # Strip trailing state from city (e.g. "Lehigh, PA") + c = city.strip() + if ", " in c: + parts = c.rsplit(", ", 1) + if len(parts[1]) <= 2: + c = parts[0] + name = c.lower() + # Try exact city match + result = _scanner_geo_cache.get(f"{name}|{st}") + if result: + return result + # Strip "County" / "Co" suffix + stripped = name.replace(" county", "").replace(" co", "").strip() + result = _scanner_geo_cache.get(f"{stripped}|{st}") + if result: + return result + # Normalize "St." / "St" → "Saint" + import re + + normed = re.sub(r"\bst\.?\s", "saint ", name) + if normed != name: + result = _scanner_geo_cache.get(f"{normed}|{st}") + if result: + return result + # Also try with "s" suffix: "St. Marys" → "Saint Marys" and "Saint Mary's" + for variant in [normed.rstrip("s"), normed.replace("ys", "y's")]: + result = _scanner_geo_cache.get(f"{variant}|{st}") + if result: + return result + # "Prince Georges" → "Prince George's" (apostrophe variants) + if "georges" in name: + key = name.replace("georges", "george's") + "|" + st + result = _scanner_geo_cache.get(key) + if result: + return result + # Multi-location: "Scott and Carver" → try first part + if " and " in name: + first = name.split(" and ")[0].strip() + result = _scanner_geo_cache.get(f"{first}|{st}") + if result: + return result + # Comma-separated list: "Adams, Jackson, Juneau" → try first + if ", " in name: + first = name.split(", ")[0].strip() + result = _scanner_geo_cache.get(f"{first}|{st}") + if result: + return result + # Drop directional prefix: "North Fulton" → "Fulton" + for prefix in ("north ", "south ", "east ", "west "): + if name.startswith(prefix): + result = _scanner_geo_cache.get(f"{name[len(prefix):]}|{st}") + if result: + return result + return None + + +@with_retry(max_retries=2, base_delay=2) +def fetch_scanners(): + from services.fetchers._store import is_any_active + + if not is_any_active("scanners"): + return + try: + from services.radio_intercept import get_openmhz_systems + + systems = get_openmhz_systems() + scanners = [] + for s in systems: + city = s.get("city", "") or s.get("county", "") or "" + state = s.get("state", "") + coords = _geocode_scanner(city, state) + if not coords: + continue + lat, lng = coords + scanners.append( + { + "shortName": s.get("shortName", ""), + "name": s.get("name", "Unknown Scanner"), + "lat": round(lat, 5), + "lng": round(lng, 5), + "city": city, + "state": state, + "clientCount": s.get("clientCount", 0), + "description": s.get("description", ""), + } + ) + with _data_lock: + latest_data["scanners"] = scanners + if scanners: + _mark_fresh("scanners") + logger.info(f"Scanners: {len(scanners)}/{len(systems)} geocoded") + except ( + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + TypeError, + json.JSONDecodeError, + ) as e: + logger.error(f"Error fetching scanners: {e}") + with _data_lock: + latest_data["scanners"] = [] diff --git a/backend/services/fetchers/meshtastic_map.py b/backend/services/fetchers/meshtastic_map.py new file mode 100644 index 00000000..58eb5dc3 --- /dev/null +++ b/backend/services/fetchers/meshtastic_map.py @@ -0,0 +1,222 @@ +"""Meshtastic Map fetcher — pulls global node positions from meshtastic.liamcottle.net. + +Bootstrap + top-up strategy: + - On startup: fetch all nodes with positions to seed the map + - Every 4 hours: refresh from the API + - Persists to JSON cache so data survives restarts + - MQTT bridge provides real-time updates between API fetches + +API source: https://meshtastic.liamcottle.net/api/v1/nodes (community project by Liam Cottle) +Polling interval deliberately kept low (4h) to be respectful to the service. +""" + +import json +import logging +import time +from datetime import datetime, timezone, timedelta +from pathlib import Path + +import requests + +from services.fetchers._store import latest_data, _data_lock, _mark_fresh + +logger = logging.getLogger("services.data_fetcher") + +_API_URL = "https://meshtastic.liamcottle.net/api/v1/nodes" +_CACHE_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "meshtastic_nodes_cache.json" +_FETCH_TIMEOUT = 90 # seconds — response is ~37MB, needs time on slow connections +_MAX_AGE_HOURS = 4 # discard nodes not seen within this window (matches refresh interval) + +# Track when we last fetched so the frontend can show staleness +_last_fetch_ts: float = 0.0 + + +def _parse_node(node: dict) -> dict | None: + """Convert an API node into a slim signal-like dict.""" + lat_i = node.get("latitude") + lng_i = node.get("longitude") + if lat_i is None or lng_i is None: + return None + + lat = lat_i / 1e7 + lng = lng_i / 1e7 + + # Basic validity + if not (-90 <= lat <= 90 and -180 <= lng <= 180): + return None + if abs(lat) < 0.1 and abs(lng) < 0.1: + return None + + callsign = node.get("node_id_hex", "") + if not callsign: + nid = node.get("node_id") + callsign = f"!{int(nid):08x}" if nid else "" + if not callsign: + return None + + # Position age from API — reject nodes older than _MAX_AGE_HOURS + pos_updated = node.get("position_updated_at") or node.get("updated_at", "") + if pos_updated: + try: + ts = datetime.fromisoformat(pos_updated.replace("Z", "+00:00")) + if datetime.now(timezone.utc) - ts > timedelta(hours=_MAX_AGE_HOURS): + return None + except (ValueError, TypeError): + pass + else: + return None # no timestamp at all — skip + + return { + "callsign": callsign[:20], + "lat": round(lat, 5), + "lng": round(lng, 5), + "source": "meshtastic", + "confidence": 0.5, + "timestamp": pos_updated, + "position_updated_at": pos_updated, + "from_api": True, + "long_name": (node.get("long_name") or "")[:40], + "short_name": (node.get("short_name") or "")[:4], + "hardware": node.get("hardware_model_name", ""), + "role": node.get("role_name", ""), + "battery_level": node.get("battery_level"), + "voltage": node.get("voltage"), + "altitude": node.get("altitude"), + } + + +def _is_fresh(node: dict) -> bool: + """Check if a cached node is still within the _MAX_AGE_HOURS window.""" + ts_str = node.get("position_updated_at") or node.get("timestamp", "") + if not ts_str: + return False + try: + ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + return datetime.now(timezone.utc) - ts <= timedelta(hours=_MAX_AGE_HOURS) + except (ValueError, TypeError): + return False + + +def _load_cache() -> list[dict]: + """Load cached nodes from disk, filtering out stale entries.""" + if _CACHE_FILE.exists(): + try: + data = json.loads(_CACHE_FILE.read_text(encoding="utf-8")) + nodes = data.get("nodes", []) + fresh = [n for n in nodes if _is_fresh(n)] + logger.info(f"Meshtastic map cache loaded: {len(fresh)} fresh / {len(nodes)} total") + return fresh + except Exception as e: + logger.warning(f"Failed to load meshtastic cache: {e}") + return [] + + +def _save_cache(nodes: list[dict], fetch_ts: float): + """Persist processed nodes to disk.""" + try: + _CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) + _CACHE_FILE.write_text( + json.dumps( + { + "fetched_at": fetch_ts, + "count": len(nodes), + "nodes": nodes, + } + ), + encoding="utf-8", + ) + except Exception as e: + logger.warning(f"Failed to save meshtastic cache: {e}") + + +def fetch_meshtastic_nodes(): + """Fetch global Meshtastic node positions from Liam Cottle's map API. + + Stores processed nodes in latest_data["meshtastic_map_nodes"]. + Persists to JSON cache for restart resilience. + """ + from services.fetchers._store import is_any_active + + if not is_any_active("sigint_meshtastic"): + return + global _last_fetch_ts + + try: + logger.info("Fetching Meshtastic map nodes from API...") + resp = requests.get( + _API_URL, + timeout=_FETCH_TIMEOUT, + headers={ + "User-Agent": "ShadowBroker/1.0 (OSINT dashboard, 4h polling)", + "Accept": "application/json", + }, + ) + resp.raise_for_status() + + raw = resp.json() + raw_nodes = raw.get("nodes", []) if isinstance(raw, dict) else raw + + # Parse and filter to only nodes with valid positions + parsed = [] + for node in raw_nodes: + sig = _parse_node(node) + if sig: + parsed.append(sig) + + _last_fetch_ts = time.time() + _save_cache(parsed, _last_fetch_ts) + + with _data_lock: + latest_data["meshtastic_map_nodes"] = parsed + latest_data["meshtastic_map_fetched_at"] = _last_fetch_ts + try: + from services.fetchers.sigint import refresh_sigint_snapshot + + refresh_sigint_snapshot() + except Exception as exc: + logger.debug(f"Meshtastic map: SIGINT snapshot refresh skipped: {exc}") + + logger.info( + f"Meshtastic map: {len(parsed)} nodes with positions " f"(from {len(raw_nodes)} total)" + ) + + except Exception as e: + logger.error(f"Meshtastic map fetch failed: {e}") + # Fall back to cache if available and we have nothing in memory + with _data_lock: + if not latest_data.get("meshtastic_map_nodes"): + cached = _load_cache() + if cached: + latest_data["meshtastic_map_nodes"] = cached + latest_data["meshtastic_map_fetched_at"] = ( + _CACHE_FILE.stat().st_mtime if _CACHE_FILE.exists() else 0 + ) + logger.info( + f"Meshtastic map: using {len(cached)} cached nodes (API unavailable)" + ) + try: + from services.fetchers.sigint import refresh_sigint_snapshot + + refresh_sigint_snapshot() + except Exception as exc: + logger.debug(f"Meshtastic map cache: SIGINT snapshot refresh skipped: {exc}") + + _mark_fresh("meshtastic_map") + + +def load_meshtastic_cache_if_available(): + """On startup, load cached nodes immediately (before first API fetch).""" + global _last_fetch_ts + cached = _load_cache() + if cached: + with _data_lock: + latest_data["meshtastic_map_nodes"] = cached + _last_fetch_ts = _CACHE_FILE.stat().st_mtime if _CACHE_FILE.exists() else 0 + latest_data["meshtastic_map_fetched_at"] = _last_fetch_ts + try: + from services.fetchers.sigint import refresh_sigint_snapshot + + refresh_sigint_snapshot() + except Exception as exc: + logger.debug(f"Meshtastic preload: SIGINT snapshot refresh skipped: {exc}") + logger.info(f"Meshtastic map: preloaded {len(cached)} nodes from cache") diff --git a/backend/services/fetchers/military.py b/backend/services/fetchers/military.py index 73fd1dfc..72c4d51c 100644 --- a/backend/services/fetchers/military.py +++ b/backend/services/fetchers/military.py @@ -1,4 +1,5 @@ """Military flight tracking and UAV detection from ADS-B data.""" + import json import logging import requests @@ -13,7 +14,21 @@ # --------------------------------------------------------------------------- _UAV_TYPE_CODES = {"Q9", "R4", "TB2", "MALE", "HALE", "HERM", "HRON"} _UAV_CALLSIGN_PREFIXES = ("FORTE", "GHAWK", "REAP", "BAMS", "UAV", "UAS") -_UAV_MODEL_KEYWORDS = ("RQ-", "MQ-", "RQ4", "MQ9", "MQ4", "MQ1", "REAPER", "GLOBALHAWK", "TRITON", "PREDATOR", "HERMES", "HERON", "BAYRAKTAR") +_UAV_MODEL_KEYWORDS = ( + "RQ-", + "MQ-", + "RQ4", + "MQ9", + "MQ4", + "MQ1", + "REAPER", + "GLOBALHAWK", + "TRITON", + "PREDATOR", + "HERMES", + "HERON", + "BAYRAKTAR", +) _UAV_WIKI = { "RQ4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk", "RQ-4": "https://en.wikipedia.org/wiki/Northrop_Grumman_RQ-4_Global_Hawk", @@ -137,13 +152,41 @@ def _classify_uav(model: str, callsign: str): def fetch_military_flights(): + from services.fetchers._store import is_any_active + + if not is_any_active("military"): + return military_flights = [] detected_uavs = [] + # Fetch from primary + supplemental military endpoints + all_mil_ac = [] + seen_hex = set() try: url = "https://api.adsb.lol/v2/mil" response = fetch_with_curl(url, timeout=10) if response.status_code == 200: - ac = response.json().get('ac', []) + for a in response.json().get("ac", []): + h = a.get("hex", "").lower() + if h and h not in seen_hex: + seen_hex.add(h) + all_mil_ac.append(a) + except Exception as e: + logger.warning(f"adsb.lol mil fetch failed: {e}") + # Supplemental: airplanes.live military endpoint + try: + resp2 = fetch_with_curl("https://api.airplanes.live/v2/mil", timeout=10) + if resp2.status_code == 200: + for a in resp2.json().get("ac", []): + h = a.get("hex", "").lower() + if h and h not in seen_hex: + seen_hex.add(h) + all_mil_ac.append(a) + logger.info(f"airplanes.live mil: +{len(resp2.json().get('ac', []))} raw, {len(all_mil_ac)} total unique") + except Exception as e: + logger.debug(f"airplanes.live mil supplemental failed: {e}") + try: + if all_mil_ac: + ac = all_mil_ac for f in ac: try: lat = f.get("lat") @@ -218,18 +261,25 @@ def fetch_military_flights(): except Exception as loop_e: logger.error(f"Mil flight interpolation error: {loop_e}") continue - except Exception as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + OSError, + ValueError, + KeyError, + ) as e: logger.error(f"Error fetching military flights: {e}") if not military_flights and not detected_uavs: logger.warning("No military flights retrieved — keeping previous data if available") with _data_lock: - if latest_data.get('military_flights'): + if latest_data.get("military_flights"): return with _data_lock: - latest_data['military_flights'] = military_flights - latest_data['uavs'] = detected_uavs + latest_data["military_flights"] = military_flights + latest_data["uavs"] = detected_uavs _mark_fresh("military_flights", "uavs") logger.info(f"UAVs: {len(detected_uavs)} real drones detected via ADS-B") @@ -238,30 +288,30 @@ def fetch_military_flights(): remaining_mil = [] for mf in military_flights: enrich_with_plane_alert(mf) - if mf.get('alert_category'): - mf['type'] = 'tracked_flight' + if mf.get("alert_category"): + mf["type"] = "tracked_flight" tracked_mil.append(mf) else: remaining_mil.append(mf) with _data_lock: - latest_data['military_flights'] = remaining_mil + latest_data["military_flights"] = remaining_mil # Store tracked military flights — update positions for existing entries with _data_lock: - existing_tracked = list(latest_data.get('tracked_flights', [])) + existing_tracked = list(latest_data.get("tracked_flights", [])) fresh_mil_map = {} for t in tracked_mil: - icao = t.get('icao24', '').upper() + icao = t.get("icao24", "").upper() if icao: fresh_mil_map[icao] = t updated_tracked = [] seen_icaos = set() for old_t in existing_tracked: - icao = old_t.get('icao24', '').upper() + icao = old_t.get("icao24", "").upper() if icao in fresh_mil_map: fresh = fresh_mil_map[icao] - for key in ('alert_category', 'alert_operator', 'alert_special', 'alert_flag'): + for key in ("alert_category", "alert_operator", "alert_special", "alert_flag"): if key in old_t and key not in fresh: fresh[key] = old_t[key] updated_tracked.append(fresh) @@ -273,5 +323,5 @@ def fetch_military_flights(): if icao not in seen_icaos: updated_tracked.append(t) with _data_lock: - latest_data['tracked_flights'] = updated_tracked + latest_data["tracked_flights"] = updated_tracked logger.info(f"Tracked flights: {len(updated_tracked)} total ({len(tracked_mil)} from military)") diff --git a/backend/services/fetchers/news.py b/backend/services/fetchers/news.py index c8256ac5..7ac2dfd4 100644 --- a/backend/services/fetchers/news.py +++ b/backend/services/fetchers/news.py @@ -7,6 +7,7 @@ from services.network_utils import fetch_with_curl from services.fetchers._store import latest_data, _data_lock, _mark_fresh from services.fetchers.retry import with_retry +from services.oracle_service import enrich_news_items, compute_global_threat_level, detect_breaking_events logger = logging.getLogger("services.data_fetcher") @@ -170,7 +171,7 @@ def _fetch_feed(item): logger.warning(f"Feed {source_name} failed: {e}") return source_name, None - with concurrent.futures.ThreadPoolExecutor(max_workers=len(feeds)) as pool: + with concurrent.futures.ThreadPoolExecutor(max_workers=min(len(feeds), 6)) as pool: feed_results = list(pool.map(_fetch_feed, feeds.items())) for source_name, feed in feed_results: @@ -191,7 +192,14 @@ def _fetch_feed(item): elif alert_level == "Orange": risk_score = 7 else: risk_score = 4 else: - risk_keywords = ['war', 'missile', 'strike', 'attack', 'crisis', 'tension', 'military', 'conflict', 'defense', 'clash', 'nuclear'] + risk_keywords = [ + 'war', 'missile', 'strike', 'attack', 'crisis', 'tension', + 'military', 'conflict', 'defense', 'clash', 'nuclear', + 'sanctions', 'ceasefire', 'invasion', 'drone', 'artillery', + 'blockade', 'escalation', 'casualties', 'airspace', + 'mobilization', 'proxy', 'insurgent', 'coup', + 'assassination', 'bioweapon', 'chemical', + ] text = (title + " " + summary).lower() risk_score = 1 @@ -268,6 +276,36 @@ def _fetch_feed(item): }) news_items.sort(key=lambda x: x['risk_score'], reverse=True) + + # Oracle enrichment: sentiment, oracle scores, prediction market odds + try: + with _data_lock: + markets = list(latest_data.get("prediction_markets", [])) + enrich_news_items(news_items, source_weights, markets) + detect_breaking_events(news_items) + except Exception as e: + logger.warning(f"Oracle enrichment failed (news still usable): {e}") + + # Global threat level computation (fuses news + markets + military + jamming) + try: + with _data_lock: + markets = list(latest_data.get("prediction_markets", [])) + mil_flights = list(latest_data.get("military_flights", [])) + jam_zones = list(latest_data.get("gps_jamming", [])) + ships = list(latest_data.get("ships", [])) + corr_alerts = list(latest_data.get("correlations", [])) + threat_level = compute_global_threat_level( + news_items, markets, + military_flights=mil_flights, + gps_jamming=jam_zones, + ships=ships, + correlations=corr_alerts, + ) + except Exception as e: + logger.warning(f"Threat level computation failed: {e}") + threat_level = {"score": 0, "level": "GREEN", "color": "#22c55e", "drivers": []} + with _data_lock: latest_data['news'] = news_items + latest_data['threat_level'] = threat_level _mark_fresh("news") diff --git a/backend/services/fetchers/plane_alert.py b/backend/services/fetchers/plane_alert.py index 715d2da7..3ce62547 100644 --- a/backend/services/fetchers/plane_alert.py +++ b/backend/services/fetchers/plane_alert.py @@ -1,4 +1,5 @@ """Plane-Alert DB — load and enrich aircraft with tracked metadata.""" + import os import json import logging @@ -71,38 +72,126 @@ "Radiohead": "purple", } + def _category_to_color(cat: str) -> str: """O(1) exact lookup. Unknown categories default to purple.""" return _CATEGORY_COLOR.get(cat, "purple") + _PLANE_ALERT_DB: dict = {} # --------------------------------------------------------------------------- # POTUS Fleet — override colors and operator names for presidential aircraft. # --------------------------------------------------------------------------- _POTUS_FLEET: dict[str, dict] = { - "ADFDF8": {"color": "#ff1493", "operator": "Air Force One (82-8000)", "category": "Head of State", "wiki": "Air_Force_One", "fleet": "AF1"}, - "ADFDF9": {"color": "#ff1493", "operator": "Air Force One (92-9000)", "category": "Head of State", "wiki": "Air_Force_One", "fleet": "AF1"}, - "ADFEB7": {"color": "blue", "operator": "Air Force Two (98-0001)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "ADFEB8": {"color": "blue", "operator": "Air Force Two (98-0002)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "ADFEB9": {"color": "blue", "operator": "Air Force Two (99-0003)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "ADFEBA": {"color": "blue", "operator": "Air Force Two (99-0004)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "AE4AE6": {"color": "blue", "operator": "Air Force Two (09-0015)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "AE4AE8": {"color": "blue", "operator": "Air Force Two (09-0016)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "AE4AEA": {"color": "blue", "operator": "Air Force Two (09-0017)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "AE4AEC": {"color": "blue", "operator": "Air Force Two (19-0018)", "category": "Governments", "wiki": "Air_Force_Two", "fleet": "AF2"}, - "AE0865": {"color": "#ff1493", "operator": "Marine One (VH-3D)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"}, - "AE5E76": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"}, - "AE5E77": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"}, - "AE5E79": {"color": "#ff1493", "operator": "Marine One (VH-92A)", "category": "Head of State", "wiki": "Marine_One", "fleet": "M1"}, + "ADFDF8": { + "color": "#ff1493", + "operator": "Air Force One (82-8000)", + "category": "Head of State", + "wiki": "Air_Force_One", + "fleet": "AF1", + }, + "ADFDF9": { + "color": "#ff1493", + "operator": "Air Force One (92-9000)", + "category": "Head of State", + "wiki": "Air_Force_One", + "fleet": "AF1", + }, + "ADFEB7": { + "color": "blue", + "operator": "Air Force Two (98-0001)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "ADFEB8": { + "color": "blue", + "operator": "Air Force Two (98-0002)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "ADFEB9": { + "color": "blue", + "operator": "Air Force Two (99-0003)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "ADFEBA": { + "color": "blue", + "operator": "Air Force Two (99-0004)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "AE4AE6": { + "color": "blue", + "operator": "Air Force Two (09-0015)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "AE4AE8": { + "color": "blue", + "operator": "Air Force Two (09-0016)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "AE4AEA": { + "color": "blue", + "operator": "Air Force Two (09-0017)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "AE4AEC": { + "color": "blue", + "operator": "Air Force Two (19-0018)", + "category": "Governments", + "wiki": "Air_Force_Two", + "fleet": "AF2", + }, + "AE0865": { + "color": "#ff1493", + "operator": "Marine One (VH-3D)", + "category": "Head of State", + "wiki": "Marine_One", + "fleet": "M1", + }, + "AE5E76": { + "color": "#ff1493", + "operator": "Marine One (VH-92A)", + "category": "Head of State", + "wiki": "Marine_One", + "fleet": "M1", + }, + "AE5E77": { + "color": "#ff1493", + "operator": "Marine One (VH-92A)", + "category": "Head of State", + "wiki": "Marine_One", + "fleet": "M1", + }, + "AE5E79": { + "color": "#ff1493", + "operator": "Marine One (VH-92A)", + "category": "Head of State", + "wiki": "Marine_One", + "fleet": "M1", + }, } + def _load_plane_alert_db(): """Load plane_alert_db.json (exported from SQLite) into memory.""" global _PLANE_ALERT_DB json_path = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "data", "plane_alert_db.json" + "data", + "plane_alert_db.json", ) if not os.path.exists(json_path): logger.warning(f"Plane-Alert DB not found at {json_path}") @@ -124,8 +213,10 @@ def _load_plane_alert_db(): except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e: logger.error(f"Failed to load Plane-Alert DB: {e}") + _load_plane_alert_db() + def enrich_with_plane_alert(flight: dict) -> dict: """If flight's icao24 is in the Plane-Alert DB, add alert metadata.""" icao = flight.get("icao24", "").strip().upper() @@ -145,13 +236,16 @@ def enrich_with_plane_alert(flight: dict) -> dict: flight["registration"] = info["registration"] return flight + _TRACKED_NAMES_DB: dict = {} + def _load_tracked_names(): global _TRACKED_NAMES_DB json_path = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "data", "tracked_names.json" + "data", + "tracked_names.json", ) if not os.path.exists(json_path): return @@ -160,16 +254,22 @@ def _load_tracked_names(): data = json.load(f) for name, info in data.get("details", {}).items(): cat = info.get("category", "Other") + socials = info.get("socials") for reg in info.get("registrations", []): reg_clean = reg.strip().upper() if reg_clean: - _TRACKED_NAMES_DB[reg_clean] = {"name": name, "category": cat} + entry = {"name": name, "category": cat} + if socials: + entry["socials"] = socials + _TRACKED_NAMES_DB[reg_clean] = entry logger.info(f"Tracked Names DB loaded: {len(_TRACKED_NAMES_DB)} registrations") except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError) as e: logger.error(f"Failed to load Tracked Names DB: {e}") + _load_tracked_names() + def enrich_with_tracked_names(flight: dict) -> dict: """If flight's registration matches our Excel extraction, tag it as tracked.""" icao = flight.get("icao24", "").strip().upper() @@ -189,11 +289,50 @@ def enrich_with_tracked_names(flight: dict) -> dict: name = match["name"] flight["alert_operator"] = name flight["alert_category"] = match["category"] + if match.get("socials"): + flight["alert_socials"] = match["socials"] name_lower = name.lower() - is_gov = any(w in name_lower for w in ['state of ', 'government', 'republic', 'ministry', 'department', 'federal', 'cia']) - is_law = any(w in name_lower for w in ['police', 'marshal', 'sheriff', 'douane', 'customs', 'patrol', 'gendarmerie', 'guardia', 'law enforcement']) - is_med = any(w in name_lower for w in ['fire', 'bomberos', 'ambulance', 'paramedic', 'medevac', 'rescue', 'hospital', 'medical', 'lifeflight']) + is_gov = any( + w in name_lower + for w in [ + "state of ", + "government", + "republic", + "ministry", + "department", + "federal", + "cia", + ] + ) + is_law = any( + w in name_lower + for w in [ + "police", + "marshal", + "sheriff", + "douane", + "customs", + "patrol", + "gendarmerie", + "guardia", + "law enforcement", + ] + ) + is_med = any( + w in name_lower + for w in [ + "fire", + "bomberos", + "ambulance", + "paramedic", + "medevac", + "rescue", + "hospital", + "medical", + "lifeflight", + ] + ) if is_gov or is_law: flight["alert_color"] = "blue" diff --git a/backend/services/fetchers/prediction_markets.py b/backend/services/fetchers/prediction_markets.py new file mode 100644 index 00000000..b8fff5a1 --- /dev/null +++ b/backend/services/fetchers/prediction_markets.py @@ -0,0 +1,647 @@ +"""Prediction market fetcher — Polymarket (Gamma API) + Kalshi. + +Fetches active prediction market events from both platforms, merges them by +topic similarity, classifies into categories, and stores merged odds with +full metadata (volume, end dates, descriptions, source badges). +""" + +import json +import logging +import math +from cachetools import TTLCache, cached + +logger = logging.getLogger("services.data_fetcher") + +_market_cache = TTLCache(maxsize=1, ttl=60) # 60-second TTL — markets change fast + +# Delta tracking: {market_title: previous_consensus_pct} +_prev_probabilities: dict[str, float] = {} + + +def _finite_or_none(value): + try: + n = float(value) + except (TypeError, ValueError): + return None + return n if math.isfinite(n) else None + +# --------------------------------------------------------------------------- +# Category classification +# --------------------------------------------------------------------------- +CATEGORIES = ["POLITICS", "CONFLICT", "NEWS", "FINANCE", "CRYPTO"] + +_KALSHI_CATEGORY_MAP = { + "Politics": "POLITICS", + "World": "NEWS", + "Economics": "FINANCE", + "Financials": "FINANCE", + "Tech": "FINANCE", + "Science": "NEWS", + "Climate and Weather": "NEWS", + "Sports": "NEWS", + "Culture": "NEWS", +} + +_TAG_CATEGORY_MAP = { + "Politics": "POLITICS", + "Elections": "POLITICS", + "US Politics": "POLITICS", + "Trump": "POLITICS", + "Congress": "POLITICS", + "Supreme Court": "POLITICS", + "Geopolitics": "CONFLICT", + "War": "CONFLICT", + "Military": "CONFLICT", + "Finance": "FINANCE", + "Stocks": "FINANCE", + "Economy": "FINANCE", + "Business": "FINANCE", + "IPOs": "FINANCE", + "Crypto": "CRYPTO", + "Bitcoin": "CRYPTO", + "Ethereum": "CRYPTO", + "AI": "NEWS", + "Science": "NEWS", + "Sports": "NEWS", + "Culture": "NEWS", + "Entertainment": "NEWS", + "Tech": "FINANCE", +} + +_KEYWORD_CATEGORIES = { + "CONFLICT": [ + "war", + "military", + "attack", + "missile", + "invasion", + "ukraine", + "russia", + "gaza", + "israel", + "nato", + "troops", + "bombing", + "nuclear", + "sanctions", + "ceasefire", + "houthi", + "iran", + "china taiwan", + "clash", + "conflict", + "strike", + "weapon", + ], + "POLITICS": [ + "trump", + "biden", + "election", + "congress", + "senate", + "governor", + "president", + "democrat", + "republican", + "vote", + "party", + "cabinet", + "impeach", + "legislation", + "scotus", + "poll", + "vance", + "speaker", + "parliament", + "prime minister", + "macron", + "starmer", + ], + "CRYPTO": [ + "bitcoin", + "btc", + "ethereum", + "eth", + "crypto", + "blockchain", + "solana", + "defi", + "nft", + "binance", + "coinbase", + "token", + "microstrategy", + "stablecoin", + ], + "FINANCE": [ + "stock", + "fed", + "interest rate", + "inflation", + "gdp", + "recession", + "s&p", + "nasdaq", + "dow", + "oil", + "gold", + "treasury", + "tariff", + "ipo", + "earnings", + "market cap", + "revenue", + ], +} + + +def _classify_category(title: str, poly_tags: list[str], kalshi_category: str) -> str: + """Classify a market into one of the 5 categories.""" + # 1. Kalshi native category + if kalshi_category: + mapped = _KALSHI_CATEGORY_MAP.get(kalshi_category) + if mapped: + return mapped + # 2. Polymarket tag labels + for tag in poly_tags: + mapped = _TAG_CATEGORY_MAP.get(tag) + if mapped: + return mapped + # 3. Keyword matching + title_lower = title.lower() + for cat, keywords in _KEYWORD_CATEGORIES.items(): + for kw in keywords: + if kw in title_lower: + return cat + # 4. Default + return "NEWS" + + +# --------------------------------------------------------------------------- +# Polymarket +# --------------------------------------------------------------------------- +def _fetch_polymarket_events() -> list[dict]: + """Fetch active events from Polymarket Gamma API (no auth required). + + Fetches up to 500 events (multiple pages) for better search coverage. + """ + from services.network_utils import fetch_with_curl + + all_events = [] + for offset in range(0, 500, 100): + try: + resp = fetch_with_curl( + f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=100&offset={offset}", + timeout=15, + ) + if not resp or resp.status_code != 200: + break + page = resp.json() + if not isinstance(page, list) or not page: + break + all_events.extend(page) + except Exception as e: + logger.warning(f"Polymarket page offset={offset} error: {e}") + break + + if not all_events: + return [] + + try: + results = [] + for ev in all_events: + title = ev.get("title", "") + if not title: + continue + # Extract best probability + outcomes from markets + markets = ev.get("markets", []) + best_pct = None + total_volume = 0 + outcomes = [] + for m in markets: + # Use outcomePrices[0] (Yes price) when available — lastTradePrice + # can be for either Yes or No side, causing "99%" for unlikely events + raw_op = m.get("outcomePrices") + price = None + try: + op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op + if isinstance(op, list) and len(op) >= 1: + price = _finite_or_none(op[0]) + except (json.JSONDecodeError, ValueError, TypeError): + pass + if price is None: + price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid")) + pct = None + if price is not None: + try: + pct = round(price * 100, 1) + if best_pct is None or pct > best_pct: + best_pct = pct + except (ValueError, TypeError): + pass + try: + volume = _finite_or_none(m.get("volume", 0) or 0) + if volume is not None: + total_volume += volume + except (ValueError, TypeError): + pass + # Collect named outcomes for multi-outcome events + oname = m.get("groupItemTitle") or "" + if oname and pct is not None: + outcomes.append({"name": oname, "pct": pct}) + # Only keep outcomes for multi-outcome markets (3+ named outcomes) + if len(outcomes) > 2: + outcomes.sort(key=lambda x: x["pct"], reverse=True) + else: + outcomes = [] + + # Extract tag labels + tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")] + + results.append( + { + "title": title, + "source": "polymarket", + "pct": best_pct, + "slug": ev.get("slug", ""), + "description": ev.get("description") or "", + "end_date": ev.get("endDate"), + "volume": round(total_volume, 2), + "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2), + "tags": tag_labels, + "outcomes": outcomes, + } + ) + logger.info(f"Polymarket: fetched {len(results)} active events") + return results + except Exception as e: + logger.error(f"Polymarket fetch error: {e}") + return [] + + +# --------------------------------------------------------------------------- +# Kalshi +# --------------------------------------------------------------------------- +def _fetch_kalshi_events() -> list[dict]: + """Fetch active events from Kalshi public API (no auth required).""" + from services.network_utils import fetch_with_curl + + try: + resp = fetch_with_curl( + "https://api.elections.kalshi.com/v1/events?status=open&limit=100", + timeout=15, + ) + if not resp or resp.status_code != 200: + logger.warning(f"Kalshi API returned {getattr(resp, 'status_code', 'N/A')}") + return [] + data = resp.json() + events = data.get("events", []) if isinstance(data, dict) else [] + + results = [] + for ev in events: + title = ev.get("title", "") + if not title: + continue + markets = ev.get("markets", []) + best_pct = None + total_volume = 0 + close_dates = [] + outcomes = [] + for m in markets: + price = m.get("yes_price") or m.get("last_price") + pct = None + if price is not None: + try: + price = _finite_or_none(price) + if price is None: + raise ValueError("non-finite") + pct = round(price, 1) + if pct <= 1: + pct = round(pct * 100, 1) + if best_pct is None or pct > best_pct: + best_pct = pct + except (ValueError, TypeError): + pass + try: + volume = _finite_or_none( + m.get("dollar_volume", 0) or m.get("volume", 0) or 0 + ) + if volume is not None: + total_volume += int(volume) + except (ValueError, TypeError): + pass + cd = m.get("close_date") + if cd: + close_dates.append(cd) + # Collect named outcomes for multi-outcome events + oname = m.get("title") or m.get("subtitle", "") + if oname and pct is not None: + outcomes.append({"name": oname, "pct": pct}) + # Only keep outcomes for multi-outcome markets (3+ named outcomes) + if len(outcomes) > 2: + outcomes.sort(key=lambda x: x["pct"], reverse=True) + else: + outcomes = [] + + # Description: settle_details or underlying + desc = (ev.get("settle_details") or ev.get("underlying") or "").strip() + sub = ev.get("sub_title", "") + + results.append( + { + "title": title, + "source": "kalshi", + "pct": best_pct, + "ticker": ev.get("ticker", ""), + "description": desc, + "sub_title": sub, + "end_date": max(close_dates) if close_dates else None, + "volume": total_volume, + "category": ev.get("category", ""), + "outcomes": outcomes, + } + ) + logger.info(f"Kalshi: fetched {len(results)} active events") + return results + except Exception as e: + logger.error(f"Kalshi fetch error: {e}") + return [] + + +# --------------------------------------------------------------------------- +# Merge + classify +# --------------------------------------------------------------------------- +def _jaccard(a: str, b: str) -> float: + """Word-level Jaccard similarity between two strings.""" + wa = set(a.lower().split()) + wb = set(b.lower().split()) + if not wa or not wb: + return 0.0 + return len(wa & wb) / len(wa | wb) + + +def _merge_markets(poly_events: list[dict], kalshi_events: list[dict]) -> list[dict]: + """Merge Polymarket and Kalshi events by title similarity. + + Returns a unified list with full metadata, categorized. + """ + merged = [] + used_kalshi = set() + + for pe in poly_events: + best_match = None + best_score = 0.0 + for i, ke in enumerate(kalshi_events): + if i in used_kalshi: + continue + score = _jaccard(pe["title"], ke["title"]) + if score > best_score and score >= 0.25: + best_score = score + best_match = (i, ke) + + poly_pct = _finite_or_none(pe.get("pct")) + kalshi_pct = None + kalshi_vol = 0 + kalshi_cat = "" + kalshi_end = None + kalshi_desc = "" + kalshi_ticker = "" + + if best_match: + used_kalshi.add(best_match[0]) + ke = best_match[1] + kalshi_pct = _finite_or_none(ke.get("pct")) + kalshi_vol = _finite_or_none(ke.get("volume", 0)) or 0 + kalshi_cat = ke.get("category", "") + kalshi_end = ke.get("end_date") + kalshi_desc = ke.get("description", "") + kalshi_ticker = ke.get("ticker", "") + + pcts = [p for p in [poly_pct, kalshi_pct] if p is not None] + consensus = round(sum(pcts) / len(pcts), 1) if pcts else None + + # Build sources list + sources = [] + if poly_pct is not None: + sources.append({"name": "POLY", "pct": poly_pct}) + if kalshi_pct is not None: + sources.append({"name": "KALSHI", "pct": kalshi_pct}) + + category = _classify_category(pe["title"], pe.get("tags", []), kalshi_cat) + + # Use best available description + desc = pe.get("description", "") or kalshi_desc + end_date = pe.get("end_date") or kalshi_end + + # Use whichever source has more outcomes + poly_outcomes = pe.get("outcomes", []) + kalshi_outcomes = best_match[1].get("outcomes", []) if best_match else [] + outcomes = poly_outcomes if len(poly_outcomes) >= len(kalshi_outcomes) else kalshi_outcomes + + merged.append( + { + "title": pe["title"], + "polymarket_pct": poly_pct, + "kalshi_pct": kalshi_pct, + "consensus_pct": consensus, + "description": desc, + "end_date": end_date, + "volume": _finite_or_none(pe.get("volume", 0)) or 0, + "volume_24h": _finite_or_none(pe.get("volume_24h", 0)) or 0, + "kalshi_volume": kalshi_vol, + "category": category, + "sources": sources, + "slug": pe.get("slug", ""), + "kalshi_ticker": kalshi_ticker, + "outcomes": outcomes, + } + ) + + # Unmatched Kalshi events + for i, ke in enumerate(kalshi_events): + if i in used_kalshi: + continue + pct = _finite_or_none(ke.get("pct")) + sources = [] + if pct is not None: + sources.append({"name": "KALSHI", "pct": pct}) + category = _classify_category(ke["title"], [], ke.get("category", "")) + merged.append( + { + "title": ke["title"], + "polymarket_pct": None, + "kalshi_pct": pct, + "consensus_pct": pct, + "description": ke.get("description", ""), + "end_date": ke.get("end_date"), + "volume": 0, + "volume_24h": 0, + "kalshi_volume": _finite_or_none(ke.get("volume", 0)) or 0, + "category": category, + "sources": sources, + "slug": "", + "kalshi_ticker": ke.get("ticker", ""), + "outcomes": ke.get("outcomes", []), + } + ) + + return merged + + +@cached(_market_cache) +def fetch_prediction_markets_raw() -> list[dict]: + """Fetch and merge prediction markets from both sources. Cached 5 min.""" + poly = _fetch_polymarket_events() + kalshi = _fetch_kalshi_events() + merged = _merge_markets(poly, kalshi) + logger.info( + f"Prediction markets: {len(merged)} merged events " + f"({len(poly)} Polymarket, {len(kalshi)} Kalshi)" + ) + return merged + + +def fetch_prediction_markets(): + """Fetcher entry point — writes merged markets to latest_data.""" + from services.fetchers._store import latest_data, _data_lock, _mark_fresh + global _prev_probabilities + + markets = fetch_prediction_markets_raw() + + # Compute probability deltas vs previous fetch + new_probs: dict[str, float] = {} + for m in markets: + title = m.get("title", "") + pct = m.get("consensus_pct") + if title and pct is not None: + prev = _prev_probabilities.get(title) + if prev is not None: + m["delta_pct"] = round(pct - prev, 1) + else: + m["delta_pct"] = None + new_probs[title] = pct + else: + m["delta_pct"] = None + _prev_probabilities = new_probs + + # Build trending list (top 10 by absolute delta) + trending = sorted( + [m for m in markets if m.get("delta_pct") is not None and m["delta_pct"] != 0], + key=lambda x: abs(x["delta_pct"]), + reverse=True, + )[:10] + + with _data_lock: + latest_data["prediction_markets"] = markets + latest_data["trending_markets"] = trending + _mark_fresh("prediction_markets") + + +# --------------------------------------------------------------------------- +# Direct API search (not limited to cached data) +# --------------------------------------------------------------------------- +def search_polymarket_direct(query: str, limit: int = 20) -> list[dict]: + """Search Polymarket by scanning API pages for title matches. + + The Gamma API has no text search parameter, so we scan cached events + plus additional pages until we find enough matches or exhaust the scan. + """ + from services.network_utils import fetch_with_curl + + q_lower = query.lower() + q_words = set(q_lower.split()) + results = [] + + # Scan up to 2000 events (10 pages of 200) looking for title matches + for offset in range(0, 2000, 200): + try: + resp = fetch_with_curl( + f"https://gamma-api.polymarket.com/events?active=true&closed=false&limit=200&offset={offset}", + timeout=15, + ) + if not resp or resp.status_code != 200: + break + events = resp.json() + if not isinstance(events, list) or not events: + break + + for ev in events: + title = ev.get("title", "") + if not title: + continue + title_lower = title.lower() + # Check if query appears in title or word overlap + if q_lower not in title_lower and not any(w in title_lower for w in q_words): + continue + + # Extract same fields as regular fetch + markets = ev.get("markets", []) + best_pct = None + total_volume = 0 + outcomes = [] + for m in markets: + # Use outcomePrices[0] (Yes price) when available + raw_op = m.get("outcomePrices") + price = None + try: + op = json.loads(raw_op) if isinstance(raw_op, str) else raw_op + if isinstance(op, list) and len(op) >= 1: + price = _finite_or_none(op[0]) + except (json.JSONDecodeError, ValueError, TypeError): + pass + if price is None: + price = _finite_or_none(m.get("lastTradePrice") or m.get("bestBid")) + pct = None + if price is not None: + try: + pct = round(price * 100, 1) + if best_pct is None or pct > best_pct: + best_pct = pct + except (ValueError, TypeError): + pass + try: + volume = _finite_or_none(m.get("volume", 0) or 0) + if volume is not None: + total_volume += volume + except (ValueError, TypeError): + pass + oname = m.get("groupItemTitle") or "" + if oname and pct is not None: + outcomes.append({"name": oname, "pct": pct}) + if len(outcomes) > 2: + outcomes.sort(key=lambda x: x["pct"], reverse=True) + else: + outcomes = [] + + tag_labels = [t.get("label", "") for t in ev.get("tags", []) if t.get("label")] + category = _classify_category(title, tag_labels, "") + sources = [] + if best_pct is not None: + sources.append({"name": "POLY", "pct": best_pct}) + + results.append( + { + "title": title, + "polymarket_pct": best_pct, + "kalshi_pct": None, + "consensus_pct": best_pct, + "description": ev.get("description") or "", + "end_date": ev.get("endDate"), + "volume": round(total_volume, 2), + "volume_24h": round(_finite_or_none(ev.get("volume24hr", 0) or 0) or 0, 2), + "kalshi_volume": 0, + "category": category, + "sources": sources, + "slug": ev.get("slug", ""), + "outcomes": outcomes, + } + ) + # Stop scanning if we have enough results + if len(results) >= limit: + break + except Exception as e: + logger.warning(f"Polymarket search scan offset={offset} error: {e}") + break + + logger.info(f"Polymarket search '{query}': {len(results)} results (scanned API)") + return results[:limit] diff --git a/backend/services/fetchers/retry.py b/backend/services/fetchers/retry.py index 64ff0440..73e2ad0f 100644 --- a/backend/services/fetchers/retry.py +++ b/backend/services/fetchers/retry.py @@ -5,22 +5,37 @@ def fetch_something(): ... """ + import time import random import logging import functools +import requests logger = logging.getLogger(__name__) +# Only retry on transient network/OS errors — not on parse errors, key errors, etc. +TRANSIENT_ERRORS = ( + TimeoutError, + ConnectionError, + OSError, + requests.RequestException, +) + def with_retry(max_retries: int = 3, base_delay: float = 2.0, max_delay: float = 30.0): - """Decorator: retries the wrapped function on any exception with exponential backoff + jitter. + """Decorator: retries the wrapped function on transient errors with exponential backoff + jitter. + + Only retries on network/OS errors (TimeoutError, ConnectionError, OSError, + requests.RequestException). Non-transient errors (ValueError, KeyError, etc.) + propagate immediately. Args: max_retries: Number of retry attempts after the initial failure. base_delay: Base delay (seconds) for exponential backoff (2 → 4 → 8 …). max_delay: Cap on the delay between retries. """ + def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): @@ -28,22 +43,30 @@ def wrapper(*args, **kwargs): for attempt in range(1 + max_retries): try: return func(*args, **kwargs) - except Exception as exc: + except TRANSIENT_ERRORS as exc: last_exc = exc if attempt < max_retries: - delay = min(base_delay * (2 ** attempt), max_delay) + delay = min(base_delay * (2**attempt), max_delay) jitter = random.uniform(0, delay * 0.25) total = delay + jitter logger.warning( "%s failed (attempt %d/%d): %s — retrying in %.1fs", - func.__name__, attempt + 1, max_retries + 1, exc, total, + func.__name__, + attempt + 1, + max_retries + 1, + exc, + total, ) time.sleep(total) else: logger.error( "%s failed after %d attempts: %s", - func.__name__, max_retries + 1, exc, + func.__name__, + max_retries + 1, + exc, ) raise last_exc # type: ignore[misc] + return wrapper + return decorator diff --git a/backend/services/fetchers/satellites.py b/backend/services/fetchers/satellites.py index 870a4a83..d6f6cab9 100644 --- a/backend/services/fetchers/satellites.py +++ b/backend/services/fetchers/satellites.py @@ -6,6 +6,7 @@ - No parallel/concurrent connections — one request at a time - Set a descriptive User-Agent """ + import math import time import json @@ -24,7 +25,9 @@ def _gmst(jd_ut1): """Greenwich Mean Sidereal Time in radians from Julian Date.""" t = (jd_ut1 - 2451545.0) / 36525.0 - gmst_sec = 67310.54841 + (876600.0 * 3600 + 8640184.812866) * t + 0.093104 * t * t - 6.2e-6 * t * t * t + gmst_sec = ( + 67310.54841 + (876600.0 * 3600 + 8640184.812866) * t + 0.093104 * t * t - 6.2e-6 * t * t * t + ) gmst_rad = (gmst_sec % 86400) / 86400.0 * 2 * math.pi return gmst_rad @@ -38,17 +41,21 @@ def _gmst(jd_ut1): _SAT_CACHE_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache.json" _SAT_CACHE_META_PATH = Path(__file__).parent.parent.parent / "data" / "sat_gp_cache_meta.json" + def _load_sat_cache(): """Load satellite GP data from local disk cache.""" try: if _SAT_CACHE_PATH.exists(): import os + age_hours = (time.time() - os.path.getmtime(str(_SAT_CACHE_PATH))) / 3600 if age_hours < 48: with open(_SAT_CACHE_PATH, "r") as f: data = json.load(f) if isinstance(data, list) and len(data) > 10: - logger.info(f"Satellites: Loaded {len(data)} records from disk cache ({age_hours:.1f}h old)") + logger.info( + f"Satellites: Loaded {len(data)} records from disk cache ({age_hours:.1f}h old)" + ) # Restore last_modified from metadata _load_cache_meta() return data @@ -58,6 +65,7 @@ def _load_sat_cache(): logger.warning(f"Satellites: Failed to load disk cache: {e}") return None + def _save_sat_cache(data): """Save satellite GP data to local disk cache.""" try: @@ -69,6 +77,7 @@ def _save_sat_cache(data): except (IOError, OSError) as e: logger.warning(f"Satellites: Failed to save disk cache: {e}") + def _load_cache_meta(): """Load cache metadata (Last-Modified timestamp) from disk.""" try: @@ -79,6 +88,7 @@ def _load_cache_meta(): except (IOError, OSError, json.JSONDecodeError, ValueError, KeyError): pass + def _save_cache_meta(): """Save cache metadata to disk.""" try: @@ -90,54 +100,357 @@ def _save_cache_meta(): # Satellite intelligence classification database _SAT_INTEL_DB = [ - ("USA 224", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}), - ("USA 245", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}), - ("USA 290", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}), - ("USA 314", {"country": "USA", "mission": "military_recon", "sat_type": "KH-11 Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}), - ("USA 338", {"country": "USA", "mission": "military_recon", "sat_type": "Keyhole Successor", "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN"}), - ("TOPAZ", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}), - ("PERSONA", {"country": "Russia", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)"}), - ("KONDOR", {"country": "Russia", "mission": "military_sar", "sat_type": "SAR Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)"}), - ("BARS-M", {"country": "Russia", "mission": "military_recon", "sat_type": "Mapping Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Bars-M"}), - ("YAOGAN", {"country": "China", "mission": "military_recon", "sat_type": "Remote Sensing / ELINT", "wiki": "https://en.wikipedia.org/wiki/Yaogan"}), - ("GAOFEN", {"country": "China", "mission": "military_recon", "sat_type": "High-Res Imaging", "wiki": "https://en.wikipedia.org/wiki/Gaofen"}), - ("JILIN", {"country": "China", "mission": "commercial_imaging", "sat_type": "Video / Imaging", "wiki": "https://en.wikipedia.org/wiki/Jilin-1"}), - ("OFEK", {"country": "Israel", "mission": "military_recon", "sat_type": "Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/Ofeq"}), - ("CSO", {"country": "France", "mission": "military_recon", "sat_type": "Optical Reconnaissance", "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)"}), - ("IGS", {"country": "Japan", "mission": "military_recon", "sat_type": "Intelligence Gathering", "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite"}), - ("CAPELLA", {"country": "USA", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/Capella_Space"}), - ("ICEYE", {"country": "Finland", "mission": "sar", "sat_type": "SAR Microsatellite", "wiki": "https://en.wikipedia.org/wiki/ICEYE"}), - ("COSMO", {"country": "Italy", "mission": "sar", "sat_type": "SAR Constellation", "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed"}), - ("TANDEM", {"country": "Germany", "mission": "sar", "sat_type": "SAR Interferometry", "wiki": "https://en.wikipedia.org/wiki/TanDEM-X"}), - ("PAZ", {"country": "Spain", "mission": "sar", "sat_type": "SAR Imaging", "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)"}), - ("WORLDVIEW", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar High-Res", "wiki": "https://en.wikipedia.org/wiki/WorldView-3"}), - ("GEOEYE", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Maxar Imaging", "wiki": "https://en.wikipedia.org/wiki/GeoEye-1"}), - ("PLEIADES", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Imaging", "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)"}), - ("SPOT", {"country": "France", "mission": "commercial_imaging", "sat_type": "Airbus Medium-Res", "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)"}), - ("PLANET", {"country": "USA", "mission": "commercial_imaging", "sat_type": "PlanetScope", "wiki": "https://en.wikipedia.org/wiki/Planet_Labs"}), - ("SKYSAT", {"country": "USA", "mission": "commercial_imaging", "sat_type": "Planet Video", "wiki": "https://en.wikipedia.org/wiki/SkySat"}), - ("BLACKSKY", {"country": "USA", "mission": "commercial_imaging", "sat_type": "BlackSky Imaging", "wiki": "https://en.wikipedia.org/wiki/BlackSky"}), - ("NROL", {"country": "USA", "mission": "sigint", "sat_type": "Classified NRO", "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office"}), - ("MENTOR", {"country": "USA", "mission": "sigint", "sat_type": "SIGINT / ELINT", "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)"}), - ("LUCH", {"country": "Russia", "mission": "sigint", "sat_type": "Relay / SIGINT", "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)"}), - ("SHIJIAN", {"country": "China", "mission": "sigint", "sat_type": "ELINT / Tech Demo", "wiki": "https://en.wikipedia.org/wiki/Shijian"}), - ("NAVSTAR", {"country": "USA", "mission": "navigation", "sat_type": "GPS", "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks"}), - ("GLONASS", {"country": "Russia", "mission": "navigation", "sat_type": "GLONASS", "wiki": "https://en.wikipedia.org/wiki/GLONASS"}), - ("BEIDOU", {"country": "China", "mission": "navigation", "sat_type": "BeiDou", "wiki": "https://en.wikipedia.org/wiki/BeiDou"}), - ("GALILEO", {"country": "EU", "mission": "navigation", "sat_type": "Galileo", "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)"}), - ("SBIRS", {"country": "USA", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System"}), - ("TUNDRA", {"country": "Russia", "mission": "early_warning", "sat_type": "Missile Warning", "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)"}), - ("ISS", {"country": "Intl", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/International_Space_Station"}), - ("TIANGONG", {"country": "China", "mission": "space_station", "sat_type": "Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}), - ("CSS", {"country": "China", "mission": "space_station", "sat_type": "Chinese Space Station", "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station"}), - # Russian military — COSMOS covers the bulk of active Russian military/SIGINT satellites - ("COSMOS", {"country": "Russia", "mission": "military_recon", "sat_type": "Russian Military / COSMOS", "wiki": "https://en.wikipedia.org/wiki/Kosmos_(satellite)"}), - # US military communications - ("WGS", {"country": "USA", "mission": "sigint", "sat_type": "Wideband Global SATCOM", "wiki": "https://en.wikipedia.org/wiki/Wideband_Global_SATCOM"}), - ("AEHF", {"country": "USA", "mission": "sigint", "sat_type": "Advanced EHF MILSATCOM", "wiki": "https://en.wikipedia.org/wiki/Advanced_Extremely_High_Frequency"}), - ("MUOS", {"country": "USA", "mission": "sigint", "sat_type": "Mobile User Objective System", "wiki": "https://en.wikipedia.org/wiki/Mobile_User_Objective_System"}), - # EU Earth observation - ("SENTINEL", {"country": "EU", "mission": "commercial_imaging", "sat_type": "ESA Copernicus", "wiki": "https://en.wikipedia.org/wiki/Sentinel_(satellite)"}), + ( + "USA 224", + { + "country": "USA", + "mission": "military_recon", + "sat_type": "KH-11 Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN", + }, + ), + ( + "USA 245", + { + "country": "USA", + "mission": "military_recon", + "sat_type": "KH-11 Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN", + }, + ), + ( + "USA 290", + { + "country": "USA", + "mission": "military_recon", + "sat_type": "KH-11 Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN", + }, + ), + ( + "USA 314", + { + "country": "USA", + "mission": "military_recon", + "sat_type": "KH-11 Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN", + }, + ), + ( + "USA 338", + { + "country": "USA", + "mission": "military_recon", + "sat_type": "Keyhole Successor", + "wiki": "https://en.wikipedia.org/wiki/KH-11_KENNEN", + }, + ), + ( + "TOPAZ", + { + "country": "Russia", + "mission": "military_recon", + "sat_type": "Optical Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)", + }, + ), + ( + "PERSONA", + { + "country": "Russia", + "mission": "military_recon", + "sat_type": "Optical Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/Persona_(satellite)", + }, + ), + ( + "KONDOR", + { + "country": "Russia", + "mission": "military_sar", + "sat_type": "SAR Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/Kondor_(satellite)", + }, + ), + ( + "BARS-M", + { + "country": "Russia", + "mission": "military_recon", + "sat_type": "Mapping Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/Bars-M", + }, + ), + ( + "YAOGAN", + { + "country": "China", + "mission": "military_recon", + "sat_type": "Remote Sensing / ELINT", + "wiki": "https://en.wikipedia.org/wiki/Yaogan", + }, + ), + ( + "GAOFEN", + { + "country": "China", + "mission": "military_recon", + "sat_type": "High-Res Imaging", + "wiki": "https://en.wikipedia.org/wiki/Gaofen", + }, + ), + ( + "JILIN", + { + "country": "China", + "mission": "commercial_imaging", + "sat_type": "Video / Imaging", + "wiki": "https://en.wikipedia.org/wiki/Jilin-1", + }, + ), + ( + "OFEK", + { + "country": "Israel", + "mission": "military_recon", + "sat_type": "Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/Ofeq", + }, + ), + ( + "CSO", + { + "country": "France", + "mission": "military_recon", + "sat_type": "Optical Reconnaissance", + "wiki": "https://en.wikipedia.org/wiki/CSO_(satellite)", + }, + ), + ( + "IGS", + { + "country": "Japan", + "mission": "military_recon", + "sat_type": "Intelligence Gathering", + "wiki": "https://en.wikipedia.org/wiki/Information_Gathering_Satellite", + }, + ), + ( + "CAPELLA", + { + "country": "USA", + "mission": "sar", + "sat_type": "SAR Imaging", + "wiki": "https://en.wikipedia.org/wiki/Capella_Space", + }, + ), + ( + "ICEYE", + { + "country": "Finland", + "mission": "sar", + "sat_type": "SAR Microsatellite", + "wiki": "https://en.wikipedia.org/wiki/ICEYE", + }, + ), + ( + "COSMO-SKYMED", + { + "country": "Italy", + "mission": "sar", + "sat_type": "SAR Constellation", + "wiki": "https://en.wikipedia.org/wiki/COSMO-SkyMed", + }, + ), + ( + "TANDEM", + { + "country": "Germany", + "mission": "sar", + "sat_type": "SAR Interferometry", + "wiki": "https://en.wikipedia.org/wiki/TanDEM-X", + }, + ), + ( + "PAZ", + { + "country": "Spain", + "mission": "sar", + "sat_type": "SAR Imaging", + "wiki": "https://en.wikipedia.org/wiki/PAZ_(satellite)", + }, + ), + ( + "WORLDVIEW", + { + "country": "USA", + "mission": "commercial_imaging", + "sat_type": "Maxar High-Res", + "wiki": "https://en.wikipedia.org/wiki/WorldView-3", + }, + ), + ( + "GEOEYE", + { + "country": "USA", + "mission": "commercial_imaging", + "sat_type": "Maxar Imaging", + "wiki": "https://en.wikipedia.org/wiki/GeoEye-1", + }, + ), + ( + "PLEIADES", + { + "country": "France", + "mission": "commercial_imaging", + "sat_type": "Airbus Imaging", + "wiki": "https://en.wikipedia.org/wiki/Pl%C3%A9iades_(satellite)", + }, + ), + ( + "SPOT", + { + "country": "France", + "mission": "commercial_imaging", + "sat_type": "Airbus Medium-Res", + "wiki": "https://en.wikipedia.org/wiki/SPOT_(satellite)", + }, + ), + ( + "PLANET", + { + "country": "USA", + "mission": "commercial_imaging", + "sat_type": "PlanetScope", + "wiki": "https://en.wikipedia.org/wiki/Planet_Labs", + }, + ), + ( + "SKYSAT", + { + "country": "USA", + "mission": "commercial_imaging", + "sat_type": "Planet Video", + "wiki": "https://en.wikipedia.org/wiki/SkySat", + }, + ), + ( + "BLACKSKY", + { + "country": "USA", + "mission": "commercial_imaging", + "sat_type": "BlackSky Imaging", + "wiki": "https://en.wikipedia.org/wiki/BlackSky", + }, + ), + ( + "NROL", + { + "country": "USA", + "mission": "sigint", + "sat_type": "Classified NRO", + "wiki": "https://en.wikipedia.org/wiki/National_Reconnaissance_Office", + }, + ), + ( + "MENTOR", + { + "country": "USA", + "mission": "sigint", + "sat_type": "SIGINT / ELINT", + "wiki": "https://en.wikipedia.org/wiki/Mentor_(satellite)", + }, + ), + ( + "LUCH", + { + "country": "Russia", + "mission": "sigint", + "sat_type": "Relay / SIGINT", + "wiki": "https://en.wikipedia.org/wiki/Luch_(satellite)", + }, + ), + ( + "SHIJIAN", + { + "country": "China", + "mission": "sigint", + "sat_type": "ELINT / Tech Demo", + "wiki": "https://en.wikipedia.org/wiki/Shijian", + }, + ), + ( + "NAVSTAR", + { + "country": "USA", + "mission": "navigation", + "sat_type": "GPS", + "wiki": "https://en.wikipedia.org/wiki/GPS_satellite_blocks", + }, + ), + ( + "GLONASS", + { + "country": "Russia", + "mission": "navigation", + "sat_type": "GLONASS", + "wiki": "https://en.wikipedia.org/wiki/GLONASS", + }, + ), + ( + "BEIDOU", + { + "country": "China", + "mission": "navigation", + "sat_type": "BeiDou", + "wiki": "https://en.wikipedia.org/wiki/BeiDou", + }, + ), + ( + "GALILEO", + { + "country": "EU", + "mission": "navigation", + "sat_type": "Galileo", + "wiki": "https://en.wikipedia.org/wiki/Galileo_(satellite_navigation)", + }, + ), + ( + "SBIRS", + { + "country": "USA", + "mission": "early_warning", + "sat_type": "Missile Warning", + "wiki": "https://en.wikipedia.org/wiki/Space-Based_Infrared_System", + }, + ), + ( + "TUNDRA", + { + "country": "Russia", + "mission": "early_warning", + "sat_type": "Missile Warning", + "wiki": "https://en.wikipedia.org/wiki/Tundra_(satellite)", + }, + ), + ( + "ISS", + { + "country": "Intl", + "mission": "space_station", + "sat_type": "Space Station", + "wiki": "https://en.wikipedia.org/wiki/International_Space_Station", + }, + ), + ( + "TIANGONG", + { + "country": "China", + "mission": "space_station", + "sat_type": "Space Station", + "wiki": "https://en.wikipedia.org/wiki/Tiangong_space_station", + }, + ), ] @@ -154,7 +467,7 @@ def _parse_tle_to_gp(name, norad_id, line1, line2): if bstar_str: mantissa = float(bstar_str[:-2]) / 1e5 exponent = int(bstar_str[-2:]) - bstar = mantissa * (10 ** exponent) + bstar = mantissa * (10**exponent) else: bstar = 0.0 epoch_yr = int(line1[18:20]) @@ -206,17 +519,50 @@ def _fetch_satellites_from_tle_api(): seen_ids.add(sat_id) all_results.append(gp) time.sleep(1) # Polite delay between requests - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.debug(f"TLE fallback search '{term}' failed: {e}") return all_results def fetch_satellites(): + from services.fetchers._store import is_any_active + + if not is_any_active("satellites"): + return sats = [] try: now_ts = time.time() - if _sat_gp_cache["data"] is None or (now_ts - _sat_gp_cache["last_fetch"]) > _CELESTRAK_FETCH_INTERVAL: + + # On first call, try disk cache before hitting CelesTrak + if _sat_gp_cache["data"] is None: + disk_data = _load_sat_cache() + if disk_data: + import os + + cache_mtime = ( + os.path.getmtime(str(_SAT_CACHE_PATH)) if _SAT_CACHE_PATH.exists() else 0 + ) + _sat_gp_cache["data"] = disk_data + _sat_gp_cache["last_fetch"] = cache_mtime # real fetch time so 24h check works + _sat_gp_cache["source"] = "disk_cache" + logger.info( + f"Satellites: Bootstrapped from disk cache ({len(disk_data)} records, " + f"{(now_ts - cache_mtime) / 3600:.1f}h old)" + ) + + if ( + _sat_gp_cache["data"] is None + or (now_ts - _sat_gp_cache["last_fetch"]) > _CELESTRAK_FETCH_INTERVAL + ): gp_urls = [ "https://celestrak.org/NORAD/elements/gp.php?GROUP=active&FORMAT=json", "https://celestrak.com/NORAD/elements/gp.php?GROUP=active&FORMAT=json", @@ -232,7 +578,9 @@ def fetch_satellites(): if response.status_code == 304: # Data unchanged — reset timer without re-downloading _sat_gp_cache["last_fetch"] = now_ts - logger.info(f"Satellites: CelesTrak returned 304 Not Modified (data unchanged)") + logger.info( + f"Satellites: CelesTrak returned 304 Not Modified (data unchanged)" + ) break if response.status_code == 200: gp_data = response.json() @@ -241,14 +589,24 @@ def fetch_satellites(): _sat_gp_cache["last_fetch"] = now_ts _sat_gp_cache["source"] = "celestrak" # Store Last-Modified header for future conditional requests - if hasattr(response, 'headers'): + if hasattr(response, "headers"): lm = response.headers.get("Last-Modified") if lm: _sat_gp_cache["last_modified"] = lm _save_sat_cache(gp_data) - logger.info(f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak") + logger.info( + f"Satellites: Downloaded {len(gp_data)} GP records from CelesTrak" + ) break - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.warning(f"Satellites: Failed to fetch from {url}: {e}") continue @@ -261,8 +619,17 @@ def fetch_satellites(): _sat_gp_cache["last_fetch"] = now_ts _sat_gp_cache["source"] = "tle_api" _save_sat_cache(fallback_data) - logger.info(f"Satellites: Got {len(fallback_data)} records from TLE fallback API") - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: + logger.info( + f"Satellites: Got {len(fallback_data)} records from TLE fallback API" + ) + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + OSError, + ) as e: logger.error(f"Satellites: TLE fallback also failed: {e}") if _sat_gp_cache["data"] is None: @@ -279,9 +646,14 @@ def fetch_satellites(): latest_data["satellites"] = sats return - if _sat_classified_cache["gp_fetch_ts"] == _sat_gp_cache["last_fetch"] and _sat_classified_cache["data"]: + if ( + _sat_classified_cache["gp_fetch_ts"] == _sat_gp_cache["last_fetch"] + and _sat_classified_cache["data"] + ): classified = _sat_classified_cache["data"] - logger.info(f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)") + logger.info( + f"Satellites: Using cached classification ({len(classified)} sats, TLEs unchanged)" + ) else: classified = [] for sat in data: @@ -309,41 +681,57 @@ def fetch_satellites(): classified.append(entry) _sat_classified_cache["data"] = classified _sat_classified_cache["gp_fetch_ts"] = _sat_gp_cache["last_fetch"] - logger.info(f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog") + logger.info( + f"Satellites: {len(classified)} intel-classified out of {len(data)} total in catalog" + ) all_sats = classified now = datetime.utcnow() - jd, fr = jday(now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6) + jd, fr = jday( + now.year, now.month, now.day, now.hour, now.minute, now.second + now.microsecond / 1e6 + ) for s in all_sats: try: - mean_motion = s.get('MEAN_MOTION') - ecc = s.get('ECCENTRICITY') - incl = s.get('INCLINATION') - raan = s.get('RA_OF_ASC_NODE') - argp = s.get('ARG_OF_PERICENTER') - ma = s.get('MEAN_ANOMALY') - bstar = s.get('BSTAR', 0) - epoch_str = s.get('EPOCH') - norad_id = s.get('id', 0) + mean_motion = s.get("MEAN_MOTION") + ecc = s.get("ECCENTRICITY") + incl = s.get("INCLINATION") + raan = s.get("RA_OF_ASC_NODE") + argp = s.get("ARG_OF_PERICENTER") + ma = s.get("MEAN_ANOMALY") + bstar = s.get("BSTAR", 0) + epoch_str = s.get("EPOCH") + norad_id = s.get("id", 0) if mean_motion is None or ecc is None or incl is None: continue - epoch_dt = datetime.strptime(epoch_str[:19], '%Y-%m-%dT%H:%M:%S') - epoch_jd, epoch_fr = jday(epoch_dt.year, epoch_dt.month, epoch_dt.day, - epoch_dt.hour, epoch_dt.minute, epoch_dt.second) + epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S") + epoch_jd, epoch_fr = jday( + epoch_dt.year, + epoch_dt.month, + epoch_dt.day, + epoch_dt.hour, + epoch_dt.minute, + epoch_dt.second, + ) sat_obj = Satrec() sat_obj.sgp4init( - WGS72, 'i', norad_id, + WGS72, + "i", + norad_id, (epoch_jd + epoch_fr) - 2433281.5, - bstar, 0.0, 0.0, ecc, - math.radians(argp), math.radians(incl), + bstar, + 0.0, + 0.0, + ecc, + math.radians(argp), + math.radians(incl), math.radians(ma), mean_motion * 2 * math.pi / 1440.0, - math.radians(raan) + math.radians(raan), ) e, r, v = sat_obj.sgp4(jd, fr) @@ -353,13 +741,13 @@ def fetch_satellites(): x, y, z = r gmst = _gmst(jd + fr) lng_rad = math.atan2(y, x) - gmst - lat_rad = math.atan2(z, math.sqrt(x*x + y*y)) - alt_km = math.sqrt(x*x + y*y + z*z) - 6371.0 + lat_rad = math.atan2(z, math.sqrt(x * x + y * y)) + alt_km = math.sqrt(x * x + y * y + z * z) - 6371.0 - s['lat'] = round(math.degrees(lat_rad), 4) + s["lat"] = round(math.degrees(lat_rad), 4) lng_deg = math.degrees(lng_rad) % 360 - s['lng'] = round(lng_deg - 360 if lng_deg > 180 else lng_deg, 4) - s['alt_km'] = round(alt_km, 1) + s["lng"] = round(lng_deg - 360 if lng_deg > 180 else lng_deg, 4) + s["alt_km"] = round(alt_km, 1) vx, vy, vz = v omega_e = 7.2921159e-5 @@ -373,23 +761,40 @@ def fetch_satellites(): v_east = -sin_lng * vx_g + cos_lng * vy_g v_north = -sin_lat * cos_lng * vx_g - sin_lat * sin_lng * vy_g + cos_lat * vz_g ground_speed_kms = math.sqrt(v_east**2 + v_north**2) - s['speed_knots'] = round(ground_speed_kms * 1943.84, 1) + s["speed_knots"] = round(ground_speed_kms * 1943.84, 1) heading_rad = math.atan2(v_east, v_north) - s['heading'] = round(math.degrees(heading_rad) % 360, 1) - sat_name = s.get('name', '') - usa_match = re.search(r'USA[\s\-]*(\d+)', sat_name) + s["heading"] = round(math.degrees(heading_rad) % 360, 1) + sat_name = s.get("name", "") + usa_match = re.search(r"USA[\s\-]*(\d+)", sat_name) if usa_match: - s['wiki'] = f"https://en.wikipedia.org/wiki/USA-{usa_match.group(1)}" - for k in ('MEAN_MOTION', 'ECCENTRICITY', 'INCLINATION', - 'RA_OF_ASC_NODE', 'ARG_OF_PERICENTER', 'MEAN_ANOMALY', - 'BSTAR', 'EPOCH', 'tle1', 'tle2'): + s["wiki"] = f"https://en.wikipedia.org/wiki/USA-{usa_match.group(1)}" + for k in ( + "MEAN_MOTION", + "ECCENTRICITY", + "INCLINATION", + "RA_OF_ASC_NODE", + "ARG_OF_PERICENTER", + "MEAN_ANOMALY", + "BSTAR", + "EPOCH", + "tle1", + "tle2", + ): s.pop(k, None) sats.append(s) except (ValueError, TypeError, KeyError, AttributeError, ZeroDivisionError): continue logger.info(f"Satellites: {len(classified)} classified, {len(sats)} positioned") - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, json.JSONDecodeError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + json.JSONDecodeError, + OSError, + ) as e: logger.error(f"Error fetching satellites: {e}") if sats: with _data_lock: diff --git a/backend/services/fetchers/sigint.py b/backend/services/fetchers/sigint.py new file mode 100644 index 00000000..f6a2c7a4 --- /dev/null +++ b/backend/services/fetchers/sigint.py @@ -0,0 +1,102 @@ +"""SIGINT fetcher — pulls latest signals from the SIGINT Grid into latest_data. + +Merges live MQTT signals with cached Meshtastic map API nodes. +Live MQTT signals always take priority (fresher) — API nodes fill in the gaps +for the thousands of nodes our MQTT listener hasn't heard yet. +""" + +import logging +from services.fetchers._store import latest_data, _data_lock, _mark_fresh + +logger = logging.getLogger("services.data_fetcher") + + +def _merge_sigint_snapshot( + live_signals: list[dict], + api_nodes: list[dict], +) -> list[dict]: + """Merge live bridge signals with cached Meshtastic map nodes. + + Live Meshtastic observations always win over map/API nodes for the same callsign + because they include fresher region/channel metadata. + """ + + merged = list(live_signals) + live_callsigns = {s["callsign"] for s in merged if s.get("source") == "meshtastic"} + for node in api_nodes: + if node.get("callsign") in live_callsigns: + continue + merged.append(node) + merged.sort(key=lambda item: str(item.get("timestamp", "") or ""), reverse=True) + return merged + + +def _sigint_totals(signals: list[dict]) -> dict[str, int]: + totals = { + "total": len(signals), + "meshtastic": 0, + "meshtastic_live": 0, + "meshtastic_map": 0, + "aprs": 0, + "js8call": 0, + } + for sig in signals: + source = str(sig.get("source", "") or "").lower() + if source == "meshtastic": + totals["meshtastic"] += 1 + if bool(sig.get("from_api")): + totals["meshtastic_map"] += 1 + else: + totals["meshtastic_live"] += 1 + elif source == "aprs": + totals["aprs"] += 1 + elif source == "js8call": + totals["js8call"] += 1 + return totals + + +def build_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]: + """Build the current merged SIGINT snapshot without hitting the network.""" + + from services.sigint_bridge import sigint_grid + + live_signals = sigint_grid.get_all_signals() + with _data_lock: + api_nodes = list(latest_data.get("meshtastic_map_nodes", [])) + merged = _merge_sigint_snapshot(live_signals, api_nodes) + channel_stats = sigint_grid.get_mesh_channel_stats(api_nodes or None) + totals = _sigint_totals(merged) + return merged, channel_stats, totals + + +def refresh_sigint_snapshot() -> tuple[list[dict], dict[str, object], dict[str, int]]: + """Refresh latest_data SIGINT state from current bridge + cache state.""" + + signals, channel_stats, totals = build_sigint_snapshot() + with _data_lock: + latest_data["sigint"] = signals + latest_data["mesh_channel_stats"] = channel_stats + latest_data["sigint_totals"] = totals + _mark_fresh("sigint") + return signals, channel_stats, totals + + +def fetch_sigint(): + """Fetch all signals from the SIGINT Grid, merge with Meshtastic map nodes.""" + from services.fetchers._store import is_any_active + + if not is_any_active("sigint_meshtastic", "sigint_aprs"): + return + from services.sigint_bridge import sigint_grid + + # Start bridges on first call (idempotent) + sigint_grid.start() + + signals, channel_stats, totals = refresh_sigint_snapshot() + + status = sigint_grid.status + logger.info( + f"SIGINT: {len(signals)} signals " + f"(APRS:{status['aprs']} MESH:{status['meshtastic']} " + f"JS8:{status['js8call']} MAP:{totals['meshtastic_map']})" + ) diff --git a/backend/services/fetchers/trains.py b/backend/services/fetchers/trains.py new file mode 100644 index 00000000..7f8528bb --- /dev/null +++ b/backend/services/fetchers/trains.py @@ -0,0 +1,457 @@ +"""Train tracking fetchers with normalized metadata and non-redundant merging.""" + +from __future__ import annotations + +import logging +import math +from collections.abc import Callable +from datetime import datetime, timezone + +from services.fetchers._store import _data_lock, _mark_fresh, latest_data +from services.network_utils import fetch_with_curl + +logger = logging.getLogger(__name__) + +_EARTH_RADIUS_KM = 6371.0 +_MERGE_DISTANCE_KM = 5.0 +_MAX_INFERRED_SPEED_KMH = 350.0 +_TRACK_CACHE_TTL_S = 6 * 60 * 60 + +_SOURCE_METADATA: dict[str, dict[str, object]] = { + "amtrak": { + "source_label": "Amtraker", + "operator": "Amtrak", + "country": "US", + "telemetry_quality": "aggregated", + "priority": 70, + }, + "digitraffic": { + "source_label": "Digitraffic Finland", + "operator": "Finnish Rail", + "country": "FI", + "telemetry_quality": "official", + "priority": 100, + }, + # Future slots so better official feeds can be merged without changing the + # rest of the train pipeline or duplicating map entities. + "networkrail": { + "source_label": "Network Rail Open Data", + "operator": "Network Rail", + "country": "GB", + "telemetry_quality": "official", + "priority": 98, + }, + "dbcargo": { + "source_label": "DB Cargo link2rail", + "operator": "DB Cargo", + "country": "DE", + "telemetry_quality": "commercial", + "priority": 96, + }, + "railinc": { + "source_label": "Railinc RailSight", + "operator": "Railinc", + "country": "US", + "telemetry_quality": "commercial", + "priority": 97, + }, + "sncf": { + "source_label": "SNCF Open Data", + "operator": "SNCF", + "country": "FR", + "telemetry_quality": "official", + "priority": 94, + }, +} + +_TRAIN_TRACK_CACHE: dict[str, dict[str, float]] = {} + + +def _safe_float(value) -> float | None: + try: + if value is None or value == "": + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _parse_observed_at(value) -> float | None: + if value is None or value == "": + return None + if isinstance(value, (int, float)): + raw = float(value) + return raw / 1000.0 if raw > 1_000_000_000_000 else raw + if not isinstance(value, str): + return None + text = value.strip() + if not text: + return None + if text.endswith("Z"): + text = f"{text[:-1]}+00:00" + try: + return datetime.fromisoformat(text).timestamp() + except ValueError: + return None + + +def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + lat1_rad, lon1_rad = math.radians(lat1), math.radians(lon1) + lat2_rad, lon2_rad = math.radians(lat2), math.radians(lon2) + dlat = lat2_rad - lat1_rad + dlon = lon2_rad - lon1_rad + a = ( + math.sin(dlat / 2.0) ** 2 + + math.cos(lat1_rad) * math.cos(lat2_rad) * math.sin(dlon / 2.0) ** 2 + ) + return 2.0 * _EARTH_RADIUS_KM * math.asin(math.sqrt(a)) + + +def _bearing_degrees(lat1: float, lon1: float, lat2: float, lon2: float) -> float | None: + if lat1 == lat2 and lon1 == lon2: + return None + lat1_rad, lat2_rad = math.radians(lat1), math.radians(lat2) + dlon_rad = math.radians(lon2 - lon1) + y = math.sin(dlon_rad) * math.cos(lat2_rad) + x = ( + math.cos(lat1_rad) * math.sin(lat2_rad) + - math.sin(lat1_rad) * math.cos(lat2_rad) * math.cos(dlon_rad) + ) + return (math.degrees(math.atan2(y, x)) + 360.0) % 360.0 + + +def _source_meta(source: str) -> dict[str, object]: + return dict(_SOURCE_METADATA.get(source, {})) + + +def _normalize_train( + *, + source: str, + raw_id: str, + number: str, + lat, + lng, + name: str = "", + status: str = "Active", + route: str = "", + speed_kmh=None, + heading=None, + operator: str | None = None, + country: str | None = None, + source_label: str | None = None, + telemetry_quality: str | None = None, + observed_at=None, +) -> dict | None: + lat_f = _safe_float(lat) + lng_f = _safe_float(lng) + if lat_f is None or lng_f is None: + return None + if not (-90.0 <= lat_f <= 90.0 and -180.0 <= lng_f <= 180.0): + return None + + number_text = str(number or "").strip() + meta = _source_meta(source) + observed_ts = _parse_observed_at(observed_at) or datetime.now(timezone.utc).timestamp() + speed_f = _safe_float(speed_kmh) + heading_f = _safe_float(heading) + normalized = { + "id": str(raw_id or f"{source}-{number_text or 'unknown'}"), + "name": str(name or f"Train {number_text or '?'}").strip(), + "number": number_text, + "source": source, + "source_label": str(source_label or meta.get("source_label") or source.upper()), + "operator": str(operator or meta.get("operator") or "").strip(), + "country": str(country or meta.get("country") or "").strip(), + "telemetry_quality": str( + telemetry_quality or meta.get("telemetry_quality") or "unknown" + ).strip(), + "lat": lat_f, + "lng": lng_f, + "speed_kmh": speed_f, + "heading": heading_f, + "status": str(status or "Active").strip(), + "route": str(route or "").strip(), + "_source_priority": int(meta.get("priority") or 0), + "_observed_ts": observed_ts, + } + _apply_motion_estimates(normalized) + return normalized + + +def _prune_track_cache(now_ts: float) -> None: + stale_before = now_ts - _TRACK_CACHE_TTL_S + stale_ids = [train_id for train_id, entry in _TRAIN_TRACK_CACHE.items() if entry["ts"] < stale_before] + for train_id in stale_ids: + _TRAIN_TRACK_CACHE.pop(train_id, None) + + +def _apply_motion_estimates(train: dict) -> None: + train_id = str(train.get("id") or "") + if not train_id: + return + now_ts = float(train.get("_observed_ts") or datetime.now(timezone.utc).timestamp()) + _prune_track_cache(now_ts) + previous = _TRAIN_TRACK_CACHE.get(train_id) + if previous: + dt_s = now_ts - previous["ts"] + if 5.0 <= dt_s <= 15.0 * 60.0: + distance_km = _haversine_km( + float(previous["lat"]), + float(previous["lng"]), + float(train["lat"]), + float(train["lng"]), + ) + if 0.02 <= distance_km <= (_MAX_INFERRED_SPEED_KMH * (dt_s / 3600.0)): + if train.get("speed_kmh") is None: + inferred_speed = distance_km / (dt_s / 3600.0) + train["speed_kmh"] = round(min(inferred_speed, _MAX_INFERRED_SPEED_KMH), 1) + if train.get("heading") is None: + inferred_heading = _bearing_degrees( + float(previous["lat"]), + float(previous["lng"]), + float(train["lat"]), + float(train["lng"]), + ) + if inferred_heading is not None: + train["heading"] = round(inferred_heading, 1) + + _TRAIN_TRACK_CACHE[train_id] = { + "lat": float(train["lat"]), + "lng": float(train["lng"]), + "ts": now_ts, + } + + +def _train_merge_key(train: dict) -> str: + operator = str(train.get("operator") or "").strip().lower() + country = str(train.get("country") or "").strip().lower() + number = str(train.get("number") or "").strip().lower() + if operator and number: + return f"{country}|{operator}|{number}" + return f"{str(train.get('source') or '').lower()}|{str(train.get('id') or '').lower()}" + + +def _train_completeness(train: dict) -> tuple[int, int, int]: + return ( + 1 if train.get("speed_kmh") is not None else 0, + 1 if train.get("heading") is not None else 0, + 1 if train.get("route") else 0, + ) + + +def _should_merge(existing: dict, candidate: dict) -> bool: + if _train_merge_key(existing) != _train_merge_key(candidate): + return False + return _haversine_km( + float(existing["lat"]), + float(existing["lng"]), + float(candidate["lat"]), + float(candidate["lng"]), + ) <= _MERGE_DISTANCE_KM + + +def _merge_train_pair(existing: dict, candidate: dict) -> dict: + existing_priority = int(existing.get("_source_priority") or 0) + candidate_priority = int(candidate.get("_source_priority") or 0) + existing_score = (existing_priority, _train_completeness(existing)) + candidate_score = (candidate_priority, _train_completeness(candidate)) + primary = candidate if candidate_score > existing_score else existing + secondary = existing if primary is candidate else candidate + merged = dict(primary) + + for field in ( + "speed_kmh", + "heading", + "route", + "status", + "operator", + "country", + "source_label", + "telemetry_quality", + ): + if merged.get(field) in (None, "", "Active"): + replacement = secondary.get(field) + if replacement not in (None, ""): + merged[field] = replacement + + if primary is not candidate and float(candidate.get("_observed_ts") or 0) > float( + primary.get("_observed_ts") or 0 + ): + merged["lat"] = candidate["lat"] + merged["lng"] = candidate["lng"] + merged["_observed_ts"] = candidate["_observed_ts"] + return merged + + +def _merge_nonredundant_trains(*sources: list[dict]) -> list[dict]: + merged: list[dict] = [] + for source_trains in sources: + for train in source_trains: + exact_match = next( + ( + idx + for idx, existing in enumerate(merged) + if existing.get("source") == train.get("source") + and existing.get("id") == train.get("id") + ), + None, + ) + if exact_match is not None: + merged[exact_match] = _merge_train_pair(merged[exact_match], train) + continue + + merged_idx = next( + (idx for idx, existing in enumerate(merged) if _should_merge(existing, train)), + None, + ) + if merged_idx is not None: + merged[merged_idx] = _merge_train_pair(merged[merged_idx], train) + continue + merged.append(train) + + merged.sort( + key=lambda train: ( + str(train.get("country") or ""), + str(train.get("operator") or ""), + str(train.get("number") or ""), + str(train.get("id") or ""), + ) + ) + for train in merged: + train.pop("_source_priority", None) + train.pop("_observed_ts", None) + return merged + + +def _fetch_amtraker() -> list[dict]: + """Fetch all active Amtrak trains from the Amtraker API.""" + try: + resp = fetch_with_curl( + "https://api.amtraker.com/v3/trains", + timeout=20, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/136.0.0.0 Safari/537.36" + ), + "Accept": "application/json,text/plain,*/*", + "Referer": "https://www.amtraker.com/", + }, + ) + if resp.status_code != 200: + logger.warning("Amtraker returned %s", resp.status_code) + return [] + raw = resp.json() + trains: list[dict] = [] + for train_num, variants in raw.items(): + if not isinstance(variants, list): + continue + for item in variants: + normalized = _normalize_train( + source="amtrak", + raw_id=f"AMTK-{item.get('trainID', train_num)}", + name=item.get("routeName", f"Train {train_num}"), + number=str(item.get("trainNum", train_num) or train_num), + lat=item.get("lat"), + lng=item.get("lon"), + speed_kmh=item.get("velocity") or item.get("speed"), + heading=item.get("heading") or item.get("bearing"), + status=item.get("trainTimely") or "On Time", + route=item.get("routeName", ""), + observed_at=item.get("updatedAt") + or item.get("lastValTS") + or item.get("eventDT"), + ) + if normalized: + trains.append(normalized) + return trains + except Exception as exc: + logger.warning("Amtraker fetch error: %s", exc) + return [] + + +def _fetch_digitraffic() -> list[dict]: + """Fetch live train positions from Finnish DigiTraffic API.""" + try: + resp = fetch_with_curl( + "https://rata.digitraffic.fi/api/v1/train-locations/latest", + timeout=15, + headers={ + "Accept-Encoding": "gzip", + "User-Agent": "ShadowBroker-OSINT/1.0", + }, + ) + if resp.status_code != 200: + logger.warning("DigiTraffic returned %s", resp.status_code) + return [] + raw = resp.json() + trains: list[dict] = [] + for item in raw: + location = item.get("location", {}) + coords = location.get("coordinates") + if not coords or len(coords) < 2: + continue + lon, lat = coords[0], coords[1] + train_number = str(item.get("trainNumber", "") or "").strip() + route_bits = [ + str(item.get("departureStationShortCode") or "").strip(), + str(item.get("stationShortCode") or "").strip(), + ] + route = " -> ".join([bit for bit in route_bits if bit]) + train_type = str(item.get("trainType") or "").strip() + normalized = _normalize_train( + source="digitraffic", + raw_id=f"FIN-{train_number or len(trains)}", + name=f"{train_type} {train_number}".strip() or f"Train {train_number or '?'}", + number=train_number, + lat=lat, + lng=lon, + speed_kmh=item.get("speed"), + heading=item.get("heading"), + status="Active", + route=route, + observed_at=item.get("timestamp"), + ) + if normalized: + trains.append(normalized) + return trains + except Exception as exc: + logger.warning("DigiTraffic fetch error: %s", exc) + return [] + + +_TRAIN_FETCHERS: tuple[tuple[str, Callable[[], list[dict]]], ...] = ( + ("amtrak", _fetch_amtraker), + ("digitraffic", _fetch_digitraffic), +) + + +def fetch_trains(): + """Fetch trains from all configured sources and merge without duplicates.""" + with _data_lock: + existing_trains = list(latest_data.get("trains") or []) + source_batches: list[list[dict]] = [] + source_counts: list[str] = [] + for source_name, fetcher in _TRAIN_FETCHERS: + batch = fetcher() + source_batches.append(batch) + if batch: + source_counts.append(f"{source_name}:{len(batch)}") + + trains = _merge_nonredundant_trains(*source_batches) + if not trains and existing_trains: + logger.warning( + "Train refresh returned 0 records — preserving %s cached trains until the next successful poll", + len(existing_trains), + ) + trains = existing_trains + + with _data_lock: + latest_data["trains"] = trains + _mark_fresh("trains") + logger.info( + "Trains: %s total%s", + len(trains), + f" ({', '.join(source_counts)})" if source_counts else "", + ) diff --git a/backend/services/fetchers/ukraine_alerts.py b/backend/services/fetchers/ukraine_alerts.py new file mode 100644 index 00000000..2c28c9c5 --- /dev/null +++ b/backend/services/fetchers/ukraine_alerts.py @@ -0,0 +1,139 @@ +"""Ukraine air raid alerts via alerts.in.ua API. + +Polls active alerts every 2 minutes, matches to oblast boundary polygons, +and produces GeoJSON-style records for map rendering. + +Requires ALERTS_IN_UA_TOKEN env var (free registration at alerts.in.ua). +Gracefully skips if token is not set. +""" + +import json +import logging +import os +from pathlib import Path + +from services.network_utils import fetch_with_curl +from services.fetchers._store import latest_data, _data_lock, _mark_fresh +from services.fetchers.retry import with_retry + +logger = logging.getLogger(__name__) + +# ─── Alert type → color mapping ────────────────────────────────────────────── +ALERT_COLORS = { + "air_raid": "#ef4444", # red + "artillery_shelling": "#f97316", # orange + "urban_fights": "#eab308", # yellow + "chemical": "#a855f7", # purple + "nuclear": "#dc2626", # dark red +} + +# ─── Load oblast boundary polygons (once) ──────────────────────────────────── +_oblast_geojson = None + + +def _load_oblasts(): + global _oblast_geojson + if _oblast_geojson is not None: + return _oblast_geojson + + data_path = Path(__file__).resolve().parent.parent.parent / "data" / "ukraine_oblasts.geojson" + if not data_path.exists(): + logger.error(f"Ukraine oblasts GeoJSON not found at {data_path}") + _oblast_geojson = {} + return _oblast_geojson + + with open(data_path, "r", encoding="utf-8") as f: + _oblast_geojson = json.load(f) + + logger.info(f"Loaded {len(_oblast_geojson.get('features', []))} Ukraine oblast boundaries") + return _oblast_geojson + + +def _find_oblast_geometry(location_title: str): + """Find the polygon geometry for an oblast by matching Ukrainian name.""" + oblasts = _load_oblasts() + features = oblasts.get("features", []) + for feat in features: + props = feat.get("properties", {}) + name = props.get("name", "") + # Exact match on Ukrainian name (e.g. "Луганська область") + if name == location_title: + return feat.get("geometry"), props.get("name_en", "") + # Fuzzy: try partial match (alert may say "Київська область" but GeoJSON says "Київ") + for feat in features: + props = feat.get("properties", {}) + name = props.get("name", "") + if location_title in name or name in location_title: + return feat.get("geometry"), props.get("name_en", "") + return None, "" + + +# ─── Fetcher ───────────────────────────────────────────────────────────────── + +@with_retry(max_retries=1, base_delay=2) +def fetch_ukraine_air_raid_alerts(): + """Fetch active Ukraine air raid alerts from alerts.in.ua.""" + from services.fetchers._store import is_any_active + + if not is_any_active("ukraine_alerts"): + return + + token = os.environ.get("ALERTS_IN_UA_TOKEN", "") + if not token: + logger.debug("ALERTS_IN_UA_TOKEN not set, skipping Ukraine air raid alerts") + return + + alerts_out = [] + try: + url = f"https://api.alerts.in.ua/v1/alerts/active.json?token={token}" + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + } + response = fetch_with_curl(url, timeout=10, headers=headers) + + if response.status_code == 200: + data = response.json() + raw_alerts = data.get("alerts", []) + + for alert in raw_alerts: + loc_type = alert.get("location_type", "") + # Only render oblast-level alerts (not raion/city/hromada) + if loc_type != "oblast": + continue + + location_title = alert.get("location_title", "") + alert_type = alert.get("alert_type", "air_raid") + geometry, name_en = _find_oblast_geometry(location_title) + + if not geometry: + logger.debug(f"No geometry for oblast: {location_title}") + continue + + alerts_out.append({ + "id": alert.get("id", 0), + "alert_type": alert_type, + "location_title": location_title, + "location_uid": alert.get("location_uid", ""), + "name_en": name_en, + "started_at": alert.get("started_at", ""), + "color": ALERT_COLORS.get(alert_type, "#ef4444"), + "geometry": geometry, + }) + + logger.info(f"Ukraine alerts: {len(alerts_out)} active oblast-level alerts " + f"(from {len(raw_alerts)} total)") + elif response.status_code == 401: + logger.warning("alerts.in.ua returned 401 — check ALERTS_IN_UA_TOKEN") + elif response.status_code == 429: + logger.warning("alerts.in.ua rate-limited (429)") + else: + logger.warning(f"alerts.in.ua returned HTTP {response.status_code}") + + except (ConnectionError, TimeoutError, OSError, ValueError, KeyError, TypeError) as e: + logger.error(f"Error fetching Ukraine alerts: {e}") + + with _data_lock: + latest_data["ukraine_alerts"] = alerts_out + if alerts_out: + _mark_fresh("ukraine_alerts") diff --git a/backend/services/fetchers/unusual_whales.py b/backend/services/fetchers/unusual_whales.py new file mode 100644 index 00000000..97757b4e --- /dev/null +++ b/backend/services/fetchers/unusual_whales.py @@ -0,0 +1,76 @@ +"""Finnhub scheduled fetcher — congress trades, insider transactions, defense quotes. + +Runs on a 15-minute schedule and stores results in latest_data["unusual_whales"]. +Also updates latest_data["stocks"] with Finnhub quotes (replaces yfinance for defense tickers). +Falls back gracefully if no API key is configured. +""" + +import logging +from services.fetchers._store import latest_data, _data_lock, _mark_fresh +from services.fetchers.retry import with_retry + +logger = logging.getLogger(__name__) + + +@with_retry(max_retries=1, base_delay=2) +def fetch_unusual_whales(): + """Fetch congress trades, insider txns, and defense quotes from Finnhub.""" + import os + + if not os.environ.get("FINNHUB_API_KEY", "").strip(): + logger.debug("FINNHUB_API_KEY not set — skipping scheduled fetch.") + return + + from services.unusual_whales_connector import ( + fetch_congress_trades, + fetch_insider_transactions, + fetch_defense_quotes, + FinnhubConnectorError, + ) + + result: dict = {} + + # Defense stock quotes (also populates latest_data["stocks"]) + try: + quotes = fetch_defense_quotes() + if quotes: + result["quotes"] = quotes + # Mirror into stocks for backward compat with existing MarketsPanel fallback + with _data_lock: + latest_data["stocks"] = quotes + _mark_fresh("stocks") + except FinnhubConnectorError as e: + logger.warning(f"Finnhub quotes fetch failed: {e.detail}") + except Exception as e: + logger.warning(f"Finnhub quotes fetch error: {e}") + + # Congress trades + try: + congress = fetch_congress_trades() + result["congress_trades"] = congress.get("trades", []) + except FinnhubConnectorError as e: + logger.warning(f"Finnhub congress trades fetch failed: {e.detail}") + except Exception as e: + logger.warning(f"Finnhub congress trades fetch error: {e}") + + # Insider transactions + try: + insiders = fetch_insider_transactions() + result["insider_transactions"] = insiders.get("transactions", []) + except FinnhubConnectorError as e: + logger.warning(f"Finnhub insider fetch failed: {e.detail}") + except Exception as e: + logger.warning(f"Finnhub insider fetch error: {e}") + + if not result: + logger.warning("Finnhub update produced no data; keeping previous cache.") + return + + with _data_lock: + latest_data["unusual_whales"] = result + _mark_fresh("unusual_whales") + logger.info( + f"Finnhub updated: {len(result.get('congress_trades', []))} congress, " + f"{len(result.get('insider_transactions', []))} insider, " + f"{len(result.get('quotes', {}))} quotes" + ) diff --git a/backend/services/fetchers/yacht_alert.py b/backend/services/fetchers/yacht_alert.py index 1ea0917d..16837c91 100644 --- a/backend/services/fetchers/yacht_alert.py +++ b/backend/services/fetchers/yacht_alert.py @@ -1,4 +1,5 @@ """Yacht-Alert DB — load and enrich AIS vessels with tracked yacht metadata.""" + import os import json import logging @@ -26,7 +27,8 @@ def _load_yacht_alert_db(): global _YACHT_ALERT_DB json_path = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "data", "yacht_alert_db.json" + "data", + "yacht_alert_db.json", ) if not os.path.exists(json_path): logger.warning(f"Yacht-Alert DB not found at {json_path}") diff --git a/backend/services/geocode.py b/backend/services/geocode.py new file mode 100644 index 00000000..740dc366 --- /dev/null +++ b/backend/services/geocode.py @@ -0,0 +1,255 @@ +"""Geocoding proxy for Nominatim with caching and proper headers.""" + +from __future__ import annotations + +import json +import os +import time +import threading +from typing import Any, Dict, List +from pathlib import Path +from urllib.parse import urlencode + +from services.network_utils import fetch_with_curl +from services.fetchers.geo import cached_airports + +_CACHE_TTL_S = 900 +_CACHE_MAX = 1000 +_cache: Dict[str, Dict[str, Any]] = {} +_cache_lock = threading.Lock() +_local_search_cache: List[Dict[str, Any]] | None = None +_local_search_lock = threading.Lock() + +_USER_AGENT = os.environ.get( + "NOMINATIM_USER_AGENT", "ShadowBroker/1.0 (https://github.com/BigBodyCobain/Shadowbroker)" +) + + +def _get_cache(key: str): + now = time.time() + with _cache_lock: + entry = _cache.get(key) + if not entry: + return None + if now - entry["ts"] > _CACHE_TTL_S: + _cache.pop(key, None) + return None + return entry["value"] + + +def _set_cache(key: str, value): + with _cache_lock: + if len(_cache) >= _CACHE_MAX: + # Simple eviction: drop ~10% oldest keys + keys = list(_cache.keys())[: max(1, _CACHE_MAX // 10)] + for k in keys: + _cache.pop(k, None) + _cache[key] = {"ts": time.time(), "value": value} + + +def _load_local_search_cache() -> List[Dict[str, Any]]: + global _local_search_cache + with _local_search_lock: + if _local_search_cache is not None: + return _local_search_cache + + results: List[Dict[str, Any]] = [] + cache_path = Path(__file__).resolve().parents[1] / "data" / "geocode_cache.json" + try: + if cache_path.exists(): + raw = json.loads(cache_path.read_text(encoding="utf-8")) + if isinstance(raw, dict): + for label, coords in raw.items(): + if ( + isinstance(label, str) + and isinstance(coords, list) + and len(coords) == 2 + and all(isinstance(v, (int, float)) for v in coords) + ): + results.append( + { + "label": label, + "lat": float(coords[0]), + "lng": float(coords[1]), + } + ) + except Exception: + results = [] + + _local_search_cache = results + return _local_search_cache + + +def _search_local_fallback(query: str, limit: int) -> List[Dict[str, Any]]: + q = query.strip().lower() + if not q: + return [] + + matches: List[Dict[str, Any]] = [] + seen: set[tuple[float, float, str]] = set() + + for item in cached_airports: + haystacks = [ + str(item.get("name", "")).lower(), + str(item.get("iata", "")).lower(), + str(item.get("id", "")).lower(), + ] + if any(q in h for h in haystacks): + label = f'{item.get("name", "Airport")} ({item.get("iata", "")})' + key = (float(item["lat"]), float(item["lng"]), label) + if key not in seen: + seen.add(key) + matches.append( + { + "label": label, + "lat": float(item["lat"]), + "lng": float(item["lng"]), + } + ) + if len(matches) >= limit: + return matches + + for item in _load_local_search_cache(): + label = str(item.get("label", "")) + if q in label.lower(): + key = (float(item["lat"]), float(item["lng"]), label) + if key not in seen: + seen.add(key) + matches.append(item) + if len(matches) >= limit: + break + + return matches + + +def _reverse_geocode_offline(lat: float, lng: float) -> Dict[str, Any]: + try: + import reverse_geocoder as rg + + hit = rg.search((lat, lng), mode=1)[0] + city = hit.get("name") or "" + state = hit.get("admin1") or "" + country = hit.get("cc") or "" + parts = [city, state, country] + label = ", ".join([p for p in parts if p]) or "Unknown" + return {"label": label} + except Exception: + return {"label": "Unknown"} + + +def search_geocode(query: str, limit: int = 5, local_only: bool = False) -> List[Dict[str, Any]]: + q = query.strip() + if not q: + return [] + limit = max(1, min(int(limit or 5), 10)) + key = f"search:{q.lower()}:{limit}:{int(local_only)}" + cached = _get_cache(key) + if cached is not None: + return cached + if local_only: + results = _search_local_fallback(q, limit) + _set_cache(key, results) + return results + + params = urlencode({"q": q, "format": "json", "limit": str(limit)}) + url = f"https://nominatim.openstreetmap.org/search?{params}" + try: + res = fetch_with_curl( + url, + headers={ + "User-Agent": _USER_AGENT, + "Accept-Language": "en", + }, + timeout=6, + ) + except Exception: + results = _search_local_fallback(q, limit) + _set_cache(key, results) + return results + + results: List[Dict[str, Any]] = [] + if res and res.status_code == 200: + try: + data = res.json() or [] + for item in data: + try: + results.append( + { + "label": item.get("display_name"), + "lat": float(item.get("lat")), + "lng": float(item.get("lon")), + } + ) + except (TypeError, ValueError): + continue + except Exception: + results = [] + if not results: + results = _search_local_fallback(q, limit) + + _set_cache(key, results) + return results + + +def reverse_geocode(lat: float, lng: float, local_only: bool = False) -> Dict[str, Any]: + key = f"reverse:{lat:.4f},{lng:.4f}:{int(local_only)}" + cached = _get_cache(key) + if cached is not None: + return cached + if local_only: + payload = _reverse_geocode_offline(lat, lng) + _set_cache(key, payload) + return payload + + params = urlencode( + { + "lat": f"{lat}", + "lon": f"{lng}", + "format": "json", + "zoom": "10", + "addressdetails": "1", + } + ) + url = f"https://nominatim.openstreetmap.org/reverse?{params}" + try: + res = fetch_with_curl( + url, + headers={ + "User-Agent": _USER_AGENT, + "Accept-Language": "en", + }, + timeout=6, + ) + except Exception: + payload = _reverse_geocode_offline(lat, lng) + _set_cache(key, payload) + return payload + + label = "Unknown" + if res and res.status_code == 200: + try: + data = res.json() or {} + addr = data.get("address") or {} + city = ( + addr.get("city") + or addr.get("town") + or addr.get("village") + or addr.get("county") + or "" + ) + state = addr.get("state") or addr.get("region") or "" + country = addr.get("country") or "" + parts = [city, state, country] + label = ", ".join([p for p in parts if p]) or ( + data.get("display_name", "") or "Unknown" + ) + except Exception: + label = "Unknown" + if label == "Unknown": + payload = _reverse_geocode_offline(lat, lng) + _set_cache(key, payload) + return payload + + payload = {"label": label} + _set_cache(key, payload) + return payload diff --git a/backend/services/geopolitics.py b/backend/services/geopolitics.py index 32cf50fb..4bb466dc 100644 --- a/backend/services/geopolitics.py +++ b/backend/services/geopolitics.py @@ -1,8 +1,11 @@ import requests import logging import zipfile +import socket +import ipaddress from cachetools import cached, TTLCache from datetime import datetime +from urllib.parse import urljoin, urlparse from services.network_utils import fetch_with_curl logger = logging.getLogger(__name__) @@ -10,6 +13,7 @@ # Cache Frontline data for 30 minutes, it doesn't move that fast frontline_cache = TTLCache(maxsize=1, ttl=1800) + @cached(frontline_cache) def fetch_ukraine_frontlines(): """ @@ -18,27 +22,34 @@ def fetch_ukraine_frontlines(): """ try: logger.info("Fetching DeepStateMap from GitHub mirror...") - + # First, query the repo tree to find the latest file name - tree_url = "https://api.github.com/repos/cyterat/deepstate-map-data/git/trees/main?recursive=1" + tree_url = ( + "https://api.github.com/repos/cyterat/deepstate-map-data/git/trees/main?recursive=1" + ) res_tree = requests.get(tree_url, timeout=10) - + if res_tree.status_code == 200: tree_data = res_tree.json().get("tree", []) # Filter for geojson files in data folder - geo_files = [item["path"] for item in tree_data if item["path"].startswith("data/deepstatemap_data_") and item["path"].endswith(".geojson")] - + geo_files = [ + item["path"] + for item in tree_data + if item["path"].startswith("data/deepstatemap_data_") + and item["path"].endswith(".geojson") + ] + if geo_files: # Get the alphabetically latest file (since it's named with YYYYMMDD) latest_file = sorted(geo_files)[-1] - + raw_url = f"https://raw.githubusercontent.com/cyterat/deepstate-map-data/main/{latest_file}" logger.info(f"Downloading latest DeepStateMap: {raw_url}") - + res_geo = requests.get(raw_url, timeout=20) if res_geo.status_code == 200: data = res_geo.json() - + # The Cyterat GitHub mirror strips all properties and just provides a raw array of Feature polygons. # Based on DeepStateMap's frontend mapping, the array index corresponds to the zone type: # 0: Russian-occupied areas @@ -49,68 +60,78 @@ def fetch_ukraine_frontlines(): 0: "Russian-occupied areas", 1: "Russian advance", 2: "Liberated area", - 3: "Russian-occupied areas", # Crimea / LPR / DPR - 4: "Directions of UA attacks" + 3: "Russian-occupied areas", # Crimea / LPR / DPR + 4: "Directions of UA attacks", } - + if "features" in data: for idx, feature in enumerate(data["features"]): if "properties" not in feature or feature["properties"] is None: feature["properties"] = {} - - feature["properties"]["name"] = name_map.get(idx, "Russian-occupied areas") + + feature["properties"]["name"] = name_map.get( + idx, "Russian-occupied areas" + ) feature["properties"]["zone_id"] = idx - + return data else: - logger.error(f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}") + logger.error( + f"Failed to fetch parsed Github Raw GeoJSON: {res_geo.status_code}" + ) else: logger.error(f"Failed to fetch Github Tree for Deepstatemap: {res_tree.status_code}") except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: logger.error(f"Error fetching DeepStateMap: {e}") return None + # Cache GDELT data for 6 hours - heavy aggregation, data doesn't change rapidly gdelt_cache = TTLCache(maxsize=1, ttl=21600) + def _extract_domain(url): """Extract a clean source name from a URL, e.g. 'nytimes.com' from 'https://www.nytimes.com/...'""" try: from urllib.parse import urlparse - host = urlparse(url).hostname or '' + + host = urlparse(url).hostname or "" # Strip www. prefix - if host.startswith('www.'): + if host.startswith("www."): host = host[4:] return host except (ValueError, AttributeError, KeyError): # non-critical return url[:40] + def _url_to_headline(url): """Extract a human-readable headline from a URL path. e.g. 'https://nytimes.com/2026/03/us-strikes-iran-nuclear-sites.html' -> 'Us Strikes Iran Nuclear Sites' Falls back to domain name if the URL slug is gibberish (hex IDs, UUIDs, etc.). """ import re + try: from urllib.parse import urlparse, unquote + parsed = urlparse(url) - domain = parsed.hostname or '' - if domain.startswith('www.'): + domain = parsed.hostname or "" + if domain.startswith("www."): domain = domain[4:] # Get last meaningful path segment - path = unquote(parsed.path).strip('/') + path = unquote(parsed.path).strip("/") if not path: return domain # Try the last path segment first, then walk backwards - segments = [s for s in path.split('/') if s] - slug = '' + segments = [s for s in path.split("/") if s] + slug = "" for seg in reversed(segments): # Remove file extensions - for ext in ['.html', '.htm', '.php', '.asp', '.aspx', '.shtml']: + for ext in [".html", ".htm", ".php", ".asp", ".aspx", ".shtml"]: if seg.lower().endswith(ext): - seg = seg[:-len(ext)] + seg = seg[: -len(ext)] # Skip segments that are clearly not headlines if _is_gibberish(seg): continue @@ -121,22 +142,22 @@ def _url_to_headline(url): return domain # Remove common ID patterns at start/end - slug = re.sub(r'^[\d]+-', '', slug) # leading "13847569-" - slug = re.sub(r'-[\da-f]{6,}$', '', slug) # trailing hex IDs - slug = re.sub(r'[-_]c-\d+$', '', slug) # trailing "-c-21803431" - slug = re.sub(r'^p=\d+$', '', slug) # WordPress ?p=1234 + slug = re.sub(r"^[\d]+-", "", slug) # leading "13847569-" + slug = re.sub(r"-[\da-f]{6,}$", "", slug) # trailing hex IDs + slug = re.sub(r"[-_]c-\d+$", "", slug) # trailing "-c-21803431" + slug = re.sub(r"^p=\d+$", "", slug) # WordPress ?p=1234 # Convert slug separators to spaces - slug = slug.replace('-', ' ').replace('_', ' ') - slug = re.sub(r'\s+', ' ', slug).strip() + slug = slug.replace("-", " ").replace("_", " ") + slug = re.sub(r"\s+", " ", slug).strip() # Final gibberish check after cleanup - if len(slug) < 8 or _is_gibberish(slug.replace(' ', '-')): + if len(slug) < 8 or _is_gibberish(slug.replace(" ", "-")): return domain # Title case and truncate headline = slug.title() if len(headline) > 90: - headline = headline[:87] + '...' + headline = headline[:87] + "..." return headline except (ValueError, AttributeError, KeyError): # non-critical return url[:60] @@ -146,19 +167,22 @@ def _is_gibberish(text): """Detect if a URL segment is gibberish (hex IDs, UUIDs, numeric IDs, etc.) rather than a real human-readable slug like 'us-strikes-iran'.""" import re + t = text.strip() if not t: return True # Pure numbers - if re.match(r'^\d+$', t): + if re.match(r"^\d+$", t): return True # UUID pattern (with or without dashes) - if re.match(r'^[0-9a-f]{8}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{12}$', t, re.I): + if re.match( + r"^[0-9a-f]{8}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{4}[_-]?[0-9a-f]{12}$", t, re.I + ): return True # Hex-heavy string: more than 40% hex digits among alphanumeric chars - alnum = re.sub(r'[^a-zA-Z0-9]', '', t) + alnum = re.sub(r"[^a-zA-Z0-9]", "", t) if alnum: - hex_chars = sum(1 for c in alnum if c in '0123456789abcdefABCDEF') + hex_chars = sum(1 for c in alnum if c in "0123456789abcdefABCDEF") if hex_chars / len(alnum) > 0.4 and len(alnum) > 6: return True # Mostly digits with a few alpha (like "article8efa6c53") @@ -169,13 +193,81 @@ def _is_gibberish(text): if len(t) < 5: return True # Query-param style segments - if '=' in t: + if "=" in t: return True return False # Persistent cache for article titles — survives across GDELT cache refreshes -_article_title_cache = {} +# Bounded to 5000 entries with 24hr TTL to prevent unbounded memory growth +_article_title_cache = TTLCache(maxsize=5000, ttl=86400) +_article_url_safety_cache = TTLCache(maxsize=5000, ttl=3600) +_TITLE_FETCH_MAX_REDIRECTS = 3 +_TITLE_FETCH_READ_BYTES = 32768 +_ALLOWED_ARTICLE_PORTS = {80, 443, 8080, 8443} + + +def _hostname_resolves_public(hostname: str, port: int) -> bool: + try: + infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except (socket.gaierror, OSError): + return False + + addresses = set() + for info in infos: + sockaddr = info[4] if len(info) > 4 else None + if not sockaddr: + continue + raw_addr = str(sockaddr[0] or "").split("%", 1)[0] + if not raw_addr: + continue + try: + addresses.add(ipaddress.ip_address(raw_addr)) + except ValueError: + continue + + return bool(addresses) and all(addr.is_global for addr in addresses) + + +def _is_safe_public_article_url(url: str) -> tuple[bool, str]: + cached = _article_url_safety_cache.get(url) + if cached is not None: + return cached + + try: + parsed = urlparse(str(url or "").strip()) + except ValueError: + result = (False, "parse_error") + _article_url_safety_cache[url] = result + return result + + scheme = str(parsed.scheme or "").lower() + host = str(parsed.hostname or "").strip().lower() + if scheme not in {"http", "https"}: + result = (False, "scheme") + elif not host: + result = (False, "host") + elif parsed.username or parsed.password: + result = (False, "userinfo") + elif host in {"localhost", "localhost.localdomain"}: + result = (False, "localhost") + else: + port = parsed.port or (443 if scheme == "https" else 80) + if port not in _ALLOWED_ARTICLE_PORTS: + result = (False, "port") + else: + try: + target_ip = ipaddress.ip_address(host.split("%", 1)[0]) + except ValueError: + target_ip = None + if target_ip is not None: + result = (True, "") if target_ip.is_global else (False, "private_ip") + else: + result = (True, "") if _hostname_resolves_public(host, port) else (False, "private_dns") + + _article_url_safety_cache[url] = result + return result + def _fetch_article_title(url): """Fetch the real headline from an article's HTML or og:title tag. @@ -183,51 +275,85 @@ def _fetch_article_title(url): Uses a persistent cache to avoid refetching.""" if url in _article_title_cache: return _article_title_cache[url] - + import re + try: - # Only read the first 32KB — the <title> is always in <head> - resp = requests.get(url, timeout=4, headers={ - 'User-Agent': 'Mozilla/5.0 (compatible; OSINT Dashboard/1.0)' - }, stream=True) - if resp.status_code != 200: + current_url = str(url or "").strip() + chunk = "" + for _ in range(_TITLE_FETCH_MAX_REDIRECTS + 1): + allowed, _reason = _is_safe_public_article_url(current_url) + if not allowed: + _article_title_cache[url] = None + return None + + resp = requests.get( + current_url, + timeout=4, + headers={"User-Agent": "Mozilla/5.0 (compatible; OSINT Dashboard/1.0)"}, + stream=True, + allow_redirects=False, + ) + try: + location = str(resp.headers.get("Location") or "").strip() + if 300 <= resp.status_code < 400 and location: + current_url = urljoin(current_url, location) + continue + if resp.status_code != 200: + _article_title_cache[url] = None + return None + chunk = resp.raw.read(_TITLE_FETCH_READ_BYTES).decode("utf-8", errors="replace") + break + finally: + resp.close() + else: _article_title_cache[url] = None return None - - chunk = resp.raw.read(32768).decode('utf-8', errors='replace') - resp.close() - + title = None - + # Try og:title first (usually the cleanest) - og_match = re.search(r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\'>]+)["\']', chunk, re.I) + og_match = re.search( + r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\'>]+)["\']', chunk, re.I + ) if not og_match: - og_match = re.search(r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+property=["\']og:title["\']', chunk, re.I) + og_match = re.search( + r'<meta[^>]+content=["\']([^"\'>]+)["\'][^>]+property=["\']og:title["\']', + chunk, + re.I, + ) if og_match: title = og_match.group(1).strip() - + # Fall back to <title> tag if not title: - title_match = re.search(r'<title[^>]*>([^<]+)', chunk, re.I) + title_match = re.search(r"]*>([^<]+)", chunk, re.I) if title_match: title = title_match.group(1).strip() - + if title: # Clean up HTML entities import html as html_mod + title = html_mod.unescape(title) # Remove site name suffixes like " | CNN" or " - BBC News" - title = re.sub(r'\s*[|\-–—]\s*[^|\-–—]{2,30}$', '', title).strip() + title = re.sub(r"\s*[|\-–—]\s*[^|\-–—]{2,30}$", "", title).strip() # Truncate very long titles if len(title) > 120: - title = title[:117] + '...' + title = title[:117] + "..." if len(title) > 10: _article_title_cache[url] = title return title - + _article_title_cache[url] = None return None - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, AttributeError): # non-critical + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + AttributeError, + ): # non-critical _article_title_cache[url] = None return None @@ -236,6 +362,7 @@ def _batch_fetch_titles(urls): """Fetch real article titles for a list of URLs in parallel. Returns a dict of url -> title (or None if fetch failed).""" from concurrent.futures import ThreadPoolExecutor + results = {} with ThreadPoolExecutor(max_workers=16) as executor: futures = {executor.submit(_fetch_article_title, u): u for u in urls} @@ -253,16 +380,19 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_ loc_index maps loc_key -> index in features list for fast duplicate merging. """ import csv, io, zipfile + try: zf = zipfile.ZipFile(io.BytesIO(zip_bytes)) csv_name = zf.namelist()[0] with zf.open(csv_name) as cf: - reader = csv.reader(io.TextIOWrapper(cf, encoding='utf-8', errors='replace'), delimiter='\t') + reader = csv.reader( + io.TextIOWrapper(cf, encoding="utf-8", errors="replace"), delimiter="\t" + ) for row in reader: try: if len(row) < 61: continue - event_code = row[26][:2] if len(row[26]) >= 2 else '' + event_code = row[26][:2] if len(row[26]) >= 2 else "" if event_code not in conflict_codes: continue lat = float(row[56]) if row[56] else None @@ -270,10 +400,10 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_ if lat is None or lng is None or (lat == 0 and lng == 0): continue - source_url = row[60].strip() if len(row) > 60 else '' - location = row[52].strip() if len(row) > 52 else 'Unknown' - actor1 = row[6].strip() if len(row) > 6 else '' - actor2 = row[16].strip() if len(row) > 16 else '' + source_url = row[60].strip() if len(row) > 60 else "" + location = row[52].strip() if len(row) > 52 else "Unknown" + actor1 = row[6].strip() if len(row) > 6 else "" + actor2 = row[16].strip() if len(row) > 16 else "" loc_key = f"{round(lat, 1)}_{round(lng, 1)}" if loc_key in seen_locs: @@ -293,25 +423,32 @@ def _parse_gdelt_export_zip(zip_bytes, conflict_codes, seen_locs, features, loc_ continue seen_locs.add(loc_key) - name = location or (f"{actor1} vs {actor2}" if actor1 and actor2 else actor1) or "Unknown Incident" - domain = _extract_domain(source_url) if source_url else '' + name = ( + location + or (f"{actor1} vs {actor2}" if actor1 and actor2 else actor1) + or "Unknown Incident" + ) + domain = _extract_domain(source_url) if source_url else "" loc_index[loc_key] = len(features) - features.append({ - "type": "Feature", - "properties": { - "name": name, - "count": 1, - "_urls": [source_url] if source_url else [], - "_domains": {domain} if domain else set(), - }, - "geometry": {"type": "Point", "coordinates": [lng, lat]}, - "_loc_key": loc_key - }) + features.append( + { + "type": "Feature", + "properties": { + "name": name, + "count": 1, + "_urls": [source_url] if source_url else [], + "_domains": {domain} if domain else set(), + }, + "geometry": {"type": "Point", "coordinates": [lng, lat]}, + "_loc_key": loc_key, + } + ) except (ValueError, IndexError): continue except (IOError, OSError, ValueError, KeyError, zipfile.BadZipFile) as e: logger.warning(f"Failed to parse GDELT export zip: {e}") + def _download_gdelt_export(url): """Download a single GDELT export file, return bytes or None.""" try: @@ -322,10 +459,12 @@ def _download_gdelt_export(url): pass return None + def _build_feature_html(features, fetched_titles=None): """Build URL + headline arrays for frontend rendering. Uses fetched_titles (real article titles) when available, falls back to URL slug parsing.""" import html as html_mod + for f in features: urls = f["properties"].pop("_urls", []) f["properties"].pop("_domains", None) @@ -338,10 +477,12 @@ def _build_feature_html(features, fetched_titles=None): if urls: links = [] for u, h in zip(urls, headlines): - safe_url = u if u.startswith(('http://', 'https://')) else 'about:blank' + safe_url = u if u.startswith(("http://", "https://")) else "about:blank" safe_h = html_mod.escape(h) - links.append(f'') - f["properties"]["html"] = ''.join(links) + links.append( + f'' + ) + f["properties"]["html"] = "".join(links) else: f["properties"]["html"] = html_mod.escape(f["properties"]["name"]) f.pop("_loc_key", None) @@ -350,6 +491,7 @@ def _build_feature_html(features, fetched_titles=None): def _enrich_gdelt_titles_background(features, all_article_urls): """Background thread: fetch real article titles then update features in-place.""" import html as html_mod + try: logger.info(f"[BG] Fetching real article titles for {len(all_article_urls)} URLs...") fetched_titles = _batch_fetch_titles(all_article_urls) @@ -368,10 +510,12 @@ def _enrich_gdelt_titles_background(features, all_article_urls): f["properties"]["_headlines_list"] = headlines links = [] for u, h in zip(urls, headlines): - safe_url = u if u.startswith(('http://', 'https://')) else 'about:blank' + safe_url = u if u.startswith(("http://", "https://")) else "about:blank" safe_h = html_mod.escape(h) - links.append(f'') - f["properties"]["html"] = ''.join(links) + links.append( + f'' + ) + f["properties"]["html"] = "".join(links) logger.info(f"[BG] GDELT title enrichment complete") except Exception as e: logger.error(f"[BG] GDELT title enrichment failed: {e}") @@ -391,16 +535,18 @@ def fetch_global_military_incidents(): logger.info("Fetching GDELT events via export CDN (multi-file)...") # Get the latest export URL to determine current timestamp - index_res = fetch_with_curl("http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10) + index_res = fetch_with_curl( + "http://data.gdeltproject.org/gdeltv2/lastupdate.txt", timeout=10 + ) if index_res.status_code != 200: logger.error(f"GDELT lastupdate failed: {index_res.status_code}") return [] # Extract latest export URL and its timestamp latest_url = None - for line in index_res.text.strip().split('\n'): + for line in index_res.text.strip().split("\n"): parts = line.strip().split() - if len(parts) >= 3 and parts[2].endswith('.export.CSV.zip'): + if len(parts) >= 3 and parts[2].endswith(".export.CSV.zip"): latest_url = parts[2] break @@ -410,19 +556,20 @@ def fetch_global_military_incidents(): # Extract timestamp from URL like: http://data.gdeltproject.org/gdeltv2/20260301120000.export.CSV.zip import re - ts_match = re.search(r'(\d{14})\.export\.CSV\.zip', latest_url) + + ts_match = re.search(r"(\d{14})\.export\.CSV\.zip", latest_url) if not ts_match: logger.error("Could not parse GDELT export timestamp") return [] - latest_ts = datetime.strptime(ts_match.group(1), '%Y%m%d%H%M%S') + latest_ts = datetime.strptime(ts_match.group(1), "%Y%m%d%H%M%S") # Generate URLs for the last 8 hours (32 files at 15-min intervals) NUM_FILES = 32 urls = [] for i in range(NUM_FILES): ts = latest_ts - timedelta(minutes=15 * i) - fname = ts.strftime('%Y%m%d%H%M%S') + '.export.CSV.zip' + fname = ts.strftime("%Y%m%d%H%M%S") + ".export.CSV.zip" url = f"http://data.gdeltproject.org/gdeltv2/{fname}" urls.append(url) @@ -436,7 +583,7 @@ def fetch_global_military_incidents(): logger.info(f"Downloaded {successful}/{len(urls)} GDELT exports") # Parse all downloaded files - CONFLICT_CODES = {'14', '17', '18', '19', '20'} + CONFLICT_CODES = {"14", "17", "18", "19", "20"} features = [] seen_locs = set() loc_index = {} # loc_key -> index in features @@ -455,7 +602,9 @@ def fetch_global_military_incidents(): # Build HTML immediately with URL-slug headlines (instant, no network) _build_feature_html(features) - logger.info(f"GDELT parsed: {len(features)} conflict locations from {successful} files (titles enriching in background)") + logger.info( + f"GDELT parsed: {len(features)} conflict locations from {successful} files (titles enriching in background)" + ) # Kick off background thread to enrich with real article titles # Features list is shared — background thread updates in-place @@ -468,6 +617,13 @@ def fetch_global_military_incidents(): return features - except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: + except ( + requests.RequestException, + ConnectionError, + TimeoutError, + ValueError, + KeyError, + OSError, + ) as e: logger.error(f"Error fetching GDELT data: {e}") return [] diff --git a/backend/services/kiwisdr_fetcher.py b/backend/services/kiwisdr_fetcher.py index 2abf9fcf..ec187af8 100644 --- a/backend/services/kiwisdr_fetcher.py +++ b/backend/services/kiwisdr_fetcher.py @@ -16,13 +16,13 @@ def _parse_comment(html: str, field: str) -> str: """Extract a field value from HTML comment like """ - m = re.search(rf'', html) + m = re.search(rf"", html) return m.group(1).strip() if m else "" def _parse_gps(html: str): """Extract lat/lon from comment.""" - m = re.search(r'', html) + m = re.search(r"", html) if m: try: return float(m.group(1)), float(m.group(2)) @@ -78,17 +78,19 @@ def fetch_kiwisdr_nodes() -> list[dict]: except ValueError: users_max = 0 - nodes.append({ - "name": name[:120], # Truncate long names - "lat": round(lat, 5), - "lon": round(lon, 5), - "url": url, - "users": users, - "users_max": users_max, - "bands": bands, - "antenna": antenna[:200] if antenna else "", - "location": location[:100] if location else "", - }) + nodes.append( + { + "name": name[:120], # Truncate long names + "lat": round(lat, 5), + "lon": round(lon, 5), + "url": url, + "users": users, + "users_max": users_max, + "bands": bands, + "antenna": antenna[:200] if antenna else "", + "location": location[:100] if location else "", + } + ) logger.info(f"KiwiSDR: parsed {len(nodes)} online receivers") return nodes diff --git a/backend/services/liveuamap_scraper.py b/backend/services/liveuamap_scraper.py index 7f9c4de0..f827a2dd 100644 --- a/backend/services/liveuamap_scraper.py +++ b/backend/services/liveuamap_scraper.py @@ -8,90 +8,101 @@ logger = logging.getLogger(__name__) + def fetch_liveuamap(): logger.info("Starting Liveuamap scraper with Playwright Stealth...") - + regions = [ {"name": "Ukraine", "url": "https://liveuamap.com"}, {"name": "Middle East", "url": "https://mideast.liveuamap.com"}, {"name": "Israel-Palestine", "url": "https://israelpalestine.liveuamap.com"}, - {"name": "Syria", "url": "https://syria.liveuamap.com"} + {"name": "Syria", "url": "https://syria.liveuamap.com"}, ] - + all_markers = [] seen_ids = set() - + with sync_playwright() as p: # Launching with a real user agent to bypass Turnstile - browser = p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled"]) + browser = p.chromium.launch( + headless=True, args=["--disable-blink-features=AutomationControlled"] + ) context = browser.new_context( user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", viewport={"width": 1920, "height": 1080}, - color_scheme="dark" + color_scheme="dark", ) page = context.new_page() stealth_sync(page) - + for region in regions: try: logger.info(f"Scraping Liveuamap region: {region['name']}") page.goto(region["url"], timeout=60000, wait_until="domcontentloaded") - + # Wait for the map canvas or markers script to load, max 10s wait try: page.wait_for_timeout(5000) except (TimeoutError, OSError): # non-critical: page load delay pass - + html = page.content() - + m = re.search(r"var\s+ovens\s*=\s*(.*?);(?!function)", html, re.DOTALL) if not m: logger.warning(f"Could not find 'ovens' data for {region['name']} in raw HTML") # Let's try grabbing the evaluated JavaScript variable if it's there try: - ovens_json = page.evaluate("() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null") + ovens_json = page.evaluate( + "() => typeof ovens !== 'undefined' ? JSON.stringify(ovens) : null" + ) if ovens_json: markers = json.loads(ovens_json) # process below html = f"var ovens={ovens_json};" m = re.search(r"var\s+ovens=(.*?);", html, re.DOTALL) except (ValueError, KeyError, OSError) as e: # non-critical: JS eval fallback - logger.debug(f"Could not evaluate ovens JS variable for {region['name']}: {e}") - + logger.debug( + f"Could not evaluate ovens JS variable for {region['name']}: {e}" + ) + if m: json_str = m.group(1).strip() if json_str.startswith("'") or json_str.startswith('"'): - json_str = json_str.strip('"\'') - json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode('utf-8') - + json_str = json_str.strip("\"'") + json_str = base64.b64decode(urllib.parse.unquote(json_str)).decode("utf-8") + try: markers = json.loads(json_str) for marker in markers: mid = marker.get("id") if mid and mid not in seen_ids: seen_ids.add(mid) - all_markers.append({ - "id": mid, - "type": "liveuamap", - "title": marker.get("s", "Unknown Event") or marker.get("title", ""), - "lat": marker.get("lat"), - "lng": marker.get("lng"), - "timestamp": marker.get("time", ""), - "link": marker.get("link", region["url"]), - "region": region["name"] - }) + all_markers.append( + { + "id": mid, + "type": "liveuamap", + "title": marker.get("s", "Unknown Event") + or marker.get("title", ""), + "lat": marker.get("lat"), + "lng": marker.get("lng"), + "timestamp": marker.get("time", ""), + "link": marker.get("link", region["url"]), + "region": region["name"], + } + ) except (json.JSONDecodeError, ValueError, KeyError) as e: logger.error(f"Error parsing JSON for {region['name']}: {e}") - + except Exception as e: logger.error(f"Error scraping Liveuamap {region['name']}: {e}") - + browser.close() - + logger.info(f"Liveuamap scraper finished, extracted {len(all_markers)} unique markers.") return all_markers + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) res = fetch_liveuamap() diff --git a/backend/services/logging_setup.py b/backend/services/logging_setup.py new file mode 100644 index 00000000..fbfddf38 --- /dev/null +++ b/backend/services/logging_setup.py @@ -0,0 +1,30 @@ +"""Structured logging setup for backend services.""" + +import json +import logging +from datetime import datetime +from typing import Any, Dict + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: Dict[str, Any] = { + "ts": datetime.utcnow().isoformat(), + "level": record.levelname, + "logger": record.name, + "msg": record.getMessage(), + } + if record.exc_info: + payload["exc"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=False) + + +def setup_logging(level: str = "INFO"): + """Configure root logger with JSON formatting.""" + root = logging.getLogger() + if root.handlers: + return # Respect existing config + root.setLevel(level.upper()) + handler = logging.StreamHandler() + handler.setFormatter(JsonFormatter()) + root.addHandler(handler) diff --git a/backend/services/mesh/__init__.py b/backend/services/mesh/__init__.py new file mode 100644 index 00000000..3180f268 --- /dev/null +++ b/backend/services/mesh/__init__.py @@ -0,0 +1 @@ +# Mesh protocol services package diff --git a/backend/services/mesh/mesh_bootstrap_manifest.py b/backend/services/mesh/mesh_bootstrap_manifest.py new file mode 100644 index 00000000..2da6cb05 --- /dev/null +++ b/backend/services/mesh/mesh_bootstrap_manifest.py @@ -0,0 +1,340 @@ +from __future__ import annotations + +import base64 +import json +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from services.config import get_settings +from services.mesh.mesh_crypto import canonical_json, normalize_peer_url + +BACKEND_DIR = Path(__file__).resolve().parents[2] +DATA_DIR = BACKEND_DIR / "data" +DEFAULT_BOOTSTRAP_MANIFEST_PATH = DATA_DIR / "bootstrap_peers.json" +BOOTSTRAP_MANIFEST_VERSION = 1 +ALLOWED_BOOTSTRAP_TRANSPORTS = {"clearnet", "onion"} +ALLOWED_BOOTSTRAP_ROLES = {"participant", "relay", "seed"} + + +class BootstrapManifestError(ValueError): + pass + + +@dataclass(frozen=True) +class BootstrapPeer: + peer_url: str + transport: str + role: str + label: str = "" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class BootstrapManifest: + version: int + issued_at: int + valid_until: int + signer_id: str + peers: tuple[BootstrapPeer, ...] + signature: str + + def payload_dict(self) -> dict[str, Any]: + return { + "version": int(self.version), + "issued_at": int(self.issued_at), + "valid_until": int(self.valid_until), + "signer_id": str(self.signer_id or ""), + "peers": [peer.to_dict() for peer in self.peers], + } + + def to_dict(self) -> dict[str, Any]: + payload = self.payload_dict() + payload["signature"] = str(self.signature or "") + return payload + + +def _resolve_manifest_path(raw_path: str) -> Path: + raw = str(raw_path or "").strip() + if not raw: + return DEFAULT_BOOTSTRAP_MANIFEST_PATH + candidate = Path(raw) + if candidate.is_absolute(): + return candidate + return BACKEND_DIR / candidate + + +def _canonical_manifest_payload(payload: dict[str, Any]) -> str: + return canonical_json(payload) + + +def _load_signer_private_key(private_key_b64: str) -> ed25519.Ed25519PrivateKey: + try: + signer_private_key = base64.b64decode( + str(private_key_b64 or "").encode("utf-8"), + validate=True, + ) + return ed25519.Ed25519PrivateKey.from_private_bytes(signer_private_key) + except Exception as exc: + raise BootstrapManifestError("bootstrap signer private key must be raw Ed25519 base64") from exc + + +def bootstrap_signer_public_key_b64(private_key_b64: str) -> str: + signer = _load_signer_private_key(private_key_b64) + public_key = signer.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + return base64.b64encode(public_key).decode("utf-8") + + +def generate_bootstrap_signer() -> dict[str, str]: + signer = ed25519.Ed25519PrivateKey.generate() + private_key = signer.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + public_key = signer.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + return { + "private_key_b64": base64.b64encode(private_key).decode("utf-8"), + "public_key_b64": base64.b64encode(public_key).decode("utf-8"), + } + + +def _verify_manifest_signature( + payload: dict[str, Any], + *, + signature_b64: str, + signer_public_key_b64: str, +) -> None: + try: + signature = base64.b64decode(str(signature_b64 or "").encode("utf-8"), validate=True) + except Exception as exc: + raise BootstrapManifestError("bootstrap manifest signature must be base64") from exc + + try: + signer_public_key = base64.b64decode( + str(signer_public_key_b64 or "").encode("utf-8"), + validate=True, + ) + verifier = ed25519.Ed25519PublicKey.from_public_bytes(signer_public_key) + except Exception as exc: + raise BootstrapManifestError("bootstrap signer public key must be raw Ed25519 base64") from exc + + serialized = _canonical_manifest_payload(payload).encode("utf-8") + try: + verifier.verify(signature, serialized) + except InvalidSignature as exc: + raise BootstrapManifestError("bootstrap manifest signature invalid") from exc + + +def _validate_bootstrap_peer(peer_data: dict[str, Any]) -> BootstrapPeer: + peer_url = str(peer_data.get("peer_url", "") or "").strip() + transport = str(peer_data.get("transport", "") or "").strip().lower() + role = str(peer_data.get("role", "") or "").strip().lower() + label = str(peer_data.get("label", "") or "").strip() + + if transport not in ALLOWED_BOOTSTRAP_TRANSPORTS: + raise BootstrapManifestError(f"unsupported bootstrap transport: {transport or 'missing'}") + if role not in ALLOWED_BOOTSTRAP_ROLES: + raise BootstrapManifestError(f"unsupported bootstrap role: {role or 'missing'}") + + normalized = normalize_peer_url(peer_url) + if not normalized or normalized != peer_url: + raise BootstrapManifestError("bootstrap peer_url must be normalized") + + parsed = urlparse(normalized) + hostname = str(parsed.hostname or "").strip().lower() + if transport == "clearnet": + if parsed.scheme != "https" or hostname.endswith(".onion"): + raise BootstrapManifestError("clearnet bootstrap peers must use https://") + elif transport == "onion": + if parsed.scheme != "http" or not hostname.endswith(".onion"): + raise BootstrapManifestError("onion bootstrap peers must use http://*.onion") + + return BootstrapPeer( + peer_url=normalized, + transport=transport, + role=role, + label=label, + ) + + +def _validate_bootstrap_manifest_payload( + payload: dict[str, Any], + *, + now: float | None = None, +) -> BootstrapManifest: + version = int(payload.get("version", 0) or 0) + issued_at = int(payload.get("issued_at", 0) or 0) + valid_until = int(payload.get("valid_until", 0) or 0) + signer_id = str(payload.get("signer_id", "") or "").strip() + peers_raw = payload.get("peers", []) + current_time = int(now if now is not None else time.time()) + + if version != BOOTSTRAP_MANIFEST_VERSION: + raise BootstrapManifestError(f"unsupported bootstrap manifest version: {version}") + if not signer_id: + raise BootstrapManifestError("bootstrap manifest signer_id is required") + if issued_at <= 0 or valid_until <= 0 or valid_until <= issued_at: + raise BootstrapManifestError("bootstrap manifest validity window is invalid") + if current_time > valid_until: + raise BootstrapManifestError("bootstrap manifest expired") + if not isinstance(peers_raw, list): + raise BootstrapManifestError("bootstrap manifest peers must be a list") + + peers: list[BootstrapPeer] = [] + seen: set[tuple[str, str]] = set() + for entry in peers_raw: + if not isinstance(entry, dict): + raise BootstrapManifestError("bootstrap manifest peers must be objects") + peer = _validate_bootstrap_peer(entry) + key = (peer.transport, peer.peer_url) + if key in seen: + raise BootstrapManifestError("bootstrap manifest peers must be unique") + seen.add(key) + peers.append(peer) + + if not peers: + raise BootstrapManifestError("bootstrap manifest must contain at least one peer") + + return BootstrapManifest( + version=version, + issued_at=issued_at, + valid_until=valid_until, + signer_id=signer_id, + peers=tuple(peers), + signature="", + ) + + +def build_bootstrap_manifest_payload( + *, + signer_id: str, + peers: list[dict[str, Any]] | tuple[dict[str, Any], ...], + issued_at: int | None = None, + valid_until: int | None = None, + valid_for_hours: int = 168, +) -> dict[str, Any]: + timestamp = int(issued_at if issued_at is not None else time.time()) + expiry = int(valid_until if valid_until is not None else timestamp + max(1, int(valid_for_hours or 0)) * 3600) + payload = { + "version": BOOTSTRAP_MANIFEST_VERSION, + "issued_at": timestamp, + "valid_until": expiry, + "signer_id": str(signer_id or "").strip(), + "peers": list(peers), + } + manifest = _validate_bootstrap_manifest_payload(payload, now=timestamp) + return manifest.payload_dict() + + +def sign_bootstrap_manifest_payload( + payload: dict[str, Any], + *, + signer_private_key_b64: str, +) -> str: + signer = _load_signer_private_key(signer_private_key_b64) + serialized = _canonical_manifest_payload(payload).encode("utf-8") + signature = signer.sign(serialized) + return base64.b64encode(signature).decode("utf-8") + + +def write_signed_bootstrap_manifest( + path: str | Path, + *, + signer_id: str, + signer_private_key_b64: str, + peers: list[dict[str, Any]] | tuple[dict[str, Any], ...], + issued_at: int | None = None, + valid_until: int | None = None, + valid_for_hours: int = 168, +) -> BootstrapManifest: + manifest_path = _resolve_manifest_path(str(path)) + payload = build_bootstrap_manifest_payload( + signer_id=signer_id, + peers=list(peers), + issued_at=issued_at, + valid_until=valid_until, + valid_for_hours=valid_for_hours, + ) + signature = sign_bootstrap_manifest_payload( + payload, + signer_private_key_b64=signer_private_key_b64, + ) + manifest = BootstrapManifest( + version=int(payload["version"]), + issued_at=int(payload["issued_at"]), + valid_until=int(payload["valid_until"]), + signer_id=str(payload["signer_id"]), + peers=tuple(_validate_bootstrap_peer(dict(peer)) for peer in payload["peers"]), + signature=signature, + ) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest.to_dict(), indent=2) + "\n", encoding="utf-8") + return manifest + + +def load_bootstrap_manifest( + path: str | Path, + *, + signer_public_key_b64: str, + now: float | None = None, +) -> BootstrapManifest: + manifest_path = _resolve_manifest_path(str(path)) + try: + raw = json.loads(manifest_path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise BootstrapManifestError(f"bootstrap manifest not found: {manifest_path}") from exc + except json.JSONDecodeError as exc: + raise BootstrapManifestError("bootstrap manifest is not valid JSON") from exc + + if not isinstance(raw, dict): + raise BootstrapManifestError("bootstrap manifest root must be an object") + + signature = str(raw.get("signature", "") or "").strip() + payload = {key: value for key, value in raw.items() if key != "signature"} + if not signature: + raise BootstrapManifestError("bootstrap manifest signature is required") + + _verify_manifest_signature( + payload, + signature_b64=signature, + signer_public_key_b64=signer_public_key_b64, + ) + manifest = _validate_bootstrap_manifest_payload(payload, now=now) + return BootstrapManifest( + version=manifest.version, + issued_at=manifest.issued_at, + valid_until=manifest.valid_until, + signer_id=manifest.signer_id, + peers=manifest.peers, + signature=signature, + ) + + +def load_bootstrap_manifest_from_settings(*, now: float | None = None) -> BootstrapManifest | None: + settings = get_settings() + if bool(getattr(settings, "MESH_BOOTSTRAP_DISABLED", False)): + return None + signer_public_key_b64 = str(getattr(settings, "MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY", "") or "").strip() + if not signer_public_key_b64: + return None + manifest_path = _resolve_manifest_path(str(getattr(settings, "MESH_BOOTSTRAP_MANIFEST_PATH", "") or "")) + return load_bootstrap_manifest( + manifest_path, + signer_public_key_b64=signer_public_key_b64, + now=now, + ) diff --git a/backend/services/mesh/mesh_crypto.py b/backend/services/mesh/mesh_crypto.py new file mode 100644 index 00000000..4ac2d8a3 --- /dev/null +++ b/backend/services/mesh/mesh_crypto.py @@ -0,0 +1,142 @@ +"""Cryptographic helpers for Mesh protocol verification.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +from typing import Any +from urllib.parse import urlparse + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec, ed25519 +from cryptography.exceptions import InvalidSignature + +from services.mesh.mesh_protocol import PROTOCOL_VERSION, NETWORK_ID, normalize_payload + +NODE_ID_PREFIX = "!sb_" +NODE_ID_HEX_LEN = 16 + + +def canonical_json(obj: dict[str, Any]) -> str: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def normalize_peer_url(peer_url: str) -> str: + raw = str(peer_url or "").strip() + if not raw: + return "" + parsed = urlparse(raw) + scheme = str(parsed.scheme or "").strip().lower() + hostname = str(parsed.hostname or "").strip().lower() + if not scheme or not hostname: + return "" + port = parsed.port + default_port = 443 if scheme == "https" else 80 if scheme == "http" else None + netloc = hostname + if port and port != default_port: + netloc = f"{hostname}:{port}" + path = str(parsed.path or "").rstrip("/") + return f"{scheme}://{netloc}{path}" + + +def _derive_peer_key(shared_secret: str, peer_url: str) -> bytes: + normalized_url = normalize_peer_url(peer_url) + if not shared_secret or not normalized_url: + return b"" + # HKDF-Extract per RFC 5869 §2.2: PRK = HMAC-Hash(salt, IKM). + # Python's hmac.new(key=salt, msg=IKM) maps directly to that definition. + prk = hmac.new( + b"sb-peer-auth-v1", + shared_secret.encode("utf-8"), + hashlib.sha256, + ).digest() + return hmac.new( + prk, + normalized_url.encode("utf-8") + b"\x01", + hashlib.sha256, + ).digest() + + +def _node_digest(public_key_b64: str) -> str: + raw = base64.b64decode(public_key_b64) + return hashlib.sha256(raw).hexdigest() + + +def derive_node_id(public_key_b64: str, *, legacy: bool = False) -> str: + digest = _node_digest(public_key_b64) + length = NODE_ID_HEX_LEN + return NODE_ID_PREFIX + digest[:length] + + +def derive_node_id_candidates(public_key_b64: str) -> tuple[str, ...]: + current = derive_node_id(public_key_b64, legacy=False) + return (current,) + + +def build_signature_payload( + *, + event_type: str, + node_id: str, + sequence: int, + payload: dict[str, Any], +) -> str: + normalized = normalize_payload(event_type, payload) + payload_json = canonical_json(normalized) + return "|".join( + [PROTOCOL_VERSION, NETWORK_ID, event_type, node_id, str(sequence), payload_json] + ) + + +def parse_public_key_algo(value: str) -> str: + val = (value or "").strip().upper() + if val in ("ED25519", "EDDSA"): + return "Ed25519" + if val in ("ECDSA", "ECDSA_P256", "P-256", "P256"): + return "ECDSA_P256" + return "" + + +def verify_signature( + *, + public_key_b64: str, + public_key_algo: str, + signature_hex: str, + payload: str, +) -> bool: + try: + sig_bytes = bytes.fromhex(signature_hex) + except Exception: + return False + + try: + pub_raw = base64.b64decode(public_key_b64) + except Exception: + return False + + algo = parse_public_key_algo(public_key_algo) + data = payload.encode("utf-8") + + try: + if algo == "Ed25519": + pub = ed25519.Ed25519PublicKey.from_public_bytes(pub_raw) + pub.verify(sig_bytes, data) + return True + if algo == "ECDSA_P256": + pub = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), pub_raw) + pub.verify(sig_bytes, data, ec.ECDSA(hashes.SHA256())) + return True + except InvalidSignature: + return False + except Exception: + return False + + return False + + +def verify_node_binding(node_id: str, public_key_b64: str) -> bool: + try: + return str(node_id or "") in derive_node_id_candidates(public_key_b64) + except Exception: + return False diff --git a/backend/services/mesh/mesh_dm_mls.py b/backend/services/mesh/mesh_dm_mls.py new file mode 100644 index 00000000..cf607c55 --- /dev/null +++ b/backend/services/mesh/mesh_dm_mls.py @@ -0,0 +1,669 @@ +"""MLS-backed DM session manager. + +This module keeps DM session orchestration in Python while privacy-core owns +the MLS session state. Python-side metadata survives via domain storage, but +Rust session state remains in-memory only. Process restart still requires +session re-establishment until Rust FFI state export is available. +""" + +from __future__ import annotations + +import base64 +import logging +import secrets +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +from services.mesh.mesh_secure_storage import ( + read_domain_json, + read_secure_json, + write_domain_json, +) +from services.mesh.mesh_privacy_logging import privacy_log_label +from services.mesh.mesh_wormhole_persona import sign_dm_alias_blob, verify_dm_alias_blob +from services.privacy_core_client import PrivacyCoreClient, PrivacyCoreError +from services.wormhole_supervisor import get_wormhole_state, transport_tier_from_state + +logger = logging.getLogger(__name__) + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +STATE_FILE = DATA_DIR / "wormhole_dm_mls.json" +STATE_FILENAME = "wormhole_dm_mls.json" +STATE_DOMAIN = "dm_alias" +_STATE_LOCK = threading.RLock() +_PRIVACY_CLIENT: PrivacyCoreClient | None = None +_STATE_LOADED = False +_TRANSPORT_TIER_ORDER = { + "public_degraded": 0, + "private_transitional": 1, + "private_strong": 2, +} +MLS_DM_FORMAT = "mls1" +MAX_DM_PLAINTEXT_SIZE = 65_536 + +try: + from nacl.public import PrivateKey as _NaclPrivateKey + from nacl.public import PublicKey as _NaclPublicKey + from nacl.public import SealedBox as _NaclSealedBox +except ImportError: + _NaclPrivateKey = None + _NaclPublicKey = None + _NaclSealedBox = None + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _decode_key_text(data: str | bytes | None) -> bytes: + raw = str(data or "").strip() + if not raw: + return b"" + try: + return bytes.fromhex(raw) + except ValueError: + return _unb64(raw) + + +def _normalize_alias(alias: str) -> str: + return str(alias or "").strip().lower() + + +def _session_id(local_alias: str, remote_alias: str) -> str: + return f"{_normalize_alias(local_alias)}::{_normalize_alias(remote_alias)}" + + +def _seal_keypair() -> dict[str, str]: + private_key = x25519.X25519PrivateKey.generate() + return { + "public_key": private_key.public_key().public_bytes_raw().hex(), + "private_key": private_key.private_bytes_raw().hex(), + } + + +def _seal_welcome_for_public_key(payload: bytes, public_key_text: str) -> bytes: + public_key_bytes = _decode_key_text(public_key_text) + if not public_key_bytes: + raise PrivacyCoreError("responder_dh_pub is required for sealed welcome") + if _NaclPublicKey is not None and _NaclSealedBox is not None: + return _NaclSealedBox(_NaclPublicKey(public_key_bytes)).encrypt(payload) + + ephemeral_private = x25519.X25519PrivateKey.generate() + ephemeral_public = ephemeral_private.public_key().public_bytes_raw() + recipient_public = x25519.X25519PublicKey.from_public_bytes(public_key_bytes) + shared_secret = ephemeral_private.exchange(recipient_public) + key = HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=None, + info=b"shadowbroker|dm-mls-welcome|v1", + ).derive(shared_secret) + nonce = secrets.token_bytes(12) + ciphertext = AESGCM(key).encrypt( + nonce, + payload, + b"shadowbroker|dm-mls-welcome|v1", + ) + return ephemeral_public + nonce + ciphertext + + +def _unseal_welcome_for_private_key(payload: bytes, private_key_text: str) -> bytes: + private_key_bytes = _decode_key_text(private_key_text) + if not private_key_bytes: + raise PrivacyCoreError("local DH secret unavailable for DM session acceptance") + if _NaclPrivateKey is not None and _NaclSealedBox is not None: + return _NaclSealedBox(_NaclPrivateKey(private_key_bytes)).decrypt(payload) + if len(payload) < 44: + raise PrivacyCoreError("sealed DM welcome is truncated") + ephemeral_public = x25519.X25519PublicKey.from_public_bytes(payload[:32]) + nonce = payload[32:44] + ciphertext = payload[44:] + private_key = x25519.X25519PrivateKey.from_private_bytes(private_key_bytes) + shared_secret = private_key.exchange(ephemeral_public) + key = HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=None, + info=b"shadowbroker|dm-mls-welcome|v1", + ).derive(shared_secret) + try: + return AESGCM(key).decrypt( + nonce, + ciphertext, + b"shadowbroker|dm-mls-welcome|v1", + ) + except Exception as exc: + raise PrivacyCoreError("sealed DM welcome decrypt failed") from exc + + +@dataclass +class _SessionBinding: + session_id: str + local_alias: str + remote_alias: str + role: str + session_handle: int + created_at: int + + +_ALIAS_IDENTITIES: dict[str, int] = {} +_ALIAS_BINDINGS: dict[str, dict[str, str]] = {} +_ALIAS_SEAL_KEYS: dict[str, dict[str, str]] = {} +_SESSIONS: dict[str, _SessionBinding] = {} +_DM_FORMAT_LOCKS: dict[str, str] = {} + + +def _default_state() -> dict[str, Any]: + return { + "version": 2, + "updated_at": 0, + "aliases": {}, + "alias_seal_keys": {}, + "sessions": {}, + "dm_format_locks": {}, + } + + +def _privacy_client() -> PrivacyCoreClient: + global _PRIVACY_CLIENT + if _PRIVACY_CLIENT is None: + _PRIVACY_CLIENT = PrivacyCoreClient.load() + return _PRIVACY_CLIENT + + +def _current_transport_tier() -> str: + return transport_tier_from_state(get_wormhole_state()) + + +def _require_private_transport() -> tuple[bool, str]: + current = _current_transport_tier() + if _TRANSPORT_TIER_ORDER.get(current, 0) < _TRANSPORT_TIER_ORDER["private_transitional"]: + return False, "DM MLS requires PRIVATE transport tier" + return True, current + + +def _serialize_session(binding: _SessionBinding) -> dict[str, Any]: + return { + "session_id": binding.session_id, + "local_alias": binding.local_alias, + "remote_alias": binding.remote_alias, + "role": binding.role, + "session_handle": int(binding.session_handle), + "created_at": int(binding.created_at), + } + + +def _binding_record(handle: int, public_bundle: bytes, binding_proof: str) -> dict[str, Any]: + return { + "handle": int(handle), + "public_bundle": _b64(public_bundle), + "binding_proof": str(binding_proof or ""), + } + + +def _load_state() -> None: + global _STATE_LOADED + with _STATE_LOCK: + if _STATE_LOADED: + return + # KNOWN LIMITATION: Persisted handles only survive when the privacy-core + # library instance is still alive in the same process. Full Rust-state + # export/import is deferred to a later sprint. + domain_path = DATA_DIR / STATE_DOMAIN / STATE_FILENAME + if not domain_path.exists() and STATE_FILE.exists(): + try: + legacy = read_secure_json(STATE_FILE, _default_state) + write_domain_json(STATE_DOMAIN, STATE_FILENAME, legacy) + STATE_FILE.unlink(missing_ok=True) + except Exception: + logger.warning( + "Legacy DM MLS state could not be decrypted — " + "discarding stale file and starting fresh" + ) + STATE_FILE.unlink(missing_ok=True) + raw = read_domain_json(STATE_DOMAIN, STATE_FILENAME, _default_state) + state = _default_state() + if isinstance(raw, dict): + state.update(raw) + + _ALIAS_IDENTITIES.clear() + _ALIAS_BINDINGS.clear() + for alias, payload in dict(state.get("aliases") or {}).items(): + alias_key = _normalize_alias(alias) + if not alias_key: + continue + if isinstance(payload, dict): + handle = int(payload.get("handle", 0) or 0) + public_bundle_b64 = str(payload.get("public_bundle", "") or "") + binding_proof = str(payload.get("binding_proof", "") or "") + else: + handle = int(payload or 0) + public_bundle_b64 = "" + binding_proof = "" + if handle <= 0 or not public_bundle_b64 or not binding_proof: + logger.warning("DM MLS alias binding missing proof; identity will be re-created") + continue + try: + public_bundle = _unb64(public_bundle_b64) + except Exception as exc: + logger.warning("DM MLS alias binding decode failed: %s", type(exc).__name__) + continue + ok, reason = verify_dm_alias_blob(alias_key, public_bundle, binding_proof) + if not ok: + logger.warning("DM MLS alias binding invalid: %s", reason) + continue + _ALIAS_IDENTITIES[alias_key] = handle + _ALIAS_BINDINGS[alias_key] = _binding_record(handle, public_bundle, binding_proof) + + _ALIAS_SEAL_KEYS.clear() + for alias, keypair in dict(state.get("alias_seal_keys") or {}).items(): + alias_key = _normalize_alias(alias) + pair = dict(keypair or {}) + public_key = str(pair.get("public_key", "") or "").strip().lower() + private_key = str(pair.get("private_key", "") or "").strip().lower() + if alias_key and public_key and private_key: + _ALIAS_SEAL_KEYS[alias_key] = { + "public_key": public_key, + "private_key": private_key, + } + + _SESSIONS.clear() + for session_id, payload in dict(state.get("sessions") or {}).items(): + if not isinstance(payload, dict): + continue + binding = _SessionBinding( + session_id=str(payload.get("session_id", session_id) or session_id), + local_alias=_normalize_alias(str(payload.get("local_alias", "") or "")), + remote_alias=_normalize_alias(str(payload.get("remote_alias", "") or "")), + role=str(payload.get("role", "initiator") or "initiator"), + session_handle=int(payload.get("session_handle", 0) or 0), + created_at=int(payload.get("created_at", 0) or 0), + ) + if ( + binding.session_id + and binding.session_handle > 0 + and binding.local_alias in _ALIAS_IDENTITIES + ): + _SESSIONS[binding.session_id] = binding + + _DM_FORMAT_LOCKS.clear() + for session_id, payload_format in dict(state.get("dm_format_locks") or {}).items(): + normalized = str(payload_format or "").strip().lower() + if normalized: + _DM_FORMAT_LOCKS[str(session_id or "")] = normalized + _STATE_LOADED = True + + +def _save_state() -> None: + with _STATE_LOCK: + write_domain_json( + STATE_DOMAIN, + STATE_FILENAME, + { + "version": 2, + "updated_at": int(time.time()), + "aliases": { + alias: dict(_ALIAS_BINDINGS.get(alias) or {}) + for alias, handle in _ALIAS_IDENTITIES.items() + if _ALIAS_BINDINGS.get(alias) + }, + "alias_seal_keys": { + alias: dict(keypair or {}) + for alias, keypair in _ALIAS_SEAL_KEYS.items() + }, + "sessions": { + session_id: _serialize_session(binding) + for session_id, binding in _SESSIONS.items() + }, + "dm_format_locks": dict(_DM_FORMAT_LOCKS), + }, + ) + STATE_FILE.unlink(missing_ok=True) + + +def reset_dm_mls_state(*, clear_privacy_core: bool = False, clear_persistence: bool = True) -> None: + global _PRIVACY_CLIENT, _STATE_LOADED + with _STATE_LOCK: + if clear_privacy_core and _PRIVACY_CLIENT is not None: + try: + _PRIVACY_CLIENT.reset_all_state() + except Exception: + logger.exception("privacy-core reset failed while clearing DM MLS state") + _ALIAS_IDENTITIES.clear() + _ALIAS_BINDINGS.clear() + _ALIAS_SEAL_KEYS.clear() + _SESSIONS.clear() + _DM_FORMAT_LOCKS.clear() + _STATE_LOADED = False + if clear_persistence and STATE_FILE.exists(): + STATE_FILE.unlink() + + +def _identity_handle_for_alias(alias: str) -> int: + alias_key = _normalize_alias(alias) + if not alias_key: + raise PrivacyCoreError("dm alias is required") + _load_state() + with _STATE_LOCK: + handle = _ALIAS_IDENTITIES.get(alias_key) + if handle: + return handle + handle = _privacy_client().create_identity() + public_bundle = _privacy_client().export_public_bundle(handle) + signed = sign_dm_alias_blob(alias_key, public_bundle) + if not signed.get("ok"): + try: + _privacy_client().release_identity(handle) + except Exception: + pass + raise PrivacyCoreError(str(signed.get("detail") or "dm_mls_identity_binding_failed")) + _ALIAS_IDENTITIES[alias_key] = handle + _ALIAS_BINDINGS[alias_key] = _binding_record( + handle, + public_bundle, + str(signed.get("signature", "") or ""), + ) + _save_state() + return handle + + +def _seal_keypair_for_alias(alias: str) -> dict[str, str]: + alias_key = _normalize_alias(alias) + if not alias_key: + raise PrivacyCoreError("dm alias is required") + _load_state() + with _STATE_LOCK: + existing = _ALIAS_SEAL_KEYS.get(alias_key) + if existing and existing.get("public_key") and existing.get("private_key"): + return dict(existing) + created = _seal_keypair() + _ALIAS_SEAL_KEYS[alias_key] = created + _save_state() + return dict(created) + + +def export_dm_key_package_for_alias(alias: str) -> dict[str, Any]: + alias_key = _normalize_alias(alias) + if not alias_key: + return {"ok": False, "detail": "alias is required"} + try: + identity_handle = _identity_handle_for_alias(alias_key) + key_package = _privacy_client().export_key_package(identity_handle) + seal_keypair = _seal_keypair_for_alias(alias_key) + return { + "ok": True, + "alias": alias_key, + "mls_key_package": _b64(key_package), + "welcome_dh_pub": str(seal_keypair.get("public_key", "") or ""), + } + except Exception: + logger.exception( + "dm mls key package export failed for %s", + privacy_log_label(alias_key, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_key_package_failed"} + + +def _remember_session(local_alias: str, remote_alias: str, *, role: str, session_handle: int) -> _SessionBinding: + binding = _SessionBinding( + session_id=_session_id(local_alias, remote_alias), + local_alias=_normalize_alias(local_alias), + remote_alias=_normalize_alias(remote_alias), + role=str(role or "initiator"), + session_handle=int(session_handle), + created_at=int(time.time()), + ) + with _STATE_LOCK: + existing = _SESSIONS.get(binding.session_id) + if existing is not None: + try: + _privacy_client().release_dm_session(session_handle) + except Exception: + pass + return existing + _SESSIONS[binding.session_id] = binding + _save_state() + return binding + + +def _forget_session(local_alias: str, remote_alias: str) -> _SessionBinding | None: + _load_state() + with _STATE_LOCK: + binding = _SESSIONS.pop(_session_id(local_alias, remote_alias), None) + _save_state() + return binding + + +def _lock_dm_format(local_alias: str, remote_alias: str, format_str: str) -> None: + _load_state() + with _STATE_LOCK: + _DM_FORMAT_LOCKS[_session_id(local_alias, remote_alias)] = str(format_str or "").strip().lower() + _save_state() + + +def is_dm_locked_to_mls(local_alias: str, remote_alias: str) -> bool: + _load_state() + return ( + str(_DM_FORMAT_LOCKS.get(_session_id(local_alias, remote_alias), "") or "").strip().lower() + == MLS_DM_FORMAT + ) + + +def _session_binding(local_alias: str, remote_alias: str) -> _SessionBinding: + _load_state() + session_id = _session_id(local_alias, remote_alias) + binding = _SESSIONS.get(session_id) + if binding is None: + raise PrivacyCoreError(f"dm session not found for {session_id}") + return binding + + +def initiate_dm_session( + local_alias: str, + remote_alias: str, + remote_prekey_bundle: dict, + responder_dh_pub: str = "", +) -> dict[str, Any]: + ok, detail = _require_private_transport() + if not ok: + return {"ok": False, "detail": detail} + local_key = _normalize_alias(local_alias) + remote_key = _normalize_alias(remote_alias) + remote_key_package_b64 = str( + (remote_prekey_bundle or {}).get("mls_key_package") + or (remote_prekey_bundle or {}).get("key_package") + or "" + ).strip() + if not local_key or not remote_key or not remote_key_package_b64: + return {"ok": False, "detail": "local_alias, remote_alias, and mls_key_package are required"} + resolved_responder_dh_pub = str( + responder_dh_pub + or (remote_prekey_bundle or {}).get("welcome_dh_pub") + or (remote_prekey_bundle or {}).get("identity_dh_pub_key") + or "" + ).strip() + key_package_handle = 0 + session_handle = 0 + remembered = False + try: + identity_handle = _identity_handle_for_alias(local_key) + key_package_handle = _privacy_client().import_key_package(_unb64(remote_key_package_b64)) + session_handle = _privacy_client().create_dm_session(identity_handle, key_package_handle) + welcome = _privacy_client().dm_session_welcome(session_handle) + sealed_welcome = _seal_welcome_for_public_key(welcome, resolved_responder_dh_pub) + binding = _remember_session(local_key, remote_key, role="initiator", session_handle=session_handle) + remembered = True + return {"ok": True, "welcome": _b64(sealed_welcome), "session_id": binding.session_id} + except Exception: + logger.exception( + "dm mls initiate failed for %s -> %s", + privacy_log_label(local_key, label="alias"), + privacy_log_label(remote_key, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_initiate_failed"} + finally: + if key_package_handle: + try: + _privacy_client().release_key_package(key_package_handle) + except Exception: + pass + if session_handle and not remembered: + try: + _privacy_client().release_dm_session(session_handle) + except Exception: + pass + + +def accept_dm_session( + local_alias: str, + remote_alias: str, + welcome_b64: str, + local_dh_secret: str = "", +) -> dict[str, Any]: + ok, detail = _require_private_transport() + if not ok: + return {"ok": False, "detail": detail} + local_key = _normalize_alias(local_alias) + remote_key = _normalize_alias(remote_alias) + if not local_key or not remote_key or not str(welcome_b64 or "").strip(): + return {"ok": False, "detail": "local_alias, remote_alias, and welcome are required"} + session_handle = 0 + remembered = False + try: + identity_handle = _identity_handle_for_alias(local_key) + seal_keypair = _seal_keypair_for_alias(local_key) + welcome = _unseal_welcome_for_private_key( + _unb64(welcome_b64), + str(local_dh_secret or seal_keypair.get("private_key") or ""), + ) + session_handle = _privacy_client().join_dm_session(identity_handle, welcome) + binding = _remember_session(local_key, remote_key, role="responder", session_handle=session_handle) + remembered = True + return {"ok": True, "session_id": binding.session_id} + except Exception: + logger.exception( + "dm mls accept failed for %s <- %s", + privacy_log_label(local_key, label="alias"), + privacy_log_label(remote_key, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_accept_failed"} + finally: + if session_handle and not remembered: + try: + _privacy_client().release_dm_session(session_handle) + except Exception: + pass + + +def has_dm_session(local_alias: str, remote_alias: str) -> dict[str, Any]: + ok, detail = _require_private_transport() + if not ok: + return {"ok": False, "detail": detail} + try: + binding = _session_binding(local_alias, remote_alias) + return {"ok": True, "exists": True, "session_id": binding.session_id} + except Exception: + return {"ok": True, "exists": False, "session_id": _session_id(local_alias, remote_alias)} + + +def ensure_dm_session(local_alias: str, remote_alias: str, welcome_b64: str) -> dict[str, Any]: + ok, detail = _require_private_transport() + if not ok: + return {"ok": False, "detail": detail} + has_session = has_dm_session(local_alias, remote_alias) + if not has_session.get("ok"): + return has_session + if has_session.get("exists"): + return {"ok": True, "session_id": _session_id(local_alias, remote_alias)} + return accept_dm_session(local_alias, remote_alias, welcome_b64) + + +def _session_expired_result(local_alias: str, remote_alias: str) -> dict[str, Any]: + binding = _forget_session(local_alias, remote_alias) + session_id = binding.session_id if binding is not None else _session_id(local_alias, remote_alias) + return {"ok": False, "detail": "session_expired", "session_id": session_id} + + +def encrypt_dm(local_alias: str, remote_alias: str, plaintext: str) -> dict[str, Any]: + ok, detail = _require_private_transport() + if not ok: + return {"ok": False, "detail": detail} + plaintext_bytes = str(plaintext or "").encode("utf-8") + if len(plaintext_bytes) > MAX_DM_PLAINTEXT_SIZE: + return {"ok": False, "detail": "plaintext exceeds maximum size"} + try: + binding = _session_binding(local_alias, remote_alias) + ciphertext = _privacy_client().dm_encrypt(binding.session_handle, plaintext_bytes) + _lock_dm_format(local_alias, remote_alias, MLS_DM_FORMAT) + return { + "ok": True, + "ciphertext": _b64(ciphertext), + # NOTE: nonce is generated for DM envelope compatibility with dm1 format. + # MLS handles its own nonce/IV internally — this field is not consumed by MLS. + "nonce": _b64(secrets.token_bytes(12)), + "session_id": binding.session_id, + } + except PrivacyCoreError as exc: + if "unknown dm session handle" in str(exc).lower(): + return _session_expired_result(local_alias, remote_alias) + logger.exception( + "dm mls encrypt failed for %s -> %s", + privacy_log_label(local_alias, label="alias"), + privacy_log_label(remote_alias, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_encrypt_failed"} + except Exception: + logger.exception( + "dm mls encrypt failed for %s -> %s", + privacy_log_label(local_alias, label="alias"), + privacy_log_label(remote_alias, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_encrypt_failed"} + + +def decrypt_dm(local_alias: str, remote_alias: str, ciphertext_b64: str, nonce_b64: str) -> dict[str, Any]: + ok, detail = _require_private_transport() + if not ok: + return {"ok": False, "detail": detail} + try: + binding = _session_binding(local_alias, remote_alias) + plaintext = _privacy_client().dm_decrypt(binding.session_handle, _unb64(ciphertext_b64)) + _lock_dm_format(local_alias, remote_alias, MLS_DM_FORMAT) + return { + "ok": True, + "plaintext": plaintext.decode("utf-8"), + "session_id": binding.session_id, + "nonce": str(nonce_b64 or ""), + } + except PrivacyCoreError as exc: + if "unknown dm session handle" in str(exc).lower(): + return _session_expired_result(local_alias, remote_alias) + logger.exception( + "dm mls decrypt failed for %s <- %s", + privacy_log_label(local_alias, label="alias"), + privacy_log_label(remote_alias, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_decrypt_failed"} + except Exception: + logger.exception( + "dm mls decrypt failed for %s <- %s", + privacy_log_label(local_alias, label="alias"), + privacy_log_label(remote_alias, label="alias"), + ) + return {"ok": False, "detail": "dm_mls_decrypt_failed"} diff --git a/backend/services/mesh/mesh_dm_relay.py b/backend/services/mesh/mesh_dm_relay.py new file mode 100644 index 00000000..d13b75a4 --- /dev/null +++ b/backend/services/mesh/mesh_dm_relay.py @@ -0,0 +1,824 @@ +"""Metadata-minimized DM relay for request and shared mailboxes. + +This relay never decrypts application payloads. In secure mode it keeps +pending ciphertext in memory only and persists just the minimum metadata +needed for continuity: accepted DH bundles, block lists, witness data, +and nonce replay windows. +""" + +from __future__ import annotations + +import atexit +import hashlib +import json +import logging +import os +import secrets +import threading +import time +from collections import OrderedDict, defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from services.config import get_settings +from services.mesh.mesh_metrics import increment as metrics_inc +from services.mesh.mesh_wormhole_prekey import _validate_bundle_record +from services.mesh.mesh_secure_storage import read_secure_json, write_secure_json + +TTL_SECONDS = 3600 +EPOCH_SECONDS = 6 * 60 * 60 +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +RELAY_FILE = DATA_DIR / "dm_relay.json" +logger = logging.getLogger(__name__) + + +def _get_token_pepper() -> str: + """Read token pepper lazily so auto-generated values from startup audit take effect.""" + pepper = os.environ.get("MESH_DM_TOKEN_PEPPER", "").strip() + if not pepper: + try: + from services.config import get_settings + from services.env_check import _ensure_dm_token_pepper + + pepper = _ensure_dm_token_pepper(get_settings()) + except Exception: + pepper = os.environ.get("MESH_DM_TOKEN_PEPPER", "").strip() + if not pepper: + raise RuntimeError("MESH_DM_TOKEN_PEPPER is unavailable at runtime") + return pepper + + +@dataclass +class DMMessage: + sender_id: str + ciphertext: str + timestamp: float + msg_id: str + delivery_class: str + sender_seal: str = "" + relay_salt: str = "" + sender_block_ref: str = "" + payload_format: str = "dm1" + session_welcome: str = "" + + +class DMRelay: + """Relay for encrypted request/shared mailboxes.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + self._mailboxes: dict[str, list[DMMessage]] = defaultdict(list) + self._dh_keys: dict[str, dict[str, Any]] = {} + self._prekey_bundles: dict[str, dict[str, Any]] = {} + self._mailbox_bindings: dict[str, dict[str, Any]] = defaultdict(dict) + self._witnesses: dict[str, list[dict[str, Any]]] = defaultdict(list) + self._blocks: dict[str, set[str]] = defaultdict(set) + self._nonce_cache: OrderedDict[str, float] = OrderedDict() + self._stats: dict[str, int] = {"messages_in_memory": 0} + self._dirty = False + self._save_timer: threading.Timer | None = None + self._SAVE_INTERVAL = 5.0 + atexit.register(self._flush) + self._load() + + def _settings(self): + return get_settings() + + def _persist_spool_enabled(self) -> bool: + return bool(self._settings().MESH_DM_PERSIST_SPOOL) + + def _request_mailbox_limit(self) -> int: + return max(1, int(self._settings().MESH_DM_REQUEST_MAILBOX_LIMIT)) + + def _shared_mailbox_limit(self) -> int: + return max(1, int(self._settings().MESH_DM_SHARED_MAILBOX_LIMIT)) + + def _self_mailbox_limit(self) -> int: + return max(1, int(self._settings().MESH_DM_SELF_MAILBOX_LIMIT)) + + def _nonce_ttl_seconds(self) -> int: + return max(30, int(self._settings().MESH_DM_NONCE_TTL_S)) + + def _nonce_cache_max_entries(self) -> int: + return max(1, int(getattr(self._settings(), "MESH_DM_NONCE_CACHE_MAX", 4096))) + + def _pepper_token(self, token: str) -> str: + material = token + pepper = _get_token_pepper() + if pepper: + material = f"{pepper}|{token}" + return hashlib.sha256(material.encode("utf-8")).hexdigest() + + def _sender_block_ref(self, sender_id: str) -> str: + sender = str(sender_id or "").strip() + if not sender: + return "" + return "ref:" + self._pepper_token(f"block|{sender}") + + def _canonical_blocked_id(self, blocked_id: str) -> str: + blocked = str(blocked_id or "").strip() + if not blocked: + return "" + if blocked.startswith("ref:"): + return blocked + return self._sender_block_ref(blocked) + + def _message_block_ref(self, message: DMMessage) -> str: + block_ref = str(getattr(message, "sender_block_ref", "") or "").strip() + if block_ref: + return block_ref + sender_id = str(message.sender_id or "").strip() + if not sender_id or sender_id.startswith("sealed:") or sender_id.startswith("sender_token:"): + return "" + return self._sender_block_ref(sender_id) + + def _mailbox_key(self, mailbox_type: str, mailbox_value: str, epoch: int | None = None) -> str: + if mailbox_type in {"self", "requests"}: + bucket = self._epoch_bucket() if epoch is None else int(epoch) + identifier = f"{mailbox_type}|{bucket}|{mailbox_value}" + else: + identifier = f"{mailbox_type}|{mailbox_value}" + return self._pepper_token(identifier) + + def _hashed_mailbox_token(self, token: str) -> str: + return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest() + + def _remember_mailbox_binding(self, agent_id: str, mailbox_type: str, token: str) -> str: + token_hash = self._hashed_mailbox_token(token) + self._mailbox_bindings[str(agent_id or "").strip()][str(mailbox_type or "").strip().lower()] = { + "token_hash": token_hash, + "last_used": time.time(), + } + self._save() + return token_hash + + def _bound_mailbox_key(self, agent_id: str, mailbox_type: str) -> str: + entry = self._mailbox_bindings.get(str(agent_id or "").strip(), {}).get( + str(mailbox_type or "").strip().lower(), + "", + ) + if isinstance(entry, dict): + return str(entry.get("token_hash", "") or "") + return str(entry or "") + + def _mailbox_keys_for_claim(self, agent_id: str, claim: dict[str, Any]) -> list[str]: + claim_type = str(claim.get("type", "")).strip().lower() + if claim_type == "shared": + token = str(claim.get("token", "")).strip() + if not token: + metrics_inc("dm_claim_invalid") + return [] + return [self._hashed_mailbox_token(token)] + if claim_type == "requests": + token = str(claim.get("token", "")).strip() + if token: + bound_key = self._remember_mailbox_binding(agent_id, "requests", token) + epoch = self._epoch_bucket() + return [ + bound_key, + self._mailbox_key("requests", agent_id, epoch), + self._mailbox_key("requests", agent_id, epoch - 1), + ] + metrics_inc("dm_claim_invalid") + return [] + if claim_type == "self": + token = str(claim.get("token", "")).strip() + if token: + bound_key = self._remember_mailbox_binding(agent_id, "self", token) + epoch = self._epoch_bucket() + return [ + bound_key, + self._mailbox_key("self", agent_id, epoch), + self._mailbox_key("self", agent_id, epoch - 1), + ] + metrics_inc("dm_claim_invalid") + return [] + metrics_inc("dm_claim_invalid") + return [] + + def mailbox_key_for_delivery( + self, + *, + recipient_id: str, + delivery_class: str, + recipient_token: str | None = None, + ) -> str: + delivery_class = str(delivery_class or "").strip().lower() + if delivery_class == "request": + bound_key = self._bound_mailbox_key(recipient_id, "requests") + if bound_key: + return bound_key + return self._mailbox_key("requests", str(recipient_id or "").strip()) + if delivery_class == "shared": + token = str(recipient_token or "").strip() + if not token: + raise ValueError("recipient_token required for shared delivery") + return self._hashed_mailbox_token(token) + raise ValueError("Unsupported delivery_class") + + def claim_mailbox_keys(self, agent_id: str, claims: list[dict[str, Any]]) -> list[str]: + keys: list[str] = [] + for claim in claims[:32]: + keys.extend(self._mailbox_keys_for_claim(agent_id, claim)) + return list(dict.fromkeys(keys)) + + def _legacy_mailbox_token(self, agent_id: str, epoch: int) -> str: + raw = f"sb_dm|{epoch}|{agent_id}".encode("utf-8") + return hashlib.sha256(raw).hexdigest() + + def _legacy_token_candidates(self, agent_id: str) -> list[str]: + epoch = self._epoch_bucket() + raw = [self._legacy_mailbox_token(agent_id, epoch), self._legacy_mailbox_token(agent_id, epoch - 1)] + peppered = [self._pepper_token(token) for token in raw] + return list(dict.fromkeys(peppered + raw)) + + def _save(self) -> None: + """Mark dirty and schedule a coalesced disk write.""" + self._dirty = True + if not RELAY_FILE.exists(): + self._flush() + return + with self._lock: + if self._save_timer is None or not self._save_timer.is_alive(): + self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush) + self._save_timer.daemon = True + self._save_timer.start() + + def _prune_stale_metadata(self) -> None: + """Remove expired DH keys, prekey bundles, and mailbox bindings.""" + now = time.time() + settings = self._settings() + key_ttl = max(1, int(getattr(settings, "MESH_DM_KEY_TTL_DAYS", 30) or 30)) * 86400 + binding_ttl = max(1, int(getattr(settings, "MESH_DM_BINDING_TTL_DAYS", 7) or 7)) * 86400 + + stale_keys = [ + aid for aid, entry in self._dh_keys.items() + if (now - float(entry.get("timestamp", 0) or 0)) > key_ttl + ] + for aid in stale_keys: + del self._dh_keys[aid] + + stale_bundles = [ + aid for aid, entry in self._prekey_bundles.items() + if (now - float(entry.get("updated_at", entry.get("timestamp", 0)) or 0)) > key_ttl + ] + for aid in stale_bundles: + del self._prekey_bundles[aid] + + stale_agents: list[str] = [] + for agent_id, kinds in self._mailbox_bindings.items(): + expired_kinds = [ + k for k, v in kinds.items() + if isinstance(v, dict) and (now - float(v.get("last_used", 0) or 0)) > binding_ttl + ] + for k in expired_kinds: + del kinds[k] + if not kinds: + stale_agents.append(agent_id) + for agent_id in stale_agents: + del self._mailbox_bindings[agent_id] + + def _metadata_persist_enabled(self) -> bool: + return bool(getattr(self._settings(), "MESH_DM_METADATA_PERSIST", True)) + + def _flush(self) -> None: + """Actually write to disk (called by timer or atexit).""" + if not self._dirty: + return + try: + self._prune_stale_metadata() + DATA_DIR.mkdir(parents=True, exist_ok=True) + payload: dict[str, Any] = { + "saved_at": int(time.time()), + "dh_keys": self._dh_keys, + "prekey_bundles": self._prekey_bundles, + "witnesses": self._witnesses, + "blocks": {k: sorted(v) for k, v in self._blocks.items()}, + "nonce_cache": dict(self._nonce_cache), + "stats": self._stats, + } + if self._metadata_persist_enabled(): + payload["mailbox_bindings"] = self._mailbox_bindings + if self._persist_spool_enabled(): + payload["mailboxes"] = { + key: [m.__dict__ for m in msgs] for key, msgs in self._mailboxes.items() + } + write_secure_json(RELAY_FILE, payload) + self._dirty = False + except Exception: + pass + + def _load(self) -> None: + if not RELAY_FILE.exists(): + return + try: + data = read_secure_json(RELAY_FILE, lambda: {}) + except Exception: + return + if self._persist_spool_enabled(): + mailboxes = data.get("mailboxes", {}) + if isinstance(mailboxes, dict): + for key, items in mailboxes.items(): + if not isinstance(items, list): + continue + restored: list[DMMessage] = [] + for item in items: + try: + restored.append( + DMMessage( + sender_id=str(item.get("sender_id", "")), + ciphertext=str(item.get("ciphertext", "")), + timestamp=float(item.get("timestamp", 0)), + msg_id=str(item.get("msg_id", "")), + delivery_class=str(item.get("delivery_class", "shared")), + sender_seal=str(item.get("sender_seal", "")), + relay_salt=str(item.get("relay_salt", "") or ""), + sender_block_ref=str(item.get("sender_block_ref", "") or ""), + payload_format=str(item.get("payload_format", item.get("format", "dm1")) or "dm1"), + session_welcome=str(item.get("session_welcome", "") or ""), + ) + ) + except Exception: + continue + for message in restored: + if not message.sender_block_ref: + message.sender_block_ref = self._message_block_ref(message) + if restored: + self._mailboxes[key] = restored + dh_keys = data.get("dh_keys", {}) + if isinstance(dh_keys, dict): + self._dh_keys = {str(k): dict(v) for k, v in dh_keys.items() if isinstance(v, dict)} + prekey_bundles = data.get("prekey_bundles", {}) + if isinstance(prekey_bundles, dict): + self._prekey_bundles = { + str(k): dict(v) for k, v in prekey_bundles.items() if isinstance(v, dict) + } + mailbox_bindings = data.get("mailbox_bindings", {}) + if isinstance(mailbox_bindings, dict): + self._mailbox_bindings = defaultdict( + dict, + { + str(agent_id): { + str(kind): str(token_hash) + for kind, token_hash in dict(bindings or {}).items() + if str(token_hash or "").strip() + } + for agent_id, bindings in mailbox_bindings.items() + if isinstance(bindings, dict) + }, + ) + witnesses = data.get("witnesses", {}) + if isinstance(witnesses, dict): + self._witnesses = defaultdict( + list, + { + str(k): list(v) + for k, v in witnesses.items() + if isinstance(v, list) + }, + ) + blocks = data.get("blocks", {}) + if isinstance(blocks, dict): + for key, values in blocks.items(): + if isinstance(values, list): + self._blocks[str(key)] = { + self._canonical_blocked_id(str(v)) + for v in values + if str(v or "").strip() + } + nonce_cache = data.get("nonce_cache", {}) + if isinstance(nonce_cache, dict): + now = time.time() + restored = sorted( + ( + (str(k), float(v)) + for k, v in nonce_cache.items() + if float(v) > now + ), + key=lambda item: item[1], + ) + self._nonce_cache = OrderedDict(restored) + stats = data.get("stats", {}) + if isinstance(stats, dict): + self._stats = {str(k): int(v) for k, v in stats.items() if isinstance(v, (int, float))} + self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) + + def _bundle_fingerprint( + self, + *, + dh_pub_key: str, + dh_algo: str, + public_key: str, + public_key_algo: str, + protocol_version: str, + ) -> str: + material = "|".join( + [ + dh_pub_key, + dh_algo, + public_key, + public_key_algo, + protocol_version, + ] + ) + return hashlib.sha256(material.encode("utf-8")).hexdigest() + + def register_dh_key( + self, + agent_id: str, + dh_pub_key: str, + dh_algo: str, + timestamp: int, + signature: str, + public_key: str, + public_key_algo: str, + protocol_version: str, + sequence: int, + ) -> tuple[bool, str, dict[str, Any] | None]: + """Register/update an agent's DH public key bundle with replay protection.""" + fingerprint = self._bundle_fingerprint( + dh_pub_key=dh_pub_key, + dh_algo=dh_algo, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + with self._lock: + existing = self._dh_keys.get(agent_id) + if existing: + existing_seq = int(existing.get("sequence", 0) or 0) + existing_ts = int(existing.get("timestamp", 0) or 0) + if sequence <= existing_seq: + metrics_inc("dm_key_replay") + return False, "DM key replay or rollback rejected", None + if timestamp < existing_ts: + metrics_inc("dm_key_stale") + return False, "DM key timestamp is older than the current bundle", None + self._dh_keys[agent_id] = { + "dh_pub_key": dh_pub_key, + "dh_algo": dh_algo, + "timestamp": timestamp, + "signature": signature, + "public_key": public_key, + "public_key_algo": public_key_algo, + "protocol_version": protocol_version, + "sequence": sequence, + "bundle_fingerprint": fingerprint, + } + self._save() + return True, "ok", { + "accepted_sequence": sequence, + "bundle_fingerprint": fingerprint, + } + + def get_dh_key(self, agent_id: str) -> dict[str, Any] | None: + return self._dh_keys.get(agent_id) + + def register_prekey_bundle( + self, + agent_id: str, + bundle: dict[str, Any], + signature: str, + public_key: str, + public_key_algo: str, + protocol_version: str, + sequence: int, + ) -> tuple[bool, str, dict[str, Any] | None]: + ok, reason = _validate_bundle_record( + { + "bundle": bundle, + "public_key": public_key, + "agent_id": agent_id, + } + ) + if not ok: + return False, reason, None + with self._lock: + existing = self._prekey_bundles.get(agent_id) + if existing: + existing_seq = int(existing.get("sequence", 0) or 0) + if sequence <= existing_seq: + return False, "Prekey bundle replay or rollback rejected", None + stored = { + "bundle": dict(bundle or {}), + "signature": signature, + "public_key": public_key, + "public_key_algo": public_key_algo, + "protocol_version": protocol_version, + "sequence": int(sequence), + "updated_at": int(time.time()), + } + self._prekey_bundles[agent_id] = stored + self._save() + return True, "ok", {"accepted_sequence": int(sequence)} + + def get_prekey_bundle(self, agent_id: str) -> dict[str, Any] | None: + stored = self._prekey_bundles.get(agent_id) + if not stored: + return None + return dict(stored) + + def consume_one_time_prekey(self, agent_id: str) -> dict[str, Any] | None: + """Atomically claim the next published one-time prekey for a peer bundle.""" + claimed: dict[str, Any] | None = None + with self._lock: + stored = self._prekey_bundles.get(agent_id) + if not stored: + return None + bundle = dict(stored.get("bundle") or {}) + otks = list(bundle.get("one_time_prekeys") or []) + if not otks: + return dict(stored) + claimed = dict(otks.pop(0) or {}) + bundle["one_time_prekeys"] = otks + bundle["one_time_prekey_count"] = len(otks) + stored = dict(stored) + stored["bundle"] = bundle + stored["updated_at"] = int(time.time()) + self._prekey_bundles[agent_id] = stored + self._save() + result = dict(stored) + result["claimed_one_time_prekey"] = claimed + return result + + def _prune_witnesses(self, target_id: str, ttl_days: int = 30) -> None: + cutoff = time.time() - (ttl_days * 86400) + self._witnesses[target_id] = [ + w for w in self._witnesses.get(target_id, []) if float(w.get("timestamp", 0)) >= cutoff + ] + if not self._witnesses[target_id]: + del self._witnesses[target_id] + + def record_witness( + self, + witness_id: str, + target_id: str, + dh_pub_key: str, + timestamp: int, + ) -> tuple[bool, str]: + if not witness_id or not target_id or not dh_pub_key: + return False, "Missing witness_id, target_id, or dh_pub_key" + if witness_id == target_id: + return False, "Cannot witness yourself" + with self._lock: + self._prune_witnesses(target_id) + entries = self._witnesses.get(target_id, []) + for entry in entries: + if entry.get("witness_id") == witness_id and entry.get("dh_pub_key") == dh_pub_key: + return False, "Duplicate witness" + entries.append( + { + "witness_id": witness_id, + "dh_pub_key": dh_pub_key, + "timestamp": int(timestamp), + } + ) + self._witnesses[target_id] = entries[-50:] + self._save() + return True, "ok" + + def get_witnesses(self, target_id: str, dh_pub_key: str | None = None, limit: int = 5) -> list[dict]: + with self._lock: + self._prune_witnesses(target_id) + entries = list(self._witnesses.get(target_id, [])) + if dh_pub_key: + entries = [e for e in entries if e.get("dh_pub_key") == dh_pub_key] + entries = sorted(entries, key=lambda e: e.get("timestamp", 0), reverse=True) + return entries[: max(1, limit)] + + def _epoch_bucket(self, ts: float | None = None) -> int: + now = ts if ts is not None else time.time() + return int(now // EPOCH_SECONDS) + + def _mailbox_limit_for_class(self, delivery_class: str) -> int: + if delivery_class == "request": + return self._request_mailbox_limit() + if delivery_class == "shared": + return self._shared_mailbox_limit() + return self._self_mailbox_limit() + + def _cleanup_expired(self) -> bool: + now = time.time() + changed = False + for mailbox_id in list(self._mailboxes): + fresh = [m for m in self._mailboxes[mailbox_id] if now - m.timestamp < TTL_SECONDS] + if len(fresh) != len(self._mailboxes[mailbox_id]): + changed = True + self._mailboxes[mailbox_id] = fresh + if not self._mailboxes[mailbox_id]: + del self._mailboxes[mailbox_id] + changed = True + self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) + return changed + + def consume_nonce(self, agent_id: str, nonce: str, timestamp: int) -> tuple[bool, str]: + nonce = str(nonce or "").strip() + if not nonce: + return False, "Missing nonce" + now = time.time() + with self._lock: + self._nonce_cache = OrderedDict( + (key, expiry) + for key, expiry in self._nonce_cache.items() + if float(expiry) > now + ) + key = f"{agent_id}:{nonce}" + if key in self._nonce_cache: + metrics_inc("dm_nonce_replay") + return False, "nonce replay detected" + if len(self._nonce_cache) >= self._nonce_cache_max_entries(): + metrics_inc("dm_nonce_cache_full") + return False, "nonce cache at capacity" + expiry = max(now + self._nonce_ttl_seconds(), float(timestamp) + self._nonce_ttl_seconds()) + self._nonce_cache[key] = expiry + self._nonce_cache.move_to_end(key) + self._save() + return True, "ok" + + def deposit( + self, + *, + sender_id: str, + raw_sender_id: str = "", + recipient_id: str = "", + ciphertext: str, + msg_id: str = "", + delivery_class: str, + recipient_token: str | None = None, + sender_seal: str = "", + relay_salt: str = "", + sender_token_hash: str = "", + payload_format: str = "dm1", + session_welcome: str = "", + ) -> dict[str, Any]: + with self._lock: + authority_sender = str(raw_sender_id or sender_id or "").strip() + sender_block_ref = self._sender_block_ref(authority_sender) + if recipient_id and sender_block_ref in self._blocks.get(recipient_id, set()): + metrics_inc("dm_drop_blocked") + return {"ok": False, "detail": "Recipient is not accepting your messages"} + if len(ciphertext) > int(self._settings().MESH_DM_MAX_MSG_BYTES): + metrics_inc("dm_drop_oversize") + return { + "ok": False, + "detail": f"Message too large ({len(ciphertext)} > {int(self._settings().MESH_DM_MAX_MSG_BYTES)})", + } + self._cleanup_expired() + if delivery_class == "request": + mailbox_key = self._mailbox_key("requests", recipient_id) + elif delivery_class == "shared": + if not recipient_token: + metrics_inc("dm_claim_invalid") + return {"ok": False, "detail": "recipient_token required for shared delivery"} + mailbox_key = self._hashed_mailbox_token(recipient_token) + else: + return {"ok": False, "detail": "Unsupported delivery_class"} + if len(self._mailboxes[mailbox_key]) >= self._mailbox_limit_for_class(delivery_class): + metrics_inc("dm_drop_full") + return {"ok": False, "detail": "Recipient mailbox full"} + if not msg_id: + msg_id = f"dm_{int(time.time() * 1000)}_{secrets.token_hex(6)}" + elif any(m.msg_id == msg_id for m in self._mailboxes[mailbox_key]): + return {"ok": True, "msg_id": msg_id} + relay_sender_id = ( + f"sender_token:{sender_token_hash}" + if sender_token_hash and delivery_class == "shared" + else sender_id + ) + self._mailboxes[mailbox_key].append( + DMMessage( + sender_id=relay_sender_id, + ciphertext=ciphertext, + timestamp=time.time(), + msg_id=msg_id, + delivery_class=delivery_class, + sender_seal=sender_seal, + sender_block_ref=sender_block_ref, + payload_format=str(payload_format or "dm1"), + session_welcome=str(session_welcome or ""), + ) + ) + self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) + self._save() + return {"ok": True, "msg_id": msg_id} + + def is_blocked(self, recipient_id: str, sender_id: str) -> bool: + with self._lock: + blocked_ref = self._sender_block_ref(sender_id) + if not recipient_id or not blocked_ref: + return False + return blocked_ref in self._blocks.get(recipient_id, set()) + + def _collect_from_keys(self, keys: list[str], *, destructive: bool) -> list[dict[str, Any]]: + messages: list[DMMessage] = [] + seen: set[str] = set() + for key in keys: + mailbox = self._mailboxes.pop(key, []) if destructive else list(self._mailboxes.get(key, [])) + for message in mailbox: + if message.msg_id in seen: + continue + seen.add(message.msg_id) + messages.append(message) + if destructive: + self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) + self._save() + return [ + { + "sender_id": message.sender_id, + "ciphertext": message.ciphertext, + "timestamp": message.timestamp, + "msg_id": message.msg_id, + "delivery_class": message.delivery_class, + "sender_seal": message.sender_seal, + "format": message.payload_format, + "session_welcome": message.session_welcome, + } + for message in sorted(messages, key=lambda item: item.timestamp) + ] + + def collect_claims(self, agent_id: str, claims: list[dict[str, Any]]) -> list[dict[str, Any]]: + with self._lock: + self._cleanup_expired() + keys: list[str] = [] + for claim in claims[:32]: + keys.extend(self._mailbox_keys_for_claim(agent_id, claim)) + return self._collect_from_keys(list(dict.fromkeys(keys)), destructive=True) + + def count_claims(self, agent_id: str, claims: list[dict[str, Any]]) -> int: + with self._lock: + self._cleanup_expired() + keys: list[str] = [] + for claim in claims[:32]: + keys.extend(self._mailbox_keys_for_claim(agent_id, claim)) + messages = self._collect_from_keys(list(dict.fromkeys(keys)), destructive=False) + return len(messages) + + def claim_message_ids(self, agent_id: str, claims: list[dict[str, Any]]) -> set[str]: + with self._lock: + self._cleanup_expired() + keys: list[str] = [] + for claim in claims[:32]: + keys.extend(self._mailbox_keys_for_claim(agent_id, claim)) + messages = self._collect_from_keys(list(dict.fromkeys(keys)), destructive=False) + return { + str(message.get("msg_id", "") or "") + for message in messages + if str(message.get("msg_id", "") or "") + } + + def collect_legacy(self, agent_id: str | None = None, agent_token: str | None = None) -> list[dict[str, Any]]: + with self._lock: + self._cleanup_expired() + if not agent_token: + return [] + keys = [self._pepper_token(agent_token), agent_token] + return self._collect_from_keys(list(dict.fromkeys(keys)), destructive=True) + + def count_legacy(self, agent_id: str | None = None, agent_token: str | None = None) -> int: + with self._lock: + self._cleanup_expired() + if not agent_token: + return 0 + keys = [self._pepper_token(agent_token), agent_token] + return len(self._collect_from_keys(list(dict.fromkeys(keys)), destructive=False)) + + def block(self, agent_id: str, blocked_id: str) -> None: + with self._lock: + blocked_ref = self._canonical_blocked_id(blocked_id) + if not blocked_ref: + return + self._blocks[agent_id].add(blocked_ref) + purge_keys = self._legacy_token_candidates(agent_id) + bound_request = self._bound_mailbox_key(agent_id, "requests") + bound_self = self._bound_mailbox_key(agent_id, "self") + if bound_request: + purge_keys.append(bound_request) + if bound_self: + purge_keys.append(bound_self) + purge_keys.extend( + [ + self._mailbox_key("self", agent_id), + self._mailbox_key("requests", agent_id), + self._mailbox_key("self", agent_id, self._epoch_bucket() - 1), + self._mailbox_key("requests", agent_id, self._epoch_bucket() - 1), + ] + ) + for key in set(purge_keys): + if key in self._mailboxes: + self._mailboxes[key] = [ + m for m in self._mailboxes[key] if self._message_block_ref(m) != blocked_ref + ] + self._stats["messages_in_memory"] = sum(len(v) for v in self._mailboxes.values()) + self._save() + + def unblock(self, agent_id: str, blocked_id: str) -> None: + with self._lock: + blocked_ref = self._canonical_blocked_id(blocked_id) + if not blocked_ref: + return + self._blocks[agent_id].discard(blocked_ref) + self._save() + + +dm_relay = DMRelay() diff --git a/backend/services/mesh/mesh_gate_mls.py b/backend/services/mesh/mesh_gate_mls.py new file mode 100644 index 00000000..86227d41 --- /dev/null +++ b/backend/services/mesh/mesh_gate_mls.py @@ -0,0 +1,1352 @@ +"""MLS-backed gate confidentiality path. + +Gate encryption now routes exclusively through privacy-core. This module keeps +the gate -> MLS mapping and confidentiality state in Python while Rust owns the +actual MLS group state. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import logging +import math +import secrets +import struct +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from services.mesh.mesh_secure_storage import ( + read_domain_json, + read_secure_json, + write_domain_json, +) +from services.mesh.mesh_privacy_logging import privacy_log_label +from services.mesh.mesh_wormhole_persona import ( + bootstrap_wormhole_persona_state, + get_active_gate_identity, + read_wormhole_persona_state, + sign_gate_persona_blob, + sign_gate_session_blob, + sign_gate_wormhole_event, + verify_gate_persona_blob, + verify_gate_session_blob, +) +from services.privacy_core_client import PrivacyCoreClient, PrivacyCoreError + +logger = logging.getLogger(__name__) + +import os as _os +from cryptography.hazmat.primitives.ciphers.aead import AESGCM as _AESGCM + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +STATE_FILE = DATA_DIR / "wormhole_gate_mls.json" +STATE_FILENAME = "wormhole_gate_mls.json" +STATE_DOMAIN = "gate_persona" +MLS_GATE_FORMAT = "mls1" +# Gate-scoped symmetric encryption domain — used for the durable envelope +# that survives MLS group rebuilds / process restarts. The key is the same +# domain key that protects the gate_persona store (AES-256-GCM, stored in an +# OS-protected key envelope). Gate members can always decrypt; outsiders +# cannot because they lack the domain key. +_GATE_ENVELOPE_DOMAIN = "gate_persona" + + +def _gate_envelope_key() -> bytes: + """Return the 256-bit AES key for gate envelope encryption.""" + from services.mesh.mesh_secure_storage import _load_domain_key # type: ignore[attr-defined] + return _load_domain_key(_GATE_ENVELOPE_DOMAIN) + + +def _gate_envelope_encrypt(gate_id: str, plaintext: str) -> str: + """Encrypt plaintext under the gate domain key. Returns base64.""" + key = _gate_envelope_key() + nonce = _os.urandom(12) + aad = f"gate_envelope|{gate_id}".encode("utf-8") + ct = _AESGCM(key).encrypt(nonce, plaintext.encode("utf-8"), aad) + return base64.b64encode(nonce + ct).decode("ascii") + + +def _gate_envelope_decrypt(gate_id: str, token: str) -> str | None: + """Decrypt a gate envelope token. Returns plaintext or None on failure.""" + try: + raw = base64.b64decode(token) + if len(raw) < 13: + return None + nonce, ct = raw[:12], raw[12:] + key = _gate_envelope_key() + aad = f"gate_envelope|{gate_id}".encode("utf-8") + return _AESGCM(key).decrypt(nonce, ct, aad).decode("utf-8") + except Exception: + return None +# Self-echo plaintext cache: MLS cannot decrypt messages authored by the same +# member, so we cache plaintext locally after compose. The TTL must comfortably +# exceed the frontend poll + batch-decrypt round-trip (often 2-5 s under load). +# 300 s keeps self-authored messages readable for the whole session while still +# bounding memory exposure. +LOCAL_CIPHERTEXT_CACHE_MAX = 128 +LOCAL_CIPHERTEXT_CACHE_TTL_S = 300 +_CT_BUCKETS = (192, 384, 768, 1536, 3072, 6144) + + +class _ComposeResult(dict[str, Any]): + """Dict response with hidden legacy epoch access for in-process callers/tests.""" + + def __init__(self, *args: Any, legacy_epoch: int = 0, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._legacy_epoch = int(legacy_epoch or 0) + + def __getitem__(self, key: str) -> Any: + if key == "epoch": + return self._legacy_epoch + return super().__getitem__(key) + + def get(self, key: str, default: Any = None) -> Any: + if key == "epoch": + return self._legacy_epoch + return super().get(key, default) + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _pad_ciphertext_raw(raw_ct: bytes) -> bytes: + """Length-prefix + pad raw ciphertext to the next bucket size.""" + prefixed = struct.pack(">H", len(raw_ct)) + raw_ct + prefixed_len = len(prefixed) + for bucket in _CT_BUCKETS: + if prefixed_len <= bucket: + return prefixed + (b"\x00" * (bucket - prefixed_len)) + target = (((prefixed_len - 1) // _CT_BUCKETS[-1]) + 1) * _CT_BUCKETS[-1] + return prefixed + (b"\x00" * (target - prefixed_len)) + + +def _unpad_ciphertext_raw(padded: bytes) -> bytes: + """Read length prefix and extract original ciphertext.""" + if len(padded) < 2: + return padded + original_len = struct.unpack(">H", padded[:2])[0] + if original_len == 0 or 2 + original_len > len(padded): + return padded + return padded[2 : 2 + original_len] + + +def _stable_gate_ref(gate_id: str) -> str: + return str(gate_id or "").strip().lower() + + +def _sender_ref_seed(identity: dict[str, Any]) -> str: + return str(identity.get("persona_id", "") or identity.get("node_id", "") or "").strip() + + +def _sender_ref(persona_id: str, msg_id: str) -> str: + persona_key = str(persona_id or "").strip() + message_id = str(msg_id or "").strip() + if not persona_key or not message_id: + return "" + return hmac.new( + persona_key.encode("utf-8"), + message_id.encode("utf-8"), + hashlib.sha256, + ).hexdigest()[:16] + + +@dataclass +class _GateMemberBinding: + persona_id: str + node_id: str + label: str + identity_scope: str + identity_handle: int + group_handle: int + member_ref: int + is_creator: bool = False + key_package_handle: int | None = None + public_bundle: bytes = b"" + binding_signature: str = "" + + +@dataclass +class _GateBinding: + gate_id: str + epoch: int + root_persona_id: str + root_group_handle: int + next_member_ref: int = 1 + members: dict[str, _GateMemberBinding] = field(default_factory=dict) + + +_STATE_LOCK = threading.RLock() +_PRIVACY_CLIENT: PrivacyCoreClient | None = None +# MLS limitation: Rust group state (ratchet trees, group secrets) is in-memory only. +# Python-side metadata (bindings, epochs, personas) is persisted via domain storage. +# Process restart requires group re-join. Rust FFI state export is still deferred. +_GATE_BINDINGS: dict[str, _GateBinding] = {} +_LOCAL_CIPHERTEXT_CACHE: OrderedDict[ + tuple[str, str, str], + tuple[str, float], +] = OrderedDict() +_HIGH_WATER_EPOCHS: dict[str, int] = {} + + +def _default_binding_store() -> dict[str, Any]: + return { + "version": 1, + "updated_at": 0, + "gates": {}, + "high_water_epochs": {}, + "gate_format_locks": {}, + } + + +def _privacy_client() -> PrivacyCoreClient: + global _PRIVACY_CLIENT + if _PRIVACY_CLIENT is None: + _PRIVACY_CLIENT = PrivacyCoreClient.load() + return _PRIVACY_CLIENT + + +def reset_gate_mls_state() -> None: + """Test helper for clearing in-memory gate -> MLS bindings.""" + + global _PRIVACY_CLIENT + with _STATE_LOCK: + if _PRIVACY_CLIENT is not None: + try: + _PRIVACY_CLIENT.reset_all_state() + except Exception: + logger.exception("privacy-core reset failed while clearing gate MLS state") + _GATE_BINDINGS.clear() + _LOCAL_CIPHERTEXT_CACHE.clear() + _HIGH_WATER_EPOCHS.clear() + + +def _gate_personas(gate_id: str) -> list[dict[str, Any]]: + gate_key = _stable_gate_ref(gate_id) + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + return [dict(item or {}) for item in list(state.get("gate_personas", {}).get(gate_key) or [])] + + +def _gate_session_identity(gate_id: str) -> dict[str, Any] | None: + gate_key = _stable_gate_ref(gate_id) + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + session = dict(state.get("gate_sessions", {}).get(gate_key) or {}) + if not session.get("private_key"): + return None + return session + + +def _gate_member_identity_id(identity: dict[str, Any]) -> str: + persona_id = str(identity.get("persona_id", "") or "").strip() + if persona_id: + return persona_id + node_id = str(identity.get("node_id", "") or "").strip() + if not node_id: + raise PrivacyCoreError("gate member identity requires node_id") + return f"session:{node_id}" + + +def _gate_member_identity_scope(identity: dict[str, Any]) -> str: + scope = str(identity.get("scope", "") or "").strip().lower() + if scope == "gate_persona": + return "persona" + return "anonymous" + + +def _active_gate_member(gate_id: str) -> tuple[dict[str, Any] | None, str]: + active = get_active_gate_identity(gate_id) + if not active.get("ok"): + return None, "" + return dict(active.get("identity") or {}), str(active.get("source", "") or "") + + +def _active_gate_persona(gate_id: str) -> dict[str, Any] | None: + active = get_active_gate_identity(gate_id) + if not active.get("ok") or str(active.get("source", "") or "") != "persona": + return None + return dict(active.get("identity") or {}) + + +def _prune_local_plaintext_cache(now: float) -> None: + expired_keys = [ + key + for key, (_plaintext, inserted_at) in _LOCAL_CIPHERTEXT_CACHE.items() + if now - inserted_at > LOCAL_CIPHERTEXT_CACHE_TTL_S + ] + for key in expired_keys: + _LOCAL_CIPHERTEXT_CACHE.pop(key, None) + + +def _cache_local_plaintext(gate_id: str, ciphertext: str, sender_ref: str, plaintext: str) -> None: + now = time.time() + cache_key = (gate_id, ciphertext, sender_ref) + with _STATE_LOCK: + _prune_local_plaintext_cache(now) + if cache_key not in _LOCAL_CIPHERTEXT_CACHE and len(_LOCAL_CIPHERTEXT_CACHE) >= LOCAL_CIPHERTEXT_CACHE_MAX: + _LOCAL_CIPHERTEXT_CACHE.popitem(last=False) + _LOCAL_CIPHERTEXT_CACHE[cache_key] = (plaintext, now) + _LOCAL_CIPHERTEXT_CACHE.move_to_end(cache_key) + + +def _consume_cached_plaintext(gate_id: str, ciphertext: str, sender_ref: str) -> str | None: + """Non-destructive read so repeated decrypt polls still find the entry.""" + now = time.time() + cache_key = (gate_id, ciphertext, sender_ref) + with _STATE_LOCK: + _prune_local_plaintext_cache(now) + entry = _LOCAL_CIPHERTEXT_CACHE.get(cache_key) + if entry is None: + return None + plaintext, inserted_at = entry + if now - inserted_at > LOCAL_CIPHERTEXT_CACHE_TTL_S: + _LOCAL_CIPHERTEXT_CACHE.pop(cache_key, None) + return None + _LOCAL_CIPHERTEXT_CACHE.move_to_end(cache_key) + return plaintext + + +def _peek_cached_plaintext(gate_id: str, ciphertext: str, sender_ref: str) -> str | None: + now = time.time() + cache_key = (gate_id, ciphertext, sender_ref) + with _STATE_LOCK: + _prune_local_plaintext_cache(now) + entry = _LOCAL_CIPHERTEXT_CACHE.get(cache_key) + if entry is None: + return None + plaintext, inserted_at = entry + if now - inserted_at > LOCAL_CIPHERTEXT_CACHE_TTL_S: + _LOCAL_CIPHERTEXT_CACHE.pop(cache_key, None) + return None + _LOCAL_CIPHERTEXT_CACHE.move_to_end(cache_key) + return plaintext + + +def _load_binding_store() -> dict[str, Any]: + # KNOWN LIMITATION: Persistence integrity depends on the gate_persona domain key. + # Cross-domain compromise no longer follows from a single derived root key, + # but any process that can read this domain's key envelope can still forge this file. + domain_path = DATA_DIR / STATE_DOMAIN / STATE_FILENAME + if not domain_path.exists() and STATE_FILE.exists(): + try: + legacy = read_secure_json(STATE_FILE, _default_binding_store) + write_domain_json(STATE_DOMAIN, STATE_FILENAME, legacy) + STATE_FILE.unlink(missing_ok=True) + except Exception: + logger.warning( + "Legacy gate MLS binding store could not be decrypted — " + "discarding stale file and starting fresh" + ) + STATE_FILE.unlink(missing_ok=True) + raw = read_domain_json(STATE_DOMAIN, STATE_FILENAME, _default_binding_store) + state = _default_binding_store() + if isinstance(raw, dict): + state.update(raw) + state["version"] = int(state.get("version", 1) or 1) + state["updated_at"] = int(state.get("updated_at", 0) or 0) + state["gates"] = { + _stable_gate_ref(gate_id): dict(item or {}) + for gate_id, item in dict(state.get("gates") or {}).items() + } + state["high_water_epochs"] = { + _stable_gate_ref(gate_id): int(epoch or 0) + for gate_id, epoch in dict(state.get("high_water_epochs") or {}).items() + } + state["gate_format_locks"] = { + _stable_gate_ref(gate_id): str(payload_format or "").strip().lower() + for gate_id, payload_format in dict(state.get("gate_format_locks") or {}).items() + if str(payload_format or "").strip().lower() + } + return state + + +def _save_binding_store(state: dict[str, Any]) -> None: + # KNOWN LIMITATION: Persistence integrity depends on the gate_persona domain key. + # Cross-domain compromise no longer follows from a single derived root key, + # but any process that can read this domain's key envelope can still forge this file. + payload = dict(state) + payload["updated_at"] = int(time.time()) + write_domain_json(STATE_DOMAIN, STATE_FILENAME, payload) + STATE_FILE.unlink(missing_ok=True) + + +def _serialize_member_binding(member: _GateMemberBinding) -> dict[str, Any]: + return { + "persona_id": member.persona_id, + "node_id": member.node_id, + "label": member.label, + "identity_scope": member.identity_scope, + "member_ref": int(member.member_ref), + "is_creator": bool(member.is_creator), + "public_bundle": _b64(member.public_bundle), + "binding_signature": member.binding_signature, + } + + +def _persist_binding(binding: _GateBinding) -> None: + for persona_id, member in binding.members.items(): + if member.identity_scope == "anonymous": + ok, reason = verify_gate_session_blob( + binding.gate_id, + member.node_id, + member.public_bundle, + member.binding_signature, + ) + else: + ok, reason = verify_gate_persona_blob( + binding.gate_id, + persona_id, + member.public_bundle, + member.binding_signature, + ) + if not ok: + logger.warning( + "Skipping MLS binding persistence for %s member %s: binding proof invalid", + privacy_log_label(binding.gate_id, label="gate"), + privacy_log_label(member.node_id if member.identity_scope == "anonymous" else persona_id, label="member"), + ) + return + state = _load_binding_store() + state.setdefault("gates", {})[binding.gate_id] = { + "gate_id": binding.gate_id, + "epoch": int(binding.epoch), + "root_persona_id": binding.root_persona_id, + "next_member_ref": int(binding.next_member_ref), + "members": { + persona_id: _serialize_member_binding(member) + for persona_id, member in binding.members.items() + }, + } + high_water = max( + int(binding.epoch), + int(_HIGH_WATER_EPOCHS.get(binding.gate_id, 0) or 0), + ) + _HIGH_WATER_EPOCHS[binding.gate_id] = high_water + state.setdefault("high_water_epochs", {})[binding.gate_id] = high_water + _save_binding_store(state) + + +def _persist_delete_binding(gate_id: str) -> None: + state = _load_binding_store() + gate_key = _stable_gate_ref(gate_id) + state.setdefault("gates", {}).pop(gate_key, None) + state.setdefault("high_water_epochs", {}).pop(gate_key, None) + _HIGH_WATER_EPOCHS.pop(gate_key, None) + _save_binding_store(state) + + +def _force_rebuild_binding(gate_id: str) -> None: + """Tear down the in-memory and persisted MLS binding for a gate. + + The next call to ``_sync_binding`` will create a fresh MLS group + with the current set of identities. + """ + gate_key = _stable_gate_ref(gate_id) + client = _privacy_client() + with _STATE_LOCK: + binding = _GATE_BINDINGS.pop(gate_key, None) + if binding is not None: + _release_binding(client, binding) + _persist_delete_binding(gate_key) + logger.info( + "Forced MLS binding rebuild for %s", + privacy_log_label(gate_key, label="gate"), + ) + + +def _persisted_gate_metadata(gate_id: str) -> dict[str, Any] | None: + state = _load_binding_store() + metadata = dict(state.get("gates", {}).get(_stable_gate_ref(gate_id)) or {}) + return metadata or None + + +def _lock_gate_format(gate_id: str, payload_format: str) -> None: + state = _load_binding_store() + gate_key = _stable_gate_ref(gate_id) + state.setdefault("gate_format_locks", {})[gate_key] = str(payload_format or "").strip().lower() + _save_binding_store(state) + + +def is_gate_locked_to_format(gate_id: str, payload_format: str) -> bool: + gate_key = _stable_gate_ref(gate_id) + locked_format = str( + _load_binding_store().get("gate_format_locks", {}).get(gate_key, "") or "" + ).strip().lower() + return bool(locked_format) and locked_format == str(payload_format or "").strip().lower() + + +def is_gate_locked_to_mls(gate_id: str) -> bool: + gate_key = _stable_gate_ref(gate_id) + if not gate_key: + return False + locked_format = str( + _load_binding_store().get("gate_format_locks", {}).get(gate_key, MLS_GATE_FORMAT) or MLS_GATE_FORMAT + ).strip().lower() + return locked_format == MLS_GATE_FORMAT + + +def get_local_gate_key_status(gate_id: str) -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + active = get_active_gate_identity(gate_key) + if not active.get("ok"): + return { + "ok": False, + "gate_id": gate_key, + "detail": str(active.get("detail") or "no active gate identity"), + } + source = str(active.get("source", "") or "") + identity = dict(active.get("identity") or {}) + metadata = _persisted_gate_metadata(gate_key) or {} + member_key = _gate_member_identity_id(identity) + has_local_access = False + try: + binding = _sync_binding(gate_key) + has_local_access = binding.members.get(member_key) is not None + except Exception: + has_local_access = False + if not has_local_access: + # Identity may have rotated — force rebuild and retry once. + try: + _force_rebuild_binding(gate_key) + binding = _sync_binding(gate_key) + pid = _gate_member_identity_id(identity) + has_local_access = pid in binding.members + if not has_local_access: + logger.warning( + "Gate status: identity %s not in binding members %s", + pid, + list(binding.members.keys()), + ) + except Exception as exc: + logger.warning("Gate status rebuild failed: %s", exc) + has_local_access = False + return { + "ok": True, + "gate_id": gate_key, + "current_epoch": int(metadata.get("epoch", 1) or 1), + "has_local_access": has_local_access, + "identity_scope": "anonymous" if source == "anonymous" else "persona", + "identity_node_id": str(identity.get("node_id", "") or ""), + "identity_persona_id": str(identity.get("persona_id", "") or ""), + "detail": "gate access ready" if has_local_access else "active gate identity is not mapped into the MLS group", + "format": MLS_GATE_FORMAT, + } + + +def ensure_gate_member_access( + *, + gate_id: str, + recipient_node_id: str, + recipient_dh_pub: str, + recipient_scope: str = "member", +) -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + recipient_node_id = str(recipient_node_id or "").strip() + if not gate_key or not recipient_node_id: + return {"ok": False, "detail": "gate_id and recipient_node_id required"} + personas = _gate_personas(gate_key) + recipient = next( + ( + persona + for persona in personas + if str(persona.get("node_id", "") or "").strip() == recipient_node_id + ), + None, + ) + if recipient is None: + return {"ok": False, "detail": "recipient identity is not a known gate member"} + binding = _sync_binding(gate_key) + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(binding.epoch), + "recipient_node_id": recipient_node_id, + "recipient_scope": str(recipient_scope or "member"), + "format": MLS_GATE_FORMAT, + "detail": "MLS gate membership is synchronized through privacy-core; no wrapped key required", + } + + +def mark_gate_rekey_recommended(gate_id: str, *, reason: str = "manual_review") -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + return { + "ok": True, + "gate_id": gate_key, + "format": MLS_GATE_FORMAT, + "detail": "MLS gate sessions rekey through membership commits; manual review recorded", + "reason": str(reason or "manual_review"), + } + + +def rotate_gate_epoch(gate_id: str, *, reason: str = "manual_rotate") -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + with _STATE_LOCK: + _GATE_BINDINGS.pop(gate_key, None) + binding = _sync_binding(gate_key) + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(binding.epoch), + "format": MLS_GATE_FORMAT, + "detail": "gate MLS state synchronized", + "reason": str(reason or "manual_rotate"), + } + + +def _validate_persisted_member( + gate_id: str, + member_meta: dict[str, Any], + identity: dict[str, Any] | None, +) -> tuple[bool, str]: + persona_id = str(member_meta.get("persona_id", "") or "") + identity_scope = str(member_meta.get("identity_scope", "") or "persona").strip().lower() + if identity is None: + return False, f"persisted MLS member identity is unknown: {persona_id}" + if str(identity.get("node_id", "") or "") != str(member_meta.get("node_id", "") or ""): + return False, f"persisted MLS member node mismatch: {persona_id}" + try: + bundle_bytes = _unb64(member_meta.get("public_bundle")) + except Exception as exc: + return False, f"persisted MLS bundle decode failed for {persona_id}: {exc}" + if identity_scope == "anonymous" or persona_id.startswith("session:"): + ok, reason = verify_gate_session_blob( + gate_id, + str(member_meta.get("node_id", "") or ""), + bundle_bytes, + str(member_meta.get("binding_signature", "") or ""), + ) + else: + if str(identity.get("persona_id", "") or "") != persona_id: + return False, f"persisted MLS member persona mismatch: {persona_id}" + ok, reason = verify_gate_persona_blob( + gate_id, + persona_id, + bundle_bytes, + str(member_meta.get("binding_signature", "") or ""), + ) + if not ok: + return False, f"persisted MLS binding proof invalid for {persona_id}: {reason}" + return True, "ok" + + +def _restore_binding_from_metadata( + gate_id: str, + identities_by_id: dict[str, dict[str, Any]], + metadata: dict[str, Any], +) -> _GateBinding | None: + gate_key = _stable_gate_ref(gate_id) + members_meta = dict(metadata.get("members") or {}) + if not members_meta: + return None + restored_epoch = max(1, int(metadata.get("epoch", 1) or 1)) + persisted_high_water = int( + _load_binding_store().get("high_water_epochs", {}).get(gate_key, _HIGH_WATER_EPOCHS.get(gate_key, 0)) or 0 + ) + _HIGH_WATER_EPOCHS[gate_key] = max(int(_HIGH_WATER_EPOCHS.get(gate_key, 0) or 0), persisted_high_water) + if restored_epoch < int(_HIGH_WATER_EPOCHS.get(gate_key, 0) or 0): + logger.warning( + "Persisted MLS epoch regressed for %s: restored=%s high_water=%s — rebuilding", + privacy_log_label(gate_key, label="gate"), + restored_epoch, + _HIGH_WATER_EPOCHS.get(gate_key, 0), + ) + return None + ordered = sorted( + members_meta.values(), + key=lambda item: ( + 0 if bool(item.get("is_creator")) else 1, + int(item.get("member_ref", 0) or 0), + str(item.get("persona_id", "") or ""), + ), + ) + identities: list[dict[str, Any]] = [] + for member_meta in ordered: + persona_id = str(member_meta.get("persona_id", "") or "") + identity = identities_by_id.get(persona_id) + ok, reason = _validate_persisted_member(gate_id, member_meta, identity) + if not ok: + logger.warning( + "Corrupted binding for %s member %s: %s — rebuilding", + privacy_log_label(gate_key, label="gate"), + privacy_log_label(persona_id, label="persona"), + type(reason).__name__ if not isinstance(reason, str) else "binding_invalid", + ) + state = _load_binding_store() + gate_entry = dict(state.get("gates", {}).get(gate_key) or {}) + members = dict(gate_entry.get("members") or {}) + members.pop(persona_id, None) + gate_entry["members"] = members + if members: + state.setdefault("gates", {})[gate_key] = gate_entry + else: + state.setdefault("gates", {}).pop(gate_key, None) + _save_binding_store(state) + return None + identities.append(dict(identity or {})) + + rebuilt = _build_binding(gate_id, identities) + rebuilt.epoch = max(1, int(metadata.get("epoch", rebuilt.epoch) or rebuilt.epoch)) + rebuilt.next_member_ref = max( + int(metadata.get("next_member_ref", rebuilt.next_member_ref) or rebuilt.next_member_ref), + max((int(item.get("member_ref", 0) or 0) for item in ordered), default=0) + 1, + ) + for member_meta in ordered: + persona_id = str(member_meta.get("persona_id", "") or "") + member = rebuilt.members.get(persona_id) + if member is None: + continue + member.member_ref = int(member_meta.get("member_ref", member.member_ref) or member.member_ref) + member.is_creator = bool(member_meta.get("is_creator")) + _HIGH_WATER_EPOCHS[gate_key] = max( + int(rebuilt.epoch), + int(_HIGH_WATER_EPOCHS.get(gate_key, 0) or 0), + ) + _persist_binding(rebuilt) + return rebuilt + + +def _release_member(client: PrivacyCoreClient, member: _GateMemberBinding) -> None: + if member.group_handle > 0: + try: + client.release_group(member.group_handle) + except Exception: + logger.exception( + "Failed to release MLS group handle for %s", + privacy_log_label(member.persona_id, label="persona"), + ) + if member.key_package_handle is not None: + try: + client.release_key_package(member.key_package_handle) + except Exception: + logger.exception( + "Failed to release MLS key package handle for %s", + privacy_log_label(member.persona_id, label="persona"), + ) + try: + client.release_identity(member.identity_handle) + except Exception: + logger.exception( + "Failed to release MLS identity handle for %s", + privacy_log_label(member.persona_id, label="persona"), + ) + + +def _release_binding(client: PrivacyCoreClient, binding: _GateBinding) -> None: + for member in list(binding.members.values()): + _release_member(client, member) + + +def _create_member_binding( + client: PrivacyCoreClient, + *, + gate_id: str, + identity: dict[str, Any], + member_ref: int, + is_creator: bool, + group_handle: int | None = None, +) -> _GateMemberBinding: + identity_handle = client.create_identity() + public_bundle = client.export_public_bundle(identity_handle) + identity_scope = _gate_member_identity_scope(identity) + binding_identity_id = _gate_member_identity_id(identity) + if identity_scope == "anonymous": + proof = sign_gate_session_blob( + gate_id, + str(identity.get("node_id", "") or ""), + public_bundle, + ) + else: + proof = sign_gate_persona_blob( + gate_id, + str(identity.get("persona_id", "") or ""), + public_bundle, + ) + if not proof.get("ok"): + try: + client.release_identity(identity_handle) + except Exception: + logger.exception("Failed to release MLS identity after binding proof failure") + raise PrivacyCoreError(str(proof.get("detail") or "persona MLS binding proof failed")) + key_package_handle: int | None = None + resolved_group_handle = group_handle + if not is_creator: + key_package_bytes = client.export_key_package(identity_handle) + key_package_handle = client.import_key_package(key_package_bytes) + resolved_group_handle = 0 + elif resolved_group_handle is None: + resolved_group_handle = client.create_group(identity_handle) + + assert resolved_group_handle is not None + return _GateMemberBinding( + persona_id=binding_identity_id, + node_id=str(identity.get("node_id", "") or ""), + label=str(identity.get("label", "") or ""), + identity_scope=identity_scope, + identity_handle=identity_handle, + group_handle=resolved_group_handle, + member_ref=member_ref, + is_creator=is_creator, + key_package_handle=key_package_handle, + public_bundle=public_bundle, + binding_signature=str(proof.get("signature", "") or ""), + ) + + +def _build_binding(gate_id: str, identities: list[dict[str, Any]]) -> _GateBinding: + if not identities: + raise PrivacyCoreError("no gate identities are available for MLS mapping") + + client = _privacy_client() + creator = identities[0] + creator_binding = _create_member_binding( + client, + gate_id=gate_id, + identity=creator, + member_ref=0, + is_creator=True, + ) + binding = _GateBinding( + gate_id=_stable_gate_ref(gate_id), + epoch=1, + root_persona_id=creator_binding.persona_id, + root_group_handle=creator_binding.group_handle, + members={creator_binding.persona_id: creator_binding}, + ) + + for identity in identities[1:]: + member_binding: _GateMemberBinding | None = None + commit_handle = 0 + try: + member_binding = _create_member_binding( + client, + gate_id=gate_id, + identity=identity, + member_ref=binding.next_member_ref, + is_creator=False, + ) + commit_handle = client.add_member(binding.root_group_handle, member_binding.key_package_handle or 0) + member_binding.group_handle = client.commit_joined_group_handle(commit_handle, 0) + binding.members[member_binding.persona_id] = member_binding + binding.next_member_ref += 1 + except Exception: + if member_binding is not None: + _release_member(client, member_binding) + raise + finally: + if commit_handle: + try: + client.release_commit(commit_handle) + except Exception: + pass + + return binding + + +def _ensure_reader_identity(gate_key: str) -> dict[str, Any]: + """Create a dedicated reader identity for cross-member MLS decrypt. + + MLS does not let the sender decrypt their own ciphertext. On a + single-operator node every message is "from self". By ensuring + the MLS group always has at least two members, the non-sender + member can always decrypt what the sender encrypted — giving + every gate member (including the author) read access. + + The reader is stored as a normal gate persona so existing signing + infrastructure (``sign_gate_persona_blob``) can find it. + """ + from services.mesh.mesh_wormhole_persona import ( + _identity_record, # type: ignore[attr-defined] + read_wormhole_persona_state, + _write_wormhole_persona_state, + bootstrap_wormhole_persona_state, + ) + + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + personas = list(state.get("gate_personas", {}).get(gate_key) or []) + # Check if a reader persona already exists. + for p in personas: + if str(p.get("label", "") or "") == "_reader": + return p + import secrets as _secrets + + reader_persona_id = f"_reader_{_secrets.token_hex(4)}" + reader = _identity_record( + scope="gate_persona", + gate_id=gate_key, + persona_id=reader_persona_id, + label="_reader", + ) + personas.append(reader) + state.setdefault("gate_personas", {})[gate_key] = personas + _write_wormhole_persona_state(state) + return reader + + +def _sync_binding(gate_id: str) -> _GateBinding: + gate_key = _stable_gate_ref(gate_id) + personas = _gate_personas(gate_key) + session_identity = _gate_session_identity(gate_key) + identities: list[dict[str, Any]] = list(personas) + if session_identity: + identities.append(session_identity) + # Ensure we always have ≥2 members so cross-member MLS decrypt works. + # MLS does not allow a sender to decrypt their own message — on a + # single-operator node, every member is "self". The reader identity + # is a dedicated second member that exists solely for this purpose. + if len(identities) < 2: + reader = _ensure_reader_identity(gate_key) + reader_id = _gate_member_identity_id(reader) + if not any(_gate_member_identity_id(i) == reader_id for i in identities): + identities.append(reader) + if not identities: + _persist_delete_binding(gate_key) + raise PrivacyCoreError("no gate identities exist for this gate") + + identities_by_id = { + _gate_member_identity_id(identity): identity + for identity in identities + } + client = _privacy_client() + active_identity, _active_source = _active_gate_member(gate_key) + active_identity_id = _gate_member_identity_id(active_identity) if active_identity else "" + + with _STATE_LOCK: + binding = _GATE_BINDINGS.get(gate_key) + if binding is None or binding.root_persona_id not in identities_by_id: + if binding is not None: + _release_binding(client, binding) + metadata = _persisted_gate_metadata(gate_key) + if metadata: + restored = _restore_binding_from_metadata(gate_key, identities_by_id, metadata) + if restored is not None: + _GATE_BINDINGS[gate_key] = restored + return restored + ordered_identities = sorted( + identities, + key=lambda item: ( + 0 if _gate_member_identity_id(item) == active_identity_id else 1, + _gate_member_identity_id(item), + ), + ) + binding = _build_binding(gate_key, ordered_identities) + _GATE_BINDINGS[gate_key] = binding + _persist_binding(binding) + return binding + + dirty = False + removed_persona_ids = [persona_id for persona_id in binding.members if persona_id not in identities_by_id] + for persona_id in removed_persona_ids: + member = binding.members.get(persona_id) + if member is None: + continue + if member.is_creator: + _release_binding(client, binding) + remaining = [identities_by_id[key] for key in sorted(identities_by_id.keys())] + rebuilt = _build_binding(gate_key, remaining) + _GATE_BINDINGS[gate_key] = rebuilt + _persist_binding(rebuilt) + return rebuilt + + commit_handle = 0 + try: + commit_handle = client.remove_member(binding.root_group_handle, member.member_ref) + finally: + if commit_handle: + try: + client.release_commit(commit_handle) + except Exception: + pass + _release_member(client, member) + binding.members.pop(persona_id, None) + binding.epoch += 1 + dirty = True + + for persona_id, persona in identities_by_id.items(): + if persona_id in binding.members: + continue + member_binding: _GateMemberBinding | None = None + commit_handle = 0 + try: + member_binding = _create_member_binding( + client, + gate_id=gate_key, + identity=persona, + member_ref=binding.next_member_ref, + is_creator=False, + ) + commit_handle = client.add_member( + binding.root_group_handle, + member_binding.key_package_handle or 0, + ) + member_binding.group_handle = client.commit_joined_group_handle(commit_handle, 0) + binding.members[persona_id] = member_binding + binding.next_member_ref += 1 + binding.epoch += 1 + dirty = True + except Exception: + if member_binding is not None: + _release_member(client, member_binding) + raise + finally: + if commit_handle: + try: + client.release_commit(commit_handle) + except Exception: + pass + + if dirty: + _persist_binding(binding) + return binding + + +def compose_encrypted_gate_message(gate_id: str, plaintext: str) -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + plaintext = str(plaintext or "") + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + if not plaintext.strip(): + return {"ok": False, "detail": "plaintext required"} + try: + from services.wormhole_supervisor import get_transport_tier + + if get_transport_tier() == "public_degraded": + return {"ok": False, "detail": "MLS gate compose requires PRIVATE transport tier"} + except Exception: + return {"ok": False, "detail": "MLS gate compose requires PRIVATE transport tier"} + + active_identity, active_source = _active_gate_member(gate_key) + if not active_identity: + return {"ok": False, "detail": "no active gate identity"} + raw_ts = time.time() + bucket_s = 60 + ts = float(math.floor(raw_ts / bucket_s) * bucket_s) + + try: + binding = _sync_binding(gate_key) + persona_id = _gate_member_identity_id(active_identity) + member = binding.members.get(persona_id) + if member is None: + _force_rebuild_binding(gate_key) + binding = _sync_binding(gate_key) + member = binding.members.get(persona_id) + if member is None: + return {"ok": False, "detail": "active gate identity is not mapped into the MLS group"} + plaintext_with_epoch = json.dumps( + { + "m": plaintext, + "e": int(binding.epoch), + }, + separators=(",", ":"), + ensure_ascii=False, + ) + ciphertext = _privacy_client().encrypt_group_message( + member.group_handle, + plaintext_with_epoch.encode("utf-8"), + ) + # MLS does not let the sender decrypt their own ciphertext. + # Immediately decrypt with a *different* group member so the + # plaintext is available to every member on this node — including + # the sender — without storing raw plaintext outside the MLS layer. + _self_decrypt_plaintext: str | None = None + for other_pid, other_member in binding.members.items(): + if other_pid == persona_id: + continue # skip the sender + try: + dec_bytes = _privacy_client().decrypt_group_message( + other_member.group_handle, + ciphertext, + ) + dec_raw = dec_bytes.decode("utf-8") + try: + dec_env = json.loads(dec_raw) + _self_decrypt_plaintext = str(dec_env["m"]) if isinstance(dec_env, dict) and "m" in dec_env else dec_raw + except (json.JSONDecodeError, ValueError, TypeError): + _self_decrypt_plaintext = dec_raw + break + except Exception: + continue + except Exception: + logger.exception( + "MLS gate compose failed for %s", + privacy_log_label(gate_key, label="gate"), + ) + return {"ok": False, "detail": "gate_mls_compose_failed"} + + message_id = base64.b64encode(secrets.token_bytes(12)).decode("ascii") + sender_ref = _sender_ref(_sender_ref_seed(active_identity), message_id) + padded_ct = _pad_ciphertext_raw(ciphertext) + # Create a durable gate envelope: the plaintext encrypted under the + # gate's domain key (AES-256-GCM). This survives MLS group rebuilds + # and process restarts. Only nodes holding the gate domain key can + # decrypt — outsiders see opaque base64. + gate_envelope: str = "" + try: + gate_envelope = _gate_envelope_encrypt(gate_key, plaintext) + except Exception: + logger.debug("gate envelope encrypt failed — MLS-only for this message") + payload = { + "gate": gate_key, + "ciphertext": _b64(padded_ct), + "nonce": message_id, + "sender_ref": sender_ref, + "format": MLS_GATE_FORMAT, + } + # gate_envelope must NOT be in the signed payload — it rides alongside. + signed = sign_gate_wormhole_event(gate_id=gate_key, event_type="gate_message", payload=payload) + if not signed.get("signature"): + return {"ok": False, "detail": str(signed.get("detail") or "gate_sign_failed")} + _HIGH_WATER_EPOCHS[gate_key] = max( + int(binding.epoch), + int(_HIGH_WATER_EPOCHS.get(gate_key, 0) or 0), + ) + _lock_gate_format(gate_key, MLS_GATE_FORMAT) + # Cache the MLS-decrypted plaintext (not raw input) so every member + # including the sender can read it back. Falls back to the original + # plaintext if the cross-member decrypt failed (single-member edge case). + _cache_local_plaintext(gate_key, payload["ciphertext"], sender_ref, str(_self_decrypt_plaintext or plaintext)) + return _ComposeResult( + { + "ok": True, + "gate_id": gate_key, + "identity_scope": "anonymous" if active_source == "anonymous" else str(signed.get("identity_scope", "") or "gate_persona"), + "sender_id": str(signed.get("node_id", "") or ""), + "public_key": str(signed.get("public_key", "") or ""), + "public_key_algo": str(signed.get("public_key_algo", "") or ""), + "protocol_version": str(signed.get("protocol_version", "") or ""), + "sequence": int(signed.get("sequence", 0) or 0), + "signature": str(signed.get("signature", "") or ""), + "ciphertext": payload["ciphertext"], + "nonce": payload["nonce"], + "sender_ref": sender_ref, + "format": MLS_GATE_FORMAT, + "timestamp": ts, + "gate_envelope": gate_envelope, + }, + legacy_epoch=int(binding.epoch), + ) + + +def decrypt_gate_message_for_local_identity( + *, + gate_id: str, + epoch: int, + ciphertext: str, + nonce: str, + sender_ref: str = "", + gate_envelope: str = "", +) -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + if not gate_key or not ciphertext: + return {"ok": False, "detail": "gate_id and ciphertext required"} + + # Fast path: gate envelope (AES-256-GCM under gate domain key). + # This always works regardless of MLS group state / restarts. + if gate_envelope: + envelope_pt = _gate_envelope_decrypt(gate_key, gate_envelope) + if envelope_pt is not None: + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(epoch or 0), + "plaintext": envelope_pt, + "identity_scope": "gate_envelope", + } + + active_identity, active_source = _active_gate_member(gate_key) + if not active_identity: + return {"ok": False, "detail": "no active gate identity"} + + expected_sender_ref = _sender_ref(_sender_ref_seed(active_identity), str(nonce or "")) + if str(sender_ref or "").strip() == expected_sender_ref: + cached_plaintext = _peek_cached_plaintext(gate_key, str(ciphertext), str(sender_ref)) + if cached_plaintext is not None: + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(epoch or 0), + "plaintext": cached_plaintext, + "identity_scope": "anonymous" if active_source == "anonymous" else "persona", + } + + # Try all group members (verifier path) — this is the primary decrypt + # strategy on a single-operator node where the sender is also a member. + # A non-sender member can decrypt even though the sender member cannot. + verifier_open = open_gate_ciphertext_for_verifier( + gate_id=gate_key, + ciphertext=str(ciphertext), + format=MLS_GATE_FORMAT, + epoch=int(epoch or 0), + ) + if verifier_open.get("ok"): + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(verifier_open.get("epoch", epoch or 0) or 0), + "plaintext": str(verifier_open.get("plaintext", "") or ""), + "identity_scope": active_source if active_source == "anonymous" else "persona", + } + # All MLS members are "self" — single-operator node authored this + # message but plaintext was not persisted (pre-fix legacy message). + if verifier_open.get("detail") == "gate_mls_self_authored": + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(epoch or 0), + "plaintext": "", + "self_authored": True, + "legacy": True, + "identity_scope": active_source if active_source == "anonymous" else "persona", + } + + try: + binding = _sync_binding(gate_key) + persona_id = _gate_member_identity_id(active_identity) + member = binding.members.get(persona_id) + if member is None: + _force_rebuild_binding(gate_key) + binding = _sync_binding(gate_key) + member = binding.members.get(persona_id) + if member is None: + return {"ok": False, "detail": "active gate identity is not mapped into the MLS group"} + decrypted_bytes = _privacy_client().decrypt_group_message( + member.group_handle, + _unpad_ciphertext_raw(_unb64(ciphertext)), + ) + except Exception: + # The verifier (all-member attempt) already ran above and failed. + # Check the in-memory cache as a last resort. + if str(sender_ref or "").strip() == expected_sender_ref: + cached_plaintext = _consume_cached_plaintext(gate_key, str(ciphertext), str(sender_ref)) + if cached_plaintext is not None: + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(epoch or binding.epoch or 0), + "plaintext": cached_plaintext, + "identity_scope": "anonymous" if active_source == "anonymous" else "persona", + } + logger.debug( + "MLS gate decrypt failed for %s (verifier already attempted)", + privacy_log_label(gate_key, label="gate"), + ) + return {"ok": False, "detail": "gate_mls_decrypt_failed"} + + raw = decrypted_bytes.decode("utf-8") + try: + envelope = json.loads(raw) + if isinstance(envelope, dict) and "m" in envelope: + actual_plaintext = str(envelope["m"]) + decrypted_epoch = int(envelope.get("e", 0) or 0) + else: + actual_plaintext = raw + decrypted_epoch = 0 + except (json.JSONDecodeError, ValueError, TypeError): + actual_plaintext = raw + decrypted_epoch = 0 + + _lock_gate_format(gate_key, MLS_GATE_FORMAT) + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(decrypted_epoch or epoch or 0), + "plaintext": actual_plaintext, + "identity_scope": "anonymous" if active_source == "anonymous" else "persona", + } + + +def open_gate_ciphertext_for_verifier( + *, + gate_id: str, + ciphertext: str, + format: str, + epoch: int, +) -> dict[str, Any]: + gate_key = _stable_gate_ref(gate_id) + if not gate_key or not ciphertext: + return {"ok": False, "detail": "gate_id and ciphertext required"} + if str(format or "").strip() != MLS_GATE_FORMAT: + return {"ok": False, "detail": "unsupported gate ciphertext format"} + + with _STATE_LOCK: + binding = _GATE_BINDINGS.get(gate_key) + if binding is None: + try: + binding = _sync_binding(gate_key) + except Exception: + logger.exception( + "MLS verifier open sync failed for %s", + privacy_log_label(gate_key, label="gate"), + ) + return {"ok": False, "detail": "gate_mls_verifier_open_failed"} + + last_error: Exception | None = None + all_self_authored = True + decoded = _unpad_ciphertext_raw(_unb64(ciphertext)) + for persona_id, member in list(binding.members.items()): + try: + decrypted_bytes = _privacy_client().decrypt_group_message( + member.group_handle, + decoded, + ) + raw = decrypted_bytes.decode("utf-8") + try: + envelope = json.loads(raw) + if isinstance(envelope, dict) and "m" in envelope: + actual_plaintext = str(envelope["m"]) + decrypted_epoch = int(envelope.get("e", 0) or 0) + else: + actual_plaintext = raw + decrypted_epoch = 0 + except (json.JSONDecodeError, ValueError, TypeError): + actual_plaintext = raw + decrypted_epoch = 0 + _lock_gate_format(gate_key, MLS_GATE_FORMAT) + return { + "ok": True, + "gate_id": gate_key, + "epoch": int(decrypted_epoch or epoch or 0), + "plaintext": actual_plaintext, + "opened_by_persona_id": persona_id, + "identity_scope": "verifier", + } + except Exception as exc: + if "message from self" not in str(exc): + all_self_authored = False + last_error = exc + continue + + if all_self_authored and last_error is not None: + logger.debug( + "MLS verifier open: all members are self for %s (self-authored message)", + privacy_log_label(gate_key, label="gate"), + ) + return {"ok": False, "detail": "gate_mls_self_authored"} + logger.error( + "MLS verifier open failed for %s", + privacy_log_label(gate_key, label="gate"), + exc_info=last_error, + ) + return {"ok": False, "detail": "gate_mls_verifier_open_failed"} diff --git a/backend/services/mesh/mesh_hashchain.py b/backend/services/mesh/mesh_hashchain.py new file mode 100644 index 00000000..3cdf463a --- /dev/null +++ b/backend/services/mesh/mesh_hashchain.py @@ -0,0 +1,2034 @@ +"""Infonet — append-only signed event ledger for decentralized consensus. + +The Infonet is ShadowBroker's consensus protocol. Every action on the mesh +(message, vote, gate creation, oracle prediction) becomes a chain event. +Each event references the previous event's hash, creating an immutable +ordered sequence. No mining, no proof-of-work — just cryptographic linking +and signature verification. + +This is the consensus layer. The reputation, gates, and oracle systems are the +application layer that sits on top. + +Event types: + - message: Public broadcast message + - vote: Reputation vote (+1/-1 on a node) + - gate_create: New gate/community creation + - prediction: Oracle market prediction + - stake: Oracle truth stake + - key_rotate: Link old and new public keys + - key_revoke: Revoke a compromised key (with grace window) + +Private DM registration, mailbox access, and transport routing metadata are +intentionally kept off-ledger. + +Each event contains: + - event_id: SHA-256 hash of (prev_hash + type + payload + timestamp + node_id) + - prev_hash: Hash of the previous event (chain link) + - type: Event type string + - node_id: Author's node ID + - payload: Event-specific data + - timestamp: Unix timestamp + - sequence: Per-node monotonic sequence number (replay protection) + - signature: Node's cryptographic signature + +Persistence: JSON file at backend/data/infonet.json + +Encrypted gate chat events are intentionally kept off the public chain and +persisted separately via GateMessageStore. +""" + +import json +import os +import time +import hmac +import hashlib +import logging +import threading +import atexit +import tempfile +from pathlib import Path +from collections import deque +from typing import Any + +from services.mesh.mesh_secure_storage import read_domain_json, write_domain_json +from services.mesh.mesh_crypto import ( + build_signature_payload, + parse_public_key_algo, + verify_node_binding, + verify_signature, +) +from services.mesh.mesh_protocol import NETWORK_ID, PROTOCOL_VERSION, normalize_payload +from services.mesh.mesh_schema import ( + PUBLIC_LEDGER_EVENT_TYPES, + validate_event_payload, + validate_protocol_fields, + validate_public_ledger_payload, +) + +logger = logging.getLogger("services.mesh_hashchain") +_PRIVACY_LOGS = os.environ.get("MESH_PRIVACY_LOGS", "").strip().lower() in ("1", "true", "yes") +_MESH_ONLY = os.environ.get("MESH_ONLY", "").strip().lower() in ("1", "true", "yes") + + +def _safe_int(val, default=0): + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _redact_node(node_id: str) -> str: + if not node_id: + return "" + if _PRIVACY_LOGS or _MESH_ONLY: + return f"{node_id[:6]}…" + return node_id + + +def _atomic_write_text(target: Path, content: str, encoding: str = "utf-8") -> None: + """Write content atomically via temp file + os.replace().""" + parent = target.parent + parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=str(parent), suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding=encoding) as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, str(target)) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +CHAIN_FILE = DATA_DIR / "infonet.json" +WAL_FILE = DATA_DIR / "infonet.wal" +GATE_STORE_DIR = DATA_DIR / "gate_messages" +GATE_STORAGE_DOMAIN = "gates" + +# ─── Constants ──────────────────────────────────────────────────────────── + +GENESIS_HASH = "0" * 64 # The "previous hash" for the first event +MAX_CHAIN_MEMORY = 50000 # Max events to keep in memory (older ones on disk only) +EPHEMERAL_TTL = 86400 # 24 hours — ephemeral messages auto-purge +MESSAGE_RETENTION_DAYS = 90 # Non-ephemeral messages kept for 90 days +CHAIN_LOCK_DEPTH = 6 +GATE_REPLAY_WINDOW_S = 86400 * 30 +GATE_REPLAY_PRUNE_INTERVAL = 256 +_PUBLIC_EVENT_APPEND_HOOKS: list[Any] = [] +_PUBLIC_EVENT_APPEND_HOOKS_LOCK = threading.Lock() + + +def register_public_event_append_hook(callback: Any) -> None: + if callback is None: + return + with _PUBLIC_EVENT_APPEND_HOOKS_LOCK: + if callback not in _PUBLIC_EVENT_APPEND_HOOKS: + _PUBLIC_EVENT_APPEND_HOOKS.append(callback) + + +def unregister_public_event_append_hook(callback: Any) -> None: + with _PUBLIC_EVENT_APPEND_HOOKS_LOCK: + if callback in _PUBLIC_EVENT_APPEND_HOOKS: + _PUBLIC_EVENT_APPEND_HOOKS.remove(callback) + + +def _notify_public_event_append_hooks(event_dict: dict[str, Any]) -> None: + with _PUBLIC_EVENT_APPEND_HOOKS_LOCK: + hooks = list(_PUBLIC_EVENT_APPEND_HOOKS) + for hook in hooks: + try: + hook(dict(event_dict)) + except Exception: + logger.exception("public event append hook failed") + + +# ─── Network Identity ──────────────────────────────────────────────────── +# NETWORK_ID is defined in services.mesh_protocol to avoid circular imports. + +# ─── Protocol Constraints ──────────────────────────────────────────────── + +ALLOWED_EVENT_TYPES = set(PUBLIC_LEDGER_EVENT_TYPES) + +MAX_PAYLOAD_BYTES = 4096 +REPLAY_FILTER_BITS = 1_000_000 +REPLAY_FILTER_HASHES = 3 +REPLAY_FILTER_ROTATE_S = 3600 +CRITICAL_EVENT_TYPES = {"key_rotate", "key_revoke"} +MIN_CONFIRMATIONS_CRITICAL = 3 + + +def _gate_wire_event_material(event: dict[str, Any]) -> str: + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + material = { + "event_type": str(event.get("event_type", "gate_message") or "gate_message"), + "timestamp": float(event.get("timestamp", 0) or 0), + "ciphertext": str(payload.get("ciphertext", "") or ""), + "format": str(payload.get("format", "") or ""), + } + return json.dumps(material, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def build_gate_replay_fingerprint(gate_id: str, event: dict[str, Any]) -> str: + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + material = { + "gate": str(gate_id or "").strip().lower(), + "event_type": "gate_message", + "ciphertext": str(payload.get("ciphertext", "") or ""), + "format": str(payload.get("format", "") or ""), + } + return hashlib.sha256( + json.dumps(material, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + + +def build_gate_wire_ref(gate_id: str, event: dict[str, Any]) -> str: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return "" + try: + from services.config import get_settings + + secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip() + except Exception: + secret = "" + if not secret: + return "" + material = f"{gate_key}|{_gate_wire_event_material(event)}".encode("utf-8") + return hmac.new(secret.encode("utf-8"), material, hashlib.sha256).hexdigest() + + +def resolve_gate_wire_ref(gate_ref: str, event: dict[str, Any]) -> str: + ref = str(gate_ref or "").strip().lower() + if not ref: + return "" + candidates: set[str] = set() + try: + candidates.update(gate_store.known_gate_ids()) + except Exception: + pass + try: + from services.mesh.mesh_reputation import gate_manager + + for gate in gate_manager.list_gates(): + gate_id = str((gate or {}).get("gate_id", "") or "").strip().lower() + if gate_id: + candidates.add(gate_id) + except Exception: + pass + for gate_id in sorted(candidates): + candidate_ref = build_gate_wire_ref(gate_id, event) + if candidate_ref and hmac.compare_digest(candidate_ref, ref): + return gate_id + return "" + + +def _private_gate_signature_payload(gate_id: str, event: dict[str, Any]) -> dict[str, Any]: + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + normalized = { + "gate": str(gate_id or "").strip().lower(), + "ciphertext": str(payload.get("ciphertext", "") or ""), + "nonce": str(payload.get("nonce", "") or ""), + "sender_ref": str(payload.get("sender_ref", "") or ""), + "format": str(payload.get("format", "mls1") or "mls1"), + } + epoch = _safe_int(payload.get("epoch", 0) or 0, 0) + if epoch > 0: + normalized["epoch"] = epoch + return normalize_payload("gate_message", normalized) + + +def _private_gate_event_id(gate_id: str, node_id: str, sequence: int, event: dict[str, Any]) -> str: + payload_json = json.dumps( + _private_gate_signature_payload(gate_id, event), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + timestamp = float(event.get("timestamp", 0) or 0) + return hashlib.sha256( + f"{gate_id}:{node_id}:{payload_json}:{timestamp}:{int(sequence)}".encode("utf-8") + ).hexdigest() + + +def _sanitize_private_gate_event(gate_id: str, event: dict[str, Any]) -> dict[str, Any]: + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + sanitized = { + "event_id": str(event.get("event_id", "") or ""), + "event_type": "gate_message", + "node_id": str(event.get("node_id", "") or ""), + "timestamp": float(event.get("timestamp", 0) or 0), + "sequence": int(event.get("sequence", 0) or 0), + "signature": str(event.get("signature", "") or ""), + "public_key": str(event.get("public_key", "") or ""), + "public_key_algo": str(event.get("public_key_algo", "") or ""), + "protocol_version": str(event.get("protocol_version", "") or ""), + "payload": { + "gate": str(gate_id or "").strip().lower(), + "ciphertext": str(payload.get("ciphertext", "") or ""), + "nonce": str(payload.get("nonce", "") or ""), + "sender_ref": str(payload.get("sender_ref", "") or ""), + "format": str(payload.get("format", "mls1") or "mls1"), + }, + } + epoch = _safe_int(payload.get("epoch", 0) or 0, 0) + if epoch > 0: + sanitized["payload"]["epoch"] = epoch + gate_envelope = str(payload.get("gate_envelope", "") or "").strip() + if gate_envelope: + sanitized["payload"]["gate_envelope"] = gate_envelope + reply_to = str(payload.get("reply_to", "") or "").strip() + if reply_to: + sanitized["payload"]["reply_to"] = reply_to + return sanitized + + +def _authorize_private_gate_transport_author( + gate_id: str, + node_id: str, + public_key: str, + public_key_algo: str, +) -> tuple[bool, str]: + gate_key = str(gate_id or "").strip().lower() + candidate = str(node_id or "").strip() + if not gate_key or not candidate: + return False, "private gate authorization unavailable" + try: + from services.mesh.mesh_reputation import gate_manager, reputation_ledger + except Exception: + return False, "private gate authorization unavailable" + try: + reputation_ledger.register_node(candidate, public_key, public_key_algo) + except Exception: + return False, "private gate authorization unavailable" + ok, reason = gate_manager.can_enter(candidate, gate_key) + if ok: + return True, "ok" + return False, str(reason or "Gate access denied") + + +def _verify_private_gate_transport_event(gate_id: str, event: dict[str, Any]) -> tuple[bool, str, dict[str, Any] | None]: + node_id = str(event.get("node_id", "") or event.get("sender_id", "") or "").strip() + public_key = str(event.get("public_key", "") or "").strip() + public_key_algo = str(event.get("public_key_algo", "") or "").strip() + signature = str(event.get("signature", "") or "").strip() + protocol_version = str(event.get("protocol_version", "") or "").strip() + sequence = _safe_int(event.get("sequence", 0) or 0, 0) + if not node_id or not public_key or not public_key_algo or not signature: + return False, "missing private gate auth fields", None + if sequence <= 0: + return False, "invalid private gate sequence", None + if protocol_version != PROTOCOL_VERSION: + return False, "Unsupported protocol_version", None + payload = _private_gate_signature_payload(gate_id, event) + ok, reason = validate_event_payload("gate_message", payload) + if not ok: + return False, reason, None + if not verify_node_binding(node_id, public_key): + return False, "node_id mismatch", None + algo = parse_public_key_algo(public_key_algo) + if not algo: + return False, "Unsupported public_key_algo", None + sig_payload = build_signature_payload( + event_type="gate_message", + node_id=node_id, + sequence=sequence, + payload=payload, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=algo, + signature_hex=signature, + payload=sig_payload, + ): + return False, "Invalid signature", None + authorized, reason = _authorize_private_gate_transport_author(gate_id, node_id, public_key, public_key_algo) + if not authorized: + return False, f"private gate access denied: {reason}", None + expected_event_id = _private_gate_event_id(gate_id, node_id, sequence, event) + provided_event_id = str(event.get("event_id", "") or "").strip() + if provided_event_id and provided_event_id != expected_event_id: + return False, "private gate event_id mismatch", None + sanitized = _sanitize_private_gate_event(gate_id, event) + sanitized["event_id"] = provided_event_id or expected_event_id + return True, "ok", sanitized + + +class GateMessageStore: + """Private-plane storage for encrypted gate messages.""" + + def __init__(self, data_dir: str = ""): + self._gates: dict[str, list[dict]] = {} + self._event_index: dict[str, dict] = {} + self._replay_index: dict[str, dict[str, Any]] = {} + self._replay_prune_counter = 0 + self._data_dir = Path(data_dir) if data_dir else GATE_STORE_DIR + self._lock = threading.Lock() + self._load() + + def _gate_file_path(self, gate_id: str) -> Path: + digest = hashlib.sha256(str(gate_id or "").encode("utf-8")).hexdigest() + return self._data_dir / f"gate_{digest}.jsonl" + + def _gate_storage_base_dir(self) -> Path: + return self._data_dir.parent + + def _gate_domain_dir(self) -> Path: + return self._gate_storage_base_dir() / GATE_STORAGE_DOMAIN + + def _sort_gate(self, gate_id: str) -> None: + events = self._gates.get(gate_id, []) + events.sort( + key=lambda evt: ( + float(evt.get("timestamp", 0) or 0), + _safe_int(evt.get("sequence", 0) or 0, 0), + str(evt.get("event_id", "") or ""), + ) + ) + + def _remember_replay_fingerprint(self, replay_fingerprint: str, event: dict) -> None: + self._replay_index[replay_fingerprint] = { + "event": event, + "timestamp": float(event.get("timestamp", 0) or 0.0), + } + + def _replay_existing_event(self, replay_fingerprint: str) -> dict | None: + entry = self._replay_index.get(replay_fingerprint) or {} + event = entry.get("event") + return event if isinstance(event, dict) else None + + def _prune_replay_index(self, now: float | None = None) -> int: + current = float(now if now is not None else time.time()) + cutoff = current - GATE_REPLAY_WINDOW_S + stale = [ + fingerprint + for fingerprint, entry in list(self._replay_index.items()) + if float((entry or {}).get("timestamp", 0) or 0.0) < cutoff + ] + for fingerprint in stale: + self._replay_index.pop(fingerprint, None) + return len(stale) + + def _maybe_prune_replay_index(self) -> None: + self._replay_prune_counter += 1 + if self._replay_prune_counter % GATE_REPLAY_PRUNE_INTERVAL == 0: + self._prune_replay_index() + + def _load(self) -> None: + encrypted_dir = self._gate_domain_dir() + if not self._data_dir.exists() and not encrypted_dir.exists(): + return + dirty_gates: set[str] = set() + file_names = { + path.name for path in self._data_dir.glob("gate_*.jsonl") + } | { + path.name for path in encrypted_dir.glob("gate_*.jsonl") + } + for file_name in sorted(file_names): + events: list[dict[str, Any]] | None = None + encrypted_path = encrypted_dir / file_name + if encrypted_path.exists(): + try: + loaded = read_domain_json( + GATE_STORAGE_DOMAIN, + file_name, + lambda: [], + base_dir=self._gate_storage_base_dir(), + ) + if isinstance(loaded, list): + events = [evt for evt in loaded if isinstance(evt, dict)] + except Exception: + events = None + if events is None: + legacy_path = self._data_dir / file_name + if not legacy_path.exists(): + continue + try: + lines = legacy_path.read_text(encoding="utf-8").splitlines() + except Exception: + continue + events = [] + for line in lines: + if not line.strip(): + continue + try: + evt = json.loads(line) + except Exception: + continue + if isinstance(evt, dict): + events.append(evt) + loaded_gate_ids: set[str] = set() + for evt in events: + payload = evt.get("payload") or {} + if not isinstance(payload, dict): + continue + gate_id = str(payload.get("gate", "") or "").strip().lower() + if not gate_id: + continue + storage_event = _sanitize_private_gate_event(gate_id, evt) + if storage_event != evt: + dirty_gates.add(gate_id) + evt = storage_event + if not str(evt.get("event_id", "") or "").strip(): + evt["event_id"] = self._synth_event_id(gate_id, evt) + dirty_gates.add(gate_id) + loaded_gate_ids.add(gate_id) + replay_fingerprint = build_gate_replay_fingerprint(gate_id, evt) + if replay_fingerprint in self._replay_index: + dirty_gates.add(gate_id) + continue + event_id = str(evt.get("event_id", "") or "") + if event_id and event_id in self._event_index: + dirty_gates.add(gate_id) + continue + self._gates.setdefault(gate_id, []).append(evt) + if event_id: + self._event_index[event_id] = evt + self._remember_replay_fingerprint(replay_fingerprint, evt) + if not encrypted_path.exists(): + dirty_gates.update(loaded_gate_ids) + self._prune_replay_index() + for gate_id in list(self._gates.keys()): + self._sort_gate(gate_id) + for gate_id in sorted(dirty_gates): + self._persist_gate(gate_id) + + def _persist_gate(self, gate_id: str) -> None: + events = self._gates.get(gate_id, []) + file_name = self._gate_file_path(gate_id).name + write_domain_json( + GATE_STORAGE_DOMAIN, + file_name, + events, + base_dir=self._gate_storage_base_dir(), + ) + self._gate_file_path(gate_id).unlink(missing_ok=True) + + def _synth_event_id(self, gate_id: str, event: dict) -> str: + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + material = { + "gate": str(gate_id or "").strip().lower(), + "event_type": str(event.get("event_type", "") or ""), + "timestamp": float(event.get("timestamp", 0) or 0), + "ciphertext": str(payload.get("ciphertext", "") or ""), + "format": str(payload.get("format", "") or ""), + } + return hashlib.sha256( + json.dumps(material, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ).hexdigest() + + def append(self, gate_id: str, event: dict) -> dict: + gate_id = str(gate_id or "").strip().lower() + if not gate_id: + return event + clean_event = _sanitize_private_gate_event(gate_id, event) + if not str(clean_event.get("event_id", "") or "").strip(): + clean_event["event_id"] = self._synth_event_id(gate_id, clean_event) + with self._lock: + self._gates.setdefault(gate_id, []) + replay_fingerprint = build_gate_replay_fingerprint(gate_id, clean_event) + existing = self._replay_existing_event(replay_fingerprint) + if existing is not None: + return existing + event_id = str(clean_event.get("event_id", "") or "") + if event_id and event_id in self._event_index: + return self._event_index[event_id] + self._gates[gate_id].append(clean_event) + self._sort_gate(gate_id) + if event_id: + self._event_index[event_id] = clean_event + self._remember_replay_fingerprint(replay_fingerprint, clean_event) + self._maybe_prune_replay_index() + self._persist_gate(gate_id) + return clean_event + + def get_messages(self, gate_id: str, limit: int = 20, offset: int = 0) -> list[dict]: + gate_id = str(gate_id or "").strip().lower() + with self._lock: + msgs = self._gates.get(gate_id, []) + return list(reversed(msgs))[offset : offset + limit] + + def known_gate_ids(self) -> list[str]: + with self._lock: + return sorted(self._gates.keys()) + + def get_event(self, event_id: str) -> dict | None: + with self._lock: + return self._event_index.get(str(event_id or "")) + + def ingest_peer_events(self, gate_id: str, events: list[dict]) -> dict: + gate_id = str(gate_id or "").strip().lower() + accepted = 0 + duplicates = 0 + rejected = 0 + if not gate_id: + return {"accepted": 0, "duplicates": 0, "rejected": 0} + with self._lock: + self._gates.setdefault(gate_id, []) + for evt in events: + if not isinstance(evt, dict): + rejected += 1 + continue + event_id = str(evt.get("event_id", "") or "") + payload = evt.get("payload") + if not isinstance(payload, dict): + rejected += 1 + continue + if not payload.get("ciphertext"): + rejected += 1 + continue + if evt.get("event_type") != "gate_message": + rejected += 1 + continue + ts = evt.get("timestamp", 0) + now = time.time() + if not isinstance(ts, (int, float)) or ts > now + 300 or ts < now - 86400 * 30: + rejected += 1 + continue + replay_fingerprint = build_gate_replay_fingerprint(gate_id, evt) + if replay_fingerprint in self._replay_index: + duplicates += 1 + continue + if event_id: + if len(event_id) != 64: + rejected += 1 + continue + try: + int(event_id, 16) + except ValueError: + rejected += 1 + continue + else: + event_id = "" + if event_id and event_id in self._event_index: + duplicates += 1 + continue + ok, reason, clean_event = _verify_private_gate_transport_event(gate_id, evt) + if not ok or clean_event is None: + logger.warning("Rejected private gate peer event: %s", reason) + rejected += 1 + continue + event_id = str(clean_event.get("event_id", "") or "") + if event_id in self._event_index: + duplicates += 1 + continue + self._gates[gate_id].append(clean_event) + self._event_index[event_id] = clean_event + self._remember_replay_fingerprint(replay_fingerprint, clean_event) + accepted += 1 + if accepted: + self._maybe_prune_replay_index() + self._sort_gate(gate_id) + self._persist_gate(gate_id) + return {"accepted": accepted, "duplicates": duplicates, "rejected": rejected} + + +class ReplayFilter: + """Bounded bloom-style replay filter with rotation.""" + + def __init__( + self, + *, + size_bits: int = REPLAY_FILTER_BITS, + hash_count: int = REPLAY_FILTER_HASHES, + rotate_s: int = REPLAY_FILTER_ROTATE_S, + ) -> None: + self._size_bits = max(1024, int(size_bits)) + self._hash_count = max(2, int(hash_count)) + self._rotate_s = max(60, int(rotate_s)) + self._salt = os.urandom(16) + self._active = bytearray(self._size_bits // 8 + 1) + self._previous = bytearray(self._size_bits // 8 + 1) + self._last_rotate = time.time() + + def _rotate_if_needed(self) -> None: + now = time.time() + if now - self._last_rotate < self._rotate_s: + return + self._previous = self._active + self._active = bytearray(self._size_bits // 8 + 1) + self._last_rotate = now + + def _positions(self, value: str) -> list[int]: + positions: list[int] = [] + data = value.encode("utf-8") + for idx in range(self._hash_count): + digest = hashlib.sha256(self._salt + idx.to_bytes(2, "big") + data).digest() + pos = int.from_bytes(digest[:8], "big") % self._size_bits + positions.append(pos) + return positions + + def add(self, value: str) -> None: + self._rotate_if_needed() + for pos in self._positions(value): + byte_idx = pos // 8 + bit = 1 << (pos % 8) + self._active[byte_idx] |= bit + + def seen(self, value: str) -> bool: + self._rotate_if_needed() + for pos in self._positions(value): + byte_idx = pos // 8 + bit = 1 << (pos % 8) + if not (self._active[byte_idx] & bit or self._previous[byte_idx] & bit): + return False + return True + + +class ChainEvent: + """Single event on the Infonet.""" + + __slots__ = ( + "event_id", + "prev_hash", + "event_type", + "node_id", + "payload", + "timestamp", + "sequence", + "signature", + "network_id", + "public_key", + "public_key_algo", + "protocol_version", + ) + + def __init__( + self, + prev_hash: str, + event_type: str, + node_id: str, + payload: dict, + timestamp: float = 0, + sequence: int = 0, + signature: str = "", + network_id: str = "", + public_key: str = "", + public_key_algo: str = "", + protocol_version: str = "", + ): + self.prev_hash = prev_hash + self.event_type = event_type + self.node_id = node_id + self.payload = payload + self.timestamp = timestamp or time.time() + self.sequence = sequence + self.signature = signature + self.network_id = network_id or NETWORK_ID + self.public_key = public_key + self.public_key_algo = public_key_algo + self.protocol_version = protocol_version or PROTOCOL_VERSION + # Compute deterministic event ID + self.event_id = self._compute_hash() + + def _compute_hash(self) -> str: + """Deterministic SHA-256 hash of the event content.""" + content = ( + f"{self.prev_hash}:{self.event_type}:{self.node_id}:" + f"{json.dumps(self.payload, sort_keys=True, separators=(',', ':'), ensure_ascii=False)}:" + f"{self.timestamp}:{self.sequence}:{self.network_id}" + ) + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + def to_dict(self) -> dict: + return { + "event_id": self.event_id, + "prev_hash": self.prev_hash, + "event_type": self.event_type, + "node_id": self.node_id, + "payload": self.payload, + "timestamp": self.timestamp, + "sequence": self.sequence, + "signature": self.signature, + "network_id": self.network_id, + "public_key": self.public_key, + "public_key_algo": self.public_key_algo, + "protocol_version": self.protocol_version, + } + + @classmethod + def from_dict(cls, d: dict) -> "ChainEvent": + evt = cls( + prev_hash=d["prev_hash"], + event_type=d["event_type"], + node_id=d["node_id"], + payload=d["payload"], + timestamp=d["timestamp"], + sequence=d.get("sequence", 0), + signature=d.get("signature", ""), + network_id=d.get("network_id", NETWORK_ID), + public_key=d.get("public_key", ""), + public_key_algo=d.get("public_key_algo", ""), + protocol_version=d.get("protocol_version", PROTOCOL_VERSION), + ) + # Verify hash matches + if evt.event_id != d.get("event_id"): + raise ValueError( + f"Hash mismatch on event load: computed {evt.event_id[:16]}, " + f"stored {d.get('event_id', '?')[:16]}" + ) + return evt + + +class Infonet: + """The Infonet — ShadowBroker's append-only signed event ledger. + + The Infonet is the single source of truth. All actions go through here. + The reputation ledger, gates, and oracle are computed views of Infonet state. + """ + + def __init__(self): + self.events: list[dict] = [] # Stored as dicts for efficiency + self.head_hash: str = GENESIS_HASH # Hash of the latest event + self.node_sequences: dict[str, int] = {} # {node_id: last_sequence} + self.event_index: dict[str, int] = {} # {event_id: index in events list} + self.public_key_bindings: dict[str, str] = {} # {public_key: canonical node_id} + self.revocations: dict[str, dict] = {} + self._replay_filter = ReplayFilter() + self._last_validated_index: int = 0 # For incremental validation + # Running counters — avoid O(N) scans in get_info() + self._type_counts: dict[str, int] = {} + self._active_count: int = 0 + self._chain_bytes: int = 2 # Start with "[]" empty JSON array + self._dirty = False + self._save_lock = threading.Lock() + self._save_timer: threading.Timer | None = None + self._SAVE_INTERVAL = 5.0 # seconds — coalesce writes + atexit.register(self._flush) + self._load() + + # ─── Persistence ────────────────────────────────────────────────── + + def _load(self): + """Load Infonet from disk.""" + if CHAIN_FILE.exists(): + try: + data = json.loads(CHAIN_FILE.read_text(encoding="utf-8")) + loaded_events = data.get("events", []) + if not isinstance(loaded_events, list): + raise ValueError("Malformed chain: events must be a list") + for evt in loaded_events: + if not isinstance(evt, dict): + raise ValueError("Malformed chain: event entry must be an object") + ChainEvent.from_dict(evt) + self.events = loaded_events + self.head_hash = data.get("head_hash", GENESIS_HASH) + self.node_sequences = data.get("node_sequences", {}) + self._rebuild_state() + self._rebuild_revocations() + self._rebuild_counters() + logger.info( + f"Loaded Infonet: {len(self.events)} events, " f"head={self.head_hash[:16]}..." + ) + except Exception as e: + logger.error(f"Failed to load Infonet: {e}") + raise + self._replay_wal() + + def _rebuild_state(self) -> None: + self.event_index = {} + self.node_sequences = {} + self.public_key_bindings = {} + self.revocations = {} + self._replay_filter = ReplayFilter() + for idx, evt in enumerate(self.events): + event_id = evt.get("event_id", "") + if event_id: + self.event_index[event_id] = idx + self._replay_filter.add(event_id) + node_id = evt.get("node_id", "") + sequence = _safe_int(evt.get("sequence", 0) or 0, 0) + if node_id and sequence: + last = self.node_sequences.get(node_id, 0) + if sequence > last: + self.node_sequences[node_id] = sequence + public_key = str(evt.get("public_key", "") or "") + if public_key and node_id: + existing = self.public_key_bindings.get(public_key) + if not existing: + self.public_key_bindings[public_key] = node_id + elif existing != node_id: + logger.warning( + "Public key binding conflict in stored chain for %s: %s vs %s", + public_key[:12], + _redact_node(existing), + _redact_node(node_id), + ) + if evt.get("event_type") == "key_revoke": + self._apply_revocation(evt) + if self.events: + self.head_hash = self.events[-1].get("event_id", GENESIS_HASH) + else: + self.head_hash = GENESIS_HASH + + def _rebuild_counters(self) -> None: + """Rebuild running counters from the full event list (called on load).""" + now = time.time() + self._type_counts = {} + self._active_count = 0 + self._chain_bytes = 2 # "[]" + for evt in self.events: + t = evt.get("event_type", "unknown") + self._type_counts[t] = self._type_counts.get(t, 0) + 1 + is_eph = evt.get("payload", {}).get("ephemeral") or evt.get("payload", {}).get("_ephemeral") + if not is_eph or (now - evt.get("timestamp", 0)) < EPHEMERAL_TTL: + self._active_count += 1 + self._chain_bytes += len(json.dumps(evt)) + 2 # +2 for ", " separator + + def _update_counters_for_event(self, evt: dict) -> None: + """Incrementally update counters when a new event is appended.""" + t = evt.get("event_type", "unknown") + self._type_counts[t] = self._type_counts.get(t, 0) + 1 + self._active_count += 1 + self._chain_bytes += len(json.dumps(evt)) + 2 + + def _write_wal(self, event_dict: dict) -> None: + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + _atomic_write_text(WAL_FILE, json.dumps({"event": event_dict}), encoding="utf-8") + except Exception as e: + logger.error(f"Failed to write WAL: {e}") + + def _clear_wal(self) -> None: + try: + if WAL_FILE.exists(): + WAL_FILE.unlink() + except Exception as e: + logger.error(f"Failed to clear WAL: {e}") + + def _replay_wal(self) -> None: + if not WAL_FILE.exists(): + return + try: + data = json.loads(WAL_FILE.read_text(encoding="utf-8")) + except Exception: + self._clear_wal() + return + evt = data.get("event") if isinstance(data, dict) else None + if not isinstance(evt, dict): + self._clear_wal() + return + if evt.get("event_id") in self.event_index: + self._clear_wal() + return + if evt.get("prev_hash") != self.head_hash: + self._clear_wal() + return + result = self.ingest_events([evt]) + if result.get("accepted"): + logger.info("Replayed WAL event after restart") + self._clear_wal() + + def reset_chain(self) -> None: + """Wipe local chain state so the next sync starts from genesis. + + Used for automatic fork recovery when the local chain is small and + has diverged from the network. Does NOT touch gate_store or WAL. + """ + prev_len = len(self.events) + self.events = [] + self.head_hash = GENESIS_HASH + self.node_sequences = {} + self.event_index = {} + self.public_key_bindings = {} + self.revocations = {} + self._replay_filter = ReplayFilter() + self._last_validated_index = 0 + self._type_counts = {} + self._active_count = 0 + self._chain_bytes = 2 + self._dirty = True + self._flush() + logger.warning("Chain reset: discarded %d local events for fork recovery", prev_len) + + def _save(self): + """Mark dirty and schedule a coalesced disk write. + + Instead of writing multi-MB JSON on every event, we set a dirty flag + and schedule a single write after _SAVE_INTERVAL seconds. Multiple + rapid calls collapse into one I/O operation. + """ + self._dirty = True + with self._save_lock: + if self._save_timer is None or not self._save_timer.is_alive(): + self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush) + self._save_timer.daemon = True + self._save_timer.start() + + def _flush(self): + """Actually write to disk (called by timer or atexit).""" + if not self._dirty: + return + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + data = { + "protocol": "infonet", + "network_id": NETWORK_ID, + "head_hash": self.head_hash, + "node_sequences": self.node_sequences, + "events": self.events, + } + _atomic_write_text(CHAIN_FILE, json.dumps(data, indent=2), encoding="utf-8") + self._dirty = False + except Exception as e: + logger.error(f"Failed to save Infonet: {e}") + + def ensure_materialized(self) -> None: + """Write the current chain state to disk even if nothing is dirty yet.""" + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + data = { + "protocol": "infonet", + "network_id": NETWORK_ID, + "head_hash": self.head_hash, + "node_sequences": self.node_sequences, + "events": self.events, + } + _atomic_write_text(CHAIN_FILE, json.dumps(data, indent=2), encoding="utf-8") + except Exception as e: + logger.error(f"Failed to materialize Infonet: {e}") + raise + + def confirmations_for_event(self, event_id: str) -> int: + idx = self.event_index.get(event_id) + if idx is None: + return 0 + return max(0, len(self.events) - 1 - idx) + + def decorate_event(self, evt: dict) -> dict: + if evt.get("event_type") not in CRITICAL_EVENT_TYPES: + return evt + confirmations = self.confirmations_for_event(evt.get("event_id", "")) + decorated = dict(evt) + decorated["confirmations"] = confirmations + decorated["confirmed"] = confirmations >= MIN_CONFIRMATIONS_CRITICAL + return decorated + + def decorate_events(self, events: list[dict]) -> list[dict]: + return [self.decorate_event(evt) for evt in events] + + def chain_lock(self) -> dict: + if len(self.events) <= CHAIN_LOCK_DEPTH: + return {"depth": CHAIN_LOCK_DEPTH, "event_id": "", "active": False} + idx = max(0, len(self.events) - 1 - CHAIN_LOCK_DEPTH) + return { + "depth": CHAIN_LOCK_DEPTH, + "event_id": self.events[idx].get("event_id", ""), + "active": True, + } + + def _bind_public_key(self, public_key: str, node_id: str) -> tuple[bool, str]: + key = str(public_key or "") + node = str(node_id or "") + if not key or not node: + return False, "Missing public key binding fields" + existing = self.public_key_bindings.get(key) + if existing and existing != node: + return False, f"public key already bound to {existing}" + self.public_key_bindings[key] = node + return True, "ok" + + def _apply_revocation(self, evt: dict) -> None: + payload = evt.get("payload", {}) + public_key = payload.get("revoked_public_key") or evt.get("public_key", "") + if not public_key: + return + revoked_at = _safe_int(payload.get("revoked_at", 0) or 0, 0) + grace_until = _safe_int(payload.get("grace_until", revoked_at) or revoked_at, revoked_at) + info = { + "public_key": public_key, + "public_key_algo": payload.get("revoked_public_key_algo") or evt.get("public_key_algo"), + "revoked_at": revoked_at, + "grace_until": grace_until, + "reason": payload.get("reason", ""), + "event_id": evt.get("event_id", ""), + "node_id": evt.get("node_id", ""), + } + existing = self.revocations.get(public_key) + if not existing or revoked_at >= _safe_int(existing.get("revoked_at", 0), 0): + self.revocations[public_key] = info + + def _rebuild_revocations(self) -> None: + self.revocations = {} + for evt in self.events: + if evt.get("event_type") == "key_revoke": + self._apply_revocation(evt) + + def _revocation_status(self, public_key: str) -> tuple[bool, dict | None]: + info = self.revocations.get(public_key) + if not info: + return False, None + now = time.time() + if now > _safe_int(info.get("grace_until", 0) or 0, 0): + return True, info + return False, info + + # ─── Append ─────────────────────────────────────────────────────── + + def validate_and_set_sequence(self, node_id: str, sequence: int) -> tuple[bool, str]: + """Validate monotonic sequence and update last-seen value if valid.""" + if sequence <= 0: + return False, "Sequence must be a positive integer" + last = self.node_sequences.get(node_id, 0) + if sequence <= last: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("replay_attempts") + return False, f"Replay detected: sequence {sequence} <= last {last}" + self.node_sequences[node_id] = sequence + self._save() + return True, "ok" + + def append( + self, + event_type: str, + node_id: str, + payload: dict, + signature: str = "", + sequence: int = 0, + ephemeral: bool = False, + public_key: str = "", + public_key_algo: str = "", + protocol_version: str = "", + timestamp_bucket_s: int = 0, + ) -> dict: + """Append a new event to the Infonet. Returns the event dict. + + Args: + event_type: Type of event (message, vote, gate_create, etc.) + node_id: Author node ID + payload: Event-specific data + signature: Cryptographic signature from node's private key + ephemeral: If True, event auto-purges after 24h + + Returns: + The event dict with computed event_id + """ + from services.mesh.mesh_crypto import ( + build_signature_payload, + parse_public_key_algo, + verify_node_binding, + verify_signature, + ) + + if event_type not in ALLOWED_EVENT_TYPES: + raise ValueError(f"Unsupported event_type: {event_type}") + + if sequence <= 0: + raise ValueError("sequence is required and must be > 0") + last = self.node_sequences.get(node_id, 0) + if sequence <= last: + raise ValueError(f"Replay detected: sequence {sequence} <= last {last}") + + payload = normalize_payload(event_type, dict(payload or {})) + + ok, reason = validate_event_payload(event_type, payload) + if not ok: + raise ValueError(reason) + ok, reason = validate_public_ledger_payload(event_type, payload) + if not ok: + raise ValueError(reason) + + if event_type == "message": + if "ephemeral" not in payload: + payload["ephemeral"] = bool(ephemeral) + else: + payload.pop("ephemeral", None) + + payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + if len(payload_json.encode("utf-8")) > MAX_PAYLOAD_BYTES: + raise ValueError("payload exceeds max size") + + protocol_version = str(protocol_version or PROTOCOL_VERSION) + ok, reason = validate_protocol_fields(protocol_version, NETWORK_ID) + if not ok: + raise ValueError(reason) + + if not (signature and public_key and public_key_algo): + raise ValueError("Missing signature fields") + if not parse_public_key_algo(public_key_algo): + raise ValueError("Unsupported public_key_algo") + if not verify_node_binding(node_id, public_key): + raise ValueError("node_id mismatch") + bound, bind_reason = self._bind_public_key(public_key, node_id) + if not bound: + raise ValueError(bind_reason) + sig_payload = build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=payload, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=public_key_algo, + signature_hex=signature, + payload=sig_payload, + ): + raise ValueError("Invalid signature") + + if public_key: + revoked, _info = self._revocation_status(public_key) + if revoked and event_type != "key_revoke": + raise ValueError("public key is revoked") + + if event_type == "key_revoke": + if payload.get("revoked_public_key") and payload.get("revoked_public_key") != public_key: + raise ValueError("revoked_public_key must match event public_key") + if payload.get("revoked_public_key_algo") and payload.get( + "revoked_public_key_algo" + ) != public_key_algo: + raise ValueError("revoked_public_key_algo must match event public_key_algo") + + if timestamp_bucket_s > 0: + ts = time.time() + ts = float(int(ts / timestamp_bucket_s) * timestamp_bucket_s) + else: + ts = time.time() + + # Create event + event = ChainEvent( + prev_hash=self.head_hash, + event_type=event_type, + node_id=node_id, + payload=payload, + timestamp=ts, + sequence=sequence, + signature=signature, + public_key=public_key, + public_key_algo=public_key_algo, + protocol_version=protocol_version, + ) + + event_dict = event.to_dict() + self._write_wal(event_dict) + self.events.append(event_dict) + self.event_index[event.event_id] = len(self.events) - 1 + self.head_hash = event.event_id + self.node_sequences[node_id] = sequence + self._replay_filter.add(event.event_id) + self._update_counters_for_event(event_dict) + + if event_type == "key_revoke": + self._apply_revocation(event_dict) + + self._save() + self._clear_wal() + + try: + from services.mesh.mesh_rns import rns_bridge + + rns_bridge.publish_event(event_dict) + except Exception: + pass + _notify_public_event_append_hooks(event_dict) + + logger.info( + f"Infonet append [{event_type}] by {_redact_node(node_id)} seq={sequence} " + f"id={event.event_id[:16]}..." + ) + return event_dict + + def ingest_events(self, events: list[dict]) -> dict: + """Ingest a sequence of external events. Requires contiguous prev_hash.""" + accepted = 0 + duplicates = 0 + rejected: list[dict] = [] + expected_prev = self.head_hash + + for idx, evt in enumerate(events): + if not isinstance(evt, dict): + rejected.append({"index": idx, "reason": "Event is not an object"}) + continue + + event_type = evt.get("event_type", "") + node_id = evt.get("node_id", "") + event_id = evt.get("event_id", "") + prev_hash = evt.get("prev_hash", "") + sequence = _safe_int(evt.get("sequence", 0) or 0, 0) + + if event_type not in ALLOWED_EVENT_TYPES: + rejected.append({"index": idx, "reason": "Unsupported event_type"}) + continue + if not event_id or not prev_hash: + rejected.append({"index": idx, "reason": "Missing event_id or prev_hash"}) + continue + if prev_hash != expected_prev: + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_prev_hash_mismatch") + except Exception: + pass + rejected.append({"index": idx, "reason": "prev_hash does not match head"}) + continue + if evt.get("network_id") != NETWORK_ID: + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_network_mismatch") + except Exception: + pass + rejected.append({"index": idx, "reason": "network_id mismatch"}) + continue + if event_id in self.event_index: + duplicates += 1 + continue + if self._replay_filter.seen(event_id): + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_replay_seen") + except Exception: + pass + duplicates += 1 + continue + if prev_hash != self.head_hash: + rejected.append({"index": idx, "reason": "prev_hash does not match head"}) + continue + if sequence <= 0: + rejected.append({"index": idx, "reason": "Invalid sequence"}) + continue + last = self.node_sequences.get(node_id, 0) + if sequence <= last: + rejected.append({"index": idx, "reason": "Replay detected"}) + continue + + payload = evt.get("payload", {}) + ok, reason = validate_event_payload(event_type, payload) + if not ok: + rejected.append({"index": idx, "reason": reason}) + continue + ok, reason = validate_public_ledger_payload(event_type, payload) + if not ok: + rejected.append({"index": idx, "reason": reason}) + continue + if event_type == "message" and "ephemeral" not in payload: + rejected.append({"index": idx, "reason": "Missing ephemeral flag"}) + continue + + payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + if len(payload_json.encode("utf-8")) > MAX_PAYLOAD_BYTES: + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_payload_too_large") + except Exception: + pass + rejected.append({"index": idx, "reason": "Payload too large"}) + continue + + proto = evt.get("protocol_version") or PROTOCOL_VERSION + if proto != PROTOCOL_VERSION: + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_proto_mismatch") + except Exception: + pass + rejected.append({"index": idx, "reason": "Unsupported protocol_version"}) + continue + + signature = evt.get("signature", "") + public_key = evt.get("public_key", "") + public_key_algo = evt.get("public_key_algo", "") + if not (signature and public_key and public_key_algo): + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_signature_missing") + except Exception: + pass + rejected.append({"index": idx, "reason": "Missing signature fields"}) + continue + from services.mesh.mesh_crypto import parse_public_key_algo + + if not parse_public_key_algo(public_key_algo): + rejected.append({"index": idx, "reason": "Unsupported public_key_algo"}) + continue + + if event_type == "key_revoke": + if payload.get("revoked_public_key") and payload.get( + "revoked_public_key" + ) != public_key: + rejected.append( + {"index": idx, "reason": "revoked_public_key must match public_key"} + ) + continue + if payload.get("revoked_public_key_algo") and payload.get( + "revoked_public_key_algo" + ) != public_key_algo: + rejected.append( + { + "index": idx, + "reason": "revoked_public_key_algo must match public_key_algo", + } + ) + continue + revoked, _info = self._revocation_status(public_key) + if revoked and event_type != "key_revoke": + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_key_revoked") + except Exception: + pass + rejected.append({"index": idx, "reason": "public key is revoked"}) + continue + last_seq = self.node_sequences.get(node_id, 0) + if sequence <= last_seq: + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_replay_sequence") + except Exception: + pass + rejected.append( + { + "index": idx, + "reason": f"Replay detected: sequence {sequence} <= last {last_seq}", + } + ) + continue + + from services.mesh.mesh_crypto import ( + build_signature_payload, + verify_signature, + verify_node_binding, + ) + + if not verify_node_binding(node_id, public_key): + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_node_mismatch") + except Exception: + pass + rejected.append({"index": idx, "reason": "node_id mismatch"}) + continue + bound, bind_reason = self._bind_public_key(public_key, node_id) + if not bound: + rejected.append({"index": idx, "reason": bind_reason}) + continue + + sig_payload = build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=payload, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=public_key_algo, + signature_hex=signature, + payload=sig_payload, + ): + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_signature_invalid") + except Exception: + pass + rejected.append({"index": idx, "reason": "Invalid signature"}) + continue + + # Verify event_id/hash linkage + try: + computed = ChainEvent.from_dict(evt).event_id + except (ValueError, KeyError, TypeError) as exc: + rejected.append({"index": idx, "reason": f"event_id hash mismatch: {exc}"}) + continue + if computed != event_id: + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ingest_event_id_mismatch") + except Exception: + pass + rejected.append({"index": idx, "reason": "event_id mismatch"}) + continue + + # Accept + self.events.append(evt) + self.event_index[event_id] = len(self.events) - 1 + self.head_hash = event_id + self.node_sequences[node_id] = sequence + accepted += 1 + expected_prev = event_id + self._replay_filter.add(event_id) + if event_type == "key_revoke": + self._apply_revocation(evt) + + if accepted: + self._save() + return {"accepted": accepted, "duplicates": duplicates, "rejected": rejected} + + # ─── Validation ─────────────────────────────────────────────────── + + def validate_chain(self, verify_signatures: bool = False) -> tuple[bool, str]: + """Verify the entire Infonet's integrity. + + Checks that each event's prev_hash matches the previous event's event_id, + and that each event's hash is correct. + + Returns (valid, reason) + """ + if not self.events: + return True, "Empty chain" + + prev = GENESIS_HASH + seen_public_keys: dict[str, str] = {} + for i, evt_dict in enumerate(self.events): + # Check prev_hash linkage + if evt_dict["prev_hash"] != prev: + return False, ( + f"Broken link at index {i}: expected prev_hash " + f"{prev[:16]}..., got {evt_dict['prev_hash'][:16]}..." + ) + + # Recompute hash and verify + evt = ChainEvent.from_dict(evt_dict) + if evt.event_id != evt_dict["event_id"]: + return False, ( + f"Hash mismatch at index {i}: computed " + f"{evt.event_id[:16]}..., stored {evt_dict['event_id'][:16]}..." + ) + + if verify_signatures: + proto = evt_dict.get("protocol_version") or PROTOCOL_VERSION + if proto != PROTOCOL_VERSION: + return False, f"Unsupported protocol_version at index {i}: {proto}" + signature = evt_dict.get("signature", "") + public_key = evt_dict.get("public_key", "") + public_key_algo = evt_dict.get("public_key_algo", "") + if not (signature and public_key and public_key_algo): + return False, f"Missing signature fields at index {i}" + + from services.mesh.mesh_crypto import ( + build_signature_payload, + parse_public_key_algo, + verify_signature, + verify_node_binding, + ) + + node_id = evt_dict.get("node_id", "") + if not parse_public_key_algo(public_key_algo): + return False, f"Unsupported public_key_algo at index {i}" + if not verify_node_binding(node_id, public_key): + return False, f"node_id mismatch at index {i}" + existing = seen_public_keys.get(public_key) + if existing and existing != node_id: + return False, f"public key binding conflict at index {i}" + seen_public_keys[public_key] = node_id + + normalized = normalize_payload( + evt_dict.get("event_type", ""), evt_dict.get("payload", {}) + ) + sig_payload = build_signature_payload( + event_type=evt_dict.get("event_type", ""), + node_id=node_id, + sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0), + payload=normalized, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=public_key_algo, + signature_hex=signature, + payload=sig_payload, + ): + return False, f"Invalid signature at index {i}" + + prev = evt_dict["event_id"] + + if prev != self.head_hash: + return ( + False, + f"Head hash mismatch: chain ends at {prev[:16]}... but head is {self.head_hash[:16]}...", + ) + + return True, f"Valid Infonet: {len(self.events)} events" + + def validate_chain_incremental(self, verify_signatures: bool = False) -> tuple[bool, str]: + """Validate only events appended since last successful validation. + + Much faster than full validate_chain() on large chains — O(new) vs O(N). + Falls back to full validation if the chain has been restructured. + """ + total = len(self.events) + start = self._last_validated_index + if start > total: + # Chain was truncated (fork resolution) — fall back to full + self._last_validated_index = 0 + return self.validate_chain(verify_signatures=verify_signatures) + if start >= total: + return True, f"No new events (chain has {total} events)" + + # Determine expected prev_hash at the start index + if start == 0: + prev = GENESIS_HASH + else: + prev = self.events[start - 1]["event_id"] + + for i in range(start, total): + evt_dict = self.events[i] + if evt_dict["prev_hash"] != prev: + return False, ( + f"Broken link at index {i}: expected prev_hash " + f"{prev[:16]}..., got {evt_dict['prev_hash'][:16]}..." + ) + evt = ChainEvent.from_dict(evt_dict) + if evt.event_id != evt_dict["event_id"]: + return False, ( + f"Hash mismatch at index {i}: computed " + f"{evt.event_id[:16]}..., stored {evt_dict['event_id'][:16]}..." + ) + + if verify_signatures: + proto = evt_dict.get("protocol_version") or PROTOCOL_VERSION + if proto != PROTOCOL_VERSION: + return False, f"Unsupported protocol_version at index {i}: {proto}" + signature = evt_dict.get("signature", "") + public_key = evt_dict.get("public_key", "") + public_key_algo = evt_dict.get("public_key_algo", "") + if not (signature and public_key and public_key_algo): + return False, f"Missing signature fields at index {i}" + + from services.mesh.mesh_crypto import ( + build_signature_payload, + parse_public_key_algo, + verify_signature, + verify_node_binding, + ) + + node_id = evt_dict.get("node_id", "") + if not parse_public_key_algo(public_key_algo): + return False, f"Unsupported public_key_algo at index {i}" + if not verify_node_binding(node_id, public_key): + return False, f"node_id mismatch at index {i}" + + normalized = normalize_payload( + evt_dict.get("event_type", ""), evt_dict.get("payload", {}) + ) + sig_payload = build_signature_payload( + event_type=evt_dict.get("event_type", ""), + node_id=node_id, + sequence=_safe_int(evt_dict.get("sequence", 0) or 0, 0), + payload=normalized, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=public_key_algo, + signature_hex=signature, + payload=sig_payload, + ): + return False, f"Invalid signature at index {i}" + prev = evt_dict["event_id"] + + if prev != self.head_hash: + return False, ( + f"Head hash mismatch: chain ends at {prev[:16]}... but head is {self.head_hash[:16]}..." + ) + + self._last_validated_index = total + return True, f"Valid Infonet: {total} events ({total - start} new)" + + def _order_chain_from(self, prev_hash: str, events: list[dict]) -> list[dict] | None: + by_prev: dict[str, dict] = {} + for evt in events: + p = evt.get("prev_hash", "") + if not p: + return None + if p in by_prev: + return None + by_prev[p] = evt + ordered = [] + current = prev_hash + while current in by_prev: + evt = by_prev[current] + ordered.append(evt) + current = evt.get("event_id", "") + if not current: + return None + if len(ordered) != len(events): + return None + return ordered + + def apply_fork(self, events: list[dict], head_hash: str, proof_count: int, quorum: int) -> tuple[bool, str]: + if not events: + return False, "empty fork" + if proof_count < max(2, int(quorum)): + return False, "insufficient quorum" + prev_hash = events[0].get("prev_hash", "") + if not prev_hash: + return False, "missing prev_hash" + prev_index = self.event_index.get(prev_hash) + if prev_index is None: + return False, "unknown ancestor" + depth_from_head = len(self.events) - 1 - prev_index + if depth_from_head > CHAIN_LOCK_DEPTH: + return False, "chain lock prevents reorg" + ordered = self._order_chain_from(prev_hash, events) + if not ordered: + return False, "non-contiguous fork" + if ordered[-1].get("event_id", "") != head_hash: + return False, "head_hash mismatch" + current_tail_len = len(self.events) - 1 - prev_index + if len(ordered) <= current_tail_len: + return False, "fork not longer" + + # Validate events and sequences against prefix + prefix = self.events[: prev_index + 1] + last_seq: dict[str, int] = {} + seen_public_keys: dict[str, str] = {} + for evt in prefix: + node_id = evt.get("node_id", "") + sequence = _safe_int(evt.get("sequence", 0) or 0, 0) + if node_id and sequence: + last_seq[node_id] = max(last_seq.get(node_id, 0), sequence) + public_key = str(evt.get("public_key", "") or "") + if public_key and node_id: + seen_public_keys.setdefault(public_key, node_id) + + for evt in ordered: + event_type = evt.get("event_type", "") + node_id = evt.get("node_id", "") + event_id = evt.get("event_id", "") + sequence = _safe_int(evt.get("sequence", 0) or 0, 0) + payload = evt.get("payload", {}) + if event_type not in ALLOWED_EVENT_TYPES: + return False, "unsupported event_type" + if not event_id or not node_id: + return False, "missing fields" + if evt.get("network_id") != NETWORK_ID: + return False, "network mismatch" + existing_idx = self.event_index.get(event_id) + if existing_idx is not None and existing_idx <= prev_index: + return False, "duplicate event_id" + payload = normalize_payload(event_type, dict(payload or {})) + ok, reason = validate_event_payload(event_type, payload) + if not ok: + return False, reason + proto = evt.get("protocol_version") or PROTOCOL_VERSION + if proto != PROTOCOL_VERSION: + return False, "unsupported protocol_version" + signature = evt.get("signature", "") + public_key = evt.get("public_key", "") + public_key_algo = evt.get("public_key_algo", "") + if not (signature and public_key and public_key_algo): + return False, "missing signature fields" + revoked, _info = self._revocation_status(public_key) + if revoked and event_type != "key_revoke": + return False, "public key revoked" + last = last_seq.get(node_id, 0) + if sequence <= last: + return False, "sequence replay" + from services.mesh.mesh_crypto import ( + build_signature_payload, + parse_public_key_algo, + verify_signature, + verify_node_binding, + ) + + if not parse_public_key_algo(public_key_algo): + return False, "unsupported public_key_algo" + if not verify_node_binding(node_id, public_key): + return False, "node_id mismatch" + existing = seen_public_keys.get(public_key) + if existing and existing != node_id: + return False, "public key binding conflict" + seen_public_keys[public_key] = node_id + sig_payload = build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=payload, + ) + if not verify_signature( + public_key_b64=public_key, + public_key_algo=public_key_algo, + signature_hex=signature, + payload=sig_payload, + ): + return False, "invalid signature" + computed = ChainEvent.from_dict(evt).event_id + if computed != event_id: + return False, "event_id mismatch" + last_seq[node_id] = sequence + + # Apply fork + self.events = prefix + ordered + self._rebuild_state() + self._save() + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("fork_applied") + except Exception: + pass + return True, "applied" + + def check_replay(self, node_id: str, sequence: int) -> bool: + """Check if a sequence number has already been used by this node. + + Returns True if this is a REPLAY (bad), False if fresh (good). + """ + return sequence <= self.node_sequences.get(node_id, 0) + + # ─── Queries ────────────────────────────────────────────────────── + + def get_event(self, event_id: str) -> dict | None: + """Look up a single event by ID.""" + idx = self.event_index.get(event_id) + if idx is not None and idx < len(self.events): + return self.events[idx] + return None + + def annotate_event(self, event_id: str, meta: dict) -> bool: + """Attach non-consensus metadata to an event (not part of hash).""" + idx = self.event_index.get(event_id) + if idx is None or idx >= len(self.events): + return False + self.events[idx]["meta"] = meta + self._save() + return True + + def get_events_by_type( + self, + event_type: str, + limit: int = 50, + offset: int = 0, + ) -> list[dict]: + """Get recent events of a specific type (newest first).""" + matching = [] + for e in reversed(self.events): + if e["event_type"] != event_type: + continue + matching.append(e) + return matching[offset : offset + limit] + + def get_events_by_node(self, node_id: str, limit: int = 50) -> list[dict]: + """Get recent events by a specific node (newest first).""" + matching = [] + for e in reversed(self.events): + if e["node_id"] != node_id: + continue + matching.append(e) + return matching[:limit] + + def get_messages( + self, + gate_id: str = "", + limit: int = 50, + offset: int = 0, + ) -> list[dict]: + """Get messages, optionally filtered by gate. + + Returns public-plane 'message' events only. + Returns newest first with message-specific fields extracted. + """ + results = [] + for evt in reversed(self.events): + if evt["event_type"] != "message": + continue + payload = evt.get("payload", {}) + # Skip ephemeral messages that have expired + if payload.get("ephemeral") or payload.get("_ephemeral"): + age = time.time() - evt["timestamp"] + if age > EPHEMERAL_TTL: + continue + # Gate filter + msg_gate = payload.get("gate", "") + if gate_id and msg_gate != gate_id: + continue + # Skip transport-routed messages (Meshtastic/APRS) from InfoNet feed — + # they belong in their own tab. Only direct InfoNet/gate posts appear here. + meta = evt.get("meta", {}) + if payload.get("routed_via") or meta.get("routed_via"): + continue + + results.append( + { + "event_id": evt["event_id"], + "event_type": evt.get("event_type", ""), + "node_id": evt["node_id"], + "message": payload.get("message", payload.get("text", "")), + "ciphertext": payload.get("ciphertext", ""), + "epoch": payload.get("epoch", 0), + "nonce": payload.get("nonce", payload.get("iv", "")), + "sender_ref": payload.get("sender_ref", ""), + "format": payload.get("format", ""), + "destination": payload.get("destination", "broadcast"), + "channel": payload.get("channel", "LongFast"), + "priority": payload.get("priority", "normal"), + "gate": msg_gate, + "timestamp": evt["timestamp"], + "sequence": evt.get("sequence", 0), + "ephemeral": payload.get("ephemeral", payload.get("_ephemeral", False)), + "signature": evt.get("signature", ""), + "public_key": evt.get("public_key", ""), + "public_key_algo": evt.get("public_key_algo", ""), + "protocol_version": evt.get("protocol_version", ""), + } + ) + + if len(results) >= offset + limit: + break + + return results[offset : offset + limit] + + def get_info(self) -> dict: + """Infonet metadata for status display. O(1) via running counters.""" + return { + "protocol": "infonet", + "network_id": NETWORK_ID, + "total_events": len(self.events), + "active_events": self._active_count, + "head_hash": self.head_hash[:16] + "...", + "head_hash_full": self.head_hash, + "chain_lock": self.chain_lock(), + "known_nodes": len(self.node_sequences), + "event_types": dict(self._type_counts), + "chain_size_kb": round(self._chain_bytes / 1024, 1), + "unsigned_events": 0, + } + + # ─── Cleanup ────────────────────────────────────────────────────── + + def cleanup(self): + """Remove expired ephemeral events and old events beyond retention window. + + Note: This breaks the chain linkage for removed events, so we only + remove from the beginning (oldest events). The chain remains valid + from the first remaining event forward. + """ + now = time.time() + retention_cutoff = now - (MESSAGE_RETENTION_DAYS * 86400) + before = len(self.events) + + # Remove events that are both old AND ephemeral-expired + new_events = [] + for evt in self.events: + payload = evt.get("payload", {}) + is_ephemeral = payload.get("ephemeral", payload.get("_ephemeral", False)) + age = now - evt["timestamp"] + + # Keep if: not ephemeral-expired AND within retention window + if is_ephemeral and age > EPHEMERAL_TTL: + continue # Expired ephemeral — drop + if evt["timestamp"] < retention_cutoff and is_ephemeral: + continue # Old ephemeral — drop + + new_events.append(evt) + + if len(new_events) != before: + self.events = new_events + # Rebuild index + self.event_index = {e["event_id"]: i for i, e in enumerate(self.events)} + self._save() + logger.info(f"Infonet cleanup: removed {before - len(new_events)} expired events") + + # ─── Gossip Sync (Future) ──────────────────────────────────────── + + def get_merkle_root(self) -> str: + """Compute a Merkle root hash of the Infonet for sync comparison. + + Two nodes with the same Merkle root have identical chains. + """ + if not self.events: + return GENESIS_HASH + + from services.mesh.mesh_merkle import merkle_root + + leaves = [e["event_id"] for e in self.events] + root = merkle_root(leaves) + return root or GENESIS_HASH + + def get_merkle_proofs(self, start_index: int, count: int) -> dict: + """Return merkle proofs for a contiguous range of events.""" + leaves = [e["event_id"] for e in self.events] + total = len(leaves) + if total == 0: + return {"root": GENESIS_HASH, "total": 0, "start": 0, "proofs": []} + + from services.mesh.mesh_merkle import build_merkle_levels, merkle_proof_from_levels + + start = max(0, start_index) + end = min(total, start + max(0, count)) + levels = build_merkle_levels(leaves) + root = levels[-1][0] if levels else GENESIS_HASH + + proofs = [] + for idx in range(start, end): + proofs.append( + { + "index": idx, + "leaf": leaves[idx], + "proof": merkle_proof_from_levels(levels, idx), + } + ) + + return {"root": root, "total": total, "start": start, "proofs": proofs} + + def get_locator(self, max_entries: int = 32) -> list[str]: + """Build a block locator for fork-aware sync.""" + if not self.events: + return [GENESIS_HASH] + + locator: list[str] = [] + idx = len(self.events) - 1 + step = 1 + count = 0 + + while idx >= 0 and len(locator) < max_entries - 1: + locator.append(self.events[idx]["event_id"]) + if count >= 9: + step *= 2 + idx -= step + count += 1 + + locator.append(GENESIS_HASH) + return locator + + def get_events_after_locator( + self, locator: list[str], limit: int = 100 + ) -> tuple[str, int, list[dict]]: + """Find a common ancestor in the locator and return events after it.""" + if not locator: + locator = [GENESIS_HASH] + + for hsh in locator: + if hsh == GENESIS_HASH: + return GENESIS_HASH, 0, self.events[:limit] + idx = self.event_index.get(hsh) + if idx is not None: + start = idx + 1 + return hsh, start, self.events[start : start + limit] + + return "", -1, [] + + def get_events_after(self, after_hash: str, limit: int = 100) -> list[dict]: + """Get events after a given hash (for delta sync). + + If after_hash is GENESIS_HASH, returns from the beginning. + """ + if after_hash == GENESIS_HASH: + return self.events[:limit] + + # Find the event with this hash + idx = self.event_index.get(after_hash) + if idx is None: + return [] # Hash not found — full sync needed + + return self.events[idx + 1 : idx + 1 + limit] + + +# ─── Module-level singleton ───────────────────────────────────────────── + +infonet = Infonet() +gate_store = GateMessageStore(data_dir=str(GATE_STORE_DIR)) + +# Backwards-compatible alias so existing imports don't break +hashchain = infonet diff --git a/backend/services/mesh/mesh_ibf.py b/backend/services/mesh/mesh_ibf.py new file mode 100644 index 00000000..216ccb38 --- /dev/null +++ b/backend/services/mesh/mesh_ibf.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import base64 +import hashlib +from dataclasses import dataclass +from typing import Iterable, List, Tuple + +KEY_SIZE = 32 +DEFAULT_SEEDS = [0x243F6A8885A308D3, 0x13198A2E03707344, 0xA4093822299F31D0] +FINGERPRINT_SEED = 0xC0FFEE1234567890 + + +def _safe_int(val, default=0) -> int: + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _hash64(data: bytes, seed: int) -> int: + key = seed.to_bytes(8, "little", signed=False) + digest = hashlib.blake2b(data, digest_size=8, key=key).digest() + return int.from_bytes(digest, "little", signed=False) + + +def _fingerprint(data: bytes) -> int: + key = FINGERPRINT_SEED.to_bytes(8, "little", signed=False) + digest = hashlib.blake2b(data, digest_size=8, key=key).digest() + return int.from_bytes(digest, "little", signed=False) + + +def _xor_bytes(a: bytes, b: bytes) -> bytes: + return bytes(x ^ y for x, y in zip(a, b)) + + +def _ensure_key(key: bytes) -> bytes: + if len(key) != KEY_SIZE: + raise ValueError(f"IBF key must be {KEY_SIZE} bytes") + return key + + +def _b64_encode(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _b64_decode(data: str) -> bytes: + return base64.b64decode(data.encode("ascii")) + + +@dataclass +class IBLTCell: + count: int = 0 + key_xor: bytes = b"\x00" * KEY_SIZE + hash_xor: int = 0 + + def add(self, key: bytes, sign: int) -> None: + self.count += sign + self.key_xor = _xor_bytes(self.key_xor, key) + self.hash_xor ^= _fingerprint(key) + + +class IBLT: + def __init__(self, size: int, seeds: List[int] | None = None) -> None: + if size <= 0: + raise ValueError("IBLT size must be positive") + self.size = size + self.seeds = seeds or list(DEFAULT_SEEDS) + self.cells: List[IBLTCell] = [IBLTCell() for _ in range(size)] + + def _indexes(self, key: bytes) -> List[int]: + key = _ensure_key(key) + return [(_hash64(key, seed) % self.size) for seed in self.seeds] + + def insert(self, key: bytes) -> None: + key = _ensure_key(key) + for idx in self._indexes(key): + self.cells[idx].add(key, 1) + + def delete(self, key: bytes) -> None: + key = _ensure_key(key) + for idx in self._indexes(key): + self.cells[idx].add(key, -1) + + def subtract(self, other: "IBLT") -> "IBLT": + if self.size != other.size or self.seeds != other.seeds: + raise ValueError("IBLT mismatch; size or seeds differ") + out = IBLT(self.size, self.seeds) + for i, cell in enumerate(self.cells): + other_cell = other.cells[i] + out.cells[i] = IBLTCell( + count=cell.count - other_cell.count, + key_xor=_xor_bytes(cell.key_xor, other_cell.key_xor), + hash_xor=cell.hash_xor ^ other_cell.hash_xor, + ) + return out + + def decode(self) -> Tuple[bool, List[bytes], List[bytes]]: + plus: List[bytes] = [] + minus: List[bytes] = [] + stack = [i for i, c in enumerate(self.cells) if abs(c.count) == 1] + + while stack: + idx = stack.pop() + cell = self.cells[idx] + if abs(cell.count) != 1: + continue + key = cell.key_xor + if _fingerprint(key) != cell.hash_xor: + continue + sign = 1 if cell.count == 1 else -1 + if sign == 1: + plus.append(key) + else: + minus.append(key) + for j in self._indexes(key): + if j == idx: + continue + self.cells[j].add(key, -sign) + if abs(self.cells[j].count) == 1: + stack.append(j) + self.cells[idx] = IBLTCell() + + success = all( + c.count == 0 and c.hash_xor == 0 and c.key_xor == b"\x00" * KEY_SIZE + for c in self.cells + ) + return success, plus, minus + + def to_compact_dict(self) -> dict: + return { + "m": self.size, + "s": self.seeds, + "c": [[cell.count, _b64_encode(cell.key_xor), cell.hash_xor] for cell in self.cells], + } + + @classmethod + def from_compact_dict(cls, data: dict) -> "IBLT": + size = _safe_int(data.get("m", 0) or 0) + seeds = data.get("s") or list(DEFAULT_SEEDS) + cells = data.get("c") or [] + iblt = cls(size, list(seeds)) + if len(cells) != size: + raise ValueError("IBLT cell count mismatch") + for i, raw in enumerate(cells): + count, key_b64, hash_xor = raw + iblt.cells[i] = IBLTCell( + count=_safe_int(count, 0), + key_xor=_b64_decode(str(key_b64)), + hash_xor=_safe_int(hash_xor, 0), + ) + return iblt + + +def build_iblt(keys: Iterable[bytes], size: int) -> IBLT: + iblt = IBLT(size) + for key in keys: + iblt.insert(key) + return iblt + + +def minhash_sketch(keys: Iterable[bytes], k: int) -> List[int]: + if k <= 0: + return [] + mins: List[int] = [] + for key in keys: + h = _hash64(key, 0x9E3779B97F4A7C15) + if len(mins) < k: + mins.append(h) + mins.sort() + elif h < mins[-1]: + mins[-1] = h + mins.sort() + return mins + + +def minhash_similarity(a: Iterable[int], b: Iterable[int]) -> float: + a_list = list(a) + b_list = list(b) + if not a_list or not b_list: + return 0.0 + k = min(len(a_list), len(b_list)) + if k <= 0: + return 0.0 + a_set = set(a_list[:k]) + b_set = set(b_list[:k]) + return len(a_set & b_set) / float(k) diff --git a/backend/services/mesh/mesh_infonet_sync_support.py b/backend/services/mesh/mesh_infonet_sync_support.py new file mode 100644 index 00000000..778f2d05 --- /dev/null +++ b/backend/services/mesh/mesh_infonet_sync_support.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import time +from dataclasses import asdict, dataclass + +from services.mesh.mesh_peer_store import PeerRecord + + +@dataclass(frozen=True) +class SyncWorkerState: + last_sync_started_at: int = 0 + last_sync_finished_at: int = 0 + last_sync_ok_at: int = 0 + next_sync_due_at: int = 0 + last_peer_url: str = "" + last_error: str = "" + last_outcome: str = "idle" + current_head: str = "" + fork_detected: bool = False + consecutive_failures: int = 0 + + def to_dict(self) -> dict[str, object]: + return asdict(self) + + +def eligible_sync_peers(records: list[PeerRecord], *, now: float | None = None) -> list[PeerRecord]: + current_time = int(now if now is not None else time.time()) + candidates = [ + record + for record in records + if record.bucket == "sync" and record.enabled and int(record.cooldown_until or 0) <= current_time + ] + return sorted( + candidates, + key=lambda record: ( + -int(record.last_sync_ok_at or 0), + int(record.failure_count or 0), + int(record.added_at or 0), + record.peer_url, + ), + ) + + +def begin_sync( + state: SyncWorkerState, + *, + peer_url: str = "", + current_head: str = "", + now: float | None = None, +) -> SyncWorkerState: + timestamp = int(now if now is not None else time.time()) + return SyncWorkerState( + last_sync_started_at=timestamp, + last_sync_finished_at=state.last_sync_finished_at, + last_sync_ok_at=state.last_sync_ok_at, + next_sync_due_at=state.next_sync_due_at, + last_peer_url=peer_url or state.last_peer_url, + last_error="", + last_outcome="running", + current_head=current_head or state.current_head, + fork_detected=False, + consecutive_failures=state.consecutive_failures, + ) + + +def finish_sync( + state: SyncWorkerState, + *, + ok: bool, + peer_url: str = "", + current_head: str = "", + error: str = "", + fork_detected: bool = False, + now: float | None = None, + interval_s: int = 300, + failure_backoff_s: int = 60, +) -> SyncWorkerState: + timestamp = int(now if now is not None else time.time()) + if ok: + return SyncWorkerState( + last_sync_started_at=state.last_sync_started_at, + last_sync_finished_at=timestamp, + last_sync_ok_at=timestamp, + next_sync_due_at=timestamp + max(0, int(interval_s or 0)), + last_peer_url=peer_url or state.last_peer_url, + last_error="", + last_outcome="ok", + current_head=current_head or state.current_head, + fork_detected=bool(fork_detected), + consecutive_failures=0, + ) + + return SyncWorkerState( + last_sync_started_at=state.last_sync_started_at, + last_sync_finished_at=timestamp, + last_sync_ok_at=state.last_sync_ok_at, + next_sync_due_at=timestamp + max(0, int(failure_backoff_s or 0)), + last_peer_url=peer_url or state.last_peer_url, + last_error=str(error or "").strip(), + last_outcome="fork" if fork_detected else "error", + current_head=current_head or state.current_head, + fork_detected=bool(fork_detected), + consecutive_failures=state.consecutive_failures + 1, + ) + + +def should_run_sync( + state: SyncWorkerState, + *, + now: float | None = None, +) -> bool: + current_time = int(now if now is not None else time.time()) + if state.last_outcome == "running": + return False + return int(state.next_sync_due_at or 0) <= current_time diff --git a/backend/services/mesh/mesh_merkle.py b/backend/services/mesh/mesh_merkle.py new file mode 100644 index 00000000..c5a9f537 --- /dev/null +++ b/backend/services/mesh/mesh_merkle.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import hashlib +from typing import Any + + +def _hash_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def hash_leaf(value: str) -> str: + return _hash_bytes(value.encode("utf-8")) + + +def hash_pair(left: str, right: str) -> str: + return _hash_bytes(f"{left}{right}".encode("utf-8")) + + +def build_merkle_levels(leaves: list[str]) -> list[list[str]]: + if not leaves: + return [] + level = [hash_leaf(leaf) for leaf in leaves] + levels = [level] + while len(level) > 1: + next_level: list[str] = [] + for idx in range(0, len(level), 2): + left = level[idx] + right = level[idx + 1] if idx + 1 < len(level) else left + next_level.append(hash_pair(left, right)) + level = next_level + levels.append(level) + return levels + + +def merkle_root(leaves: list[str]) -> str: + levels = build_merkle_levels(leaves) + if not levels: + return "" + return levels[-1][0] + + +def merkle_proof_from_levels(levels: list[list[str]], index: int) -> list[dict[str, Any]]: + if not levels: + return [] + if index < 0 or index >= len(levels[0]): + return [] + proof: list[dict[str, Any]] = [] + idx = index + for level in levels[:-1]: + is_right = idx % 2 == 1 + sibling_idx = idx - 1 if is_right else idx + 1 + if sibling_idx >= len(level): + sibling_hash = level[idx] + else: + sibling_hash = level[sibling_idx] + proof.append({"hash": sibling_hash, "side": "left" if is_right else "right"}) + idx //= 2 + return proof + + +def verify_merkle_proof( + leaf_value: str, index: int, proof: list[dict[str, Any]], root: str +) -> bool: + current = hash_leaf(leaf_value) + idx = index + for step in proof: + sibling = str(step.get("hash", "")) + side = str(step.get("side", "right")).lower() + if side == "left": + current = hash_pair(sibling, current) + else: + current = hash_pair(current, sibling) + idx //= 2 + return current == root diff --git a/backend/services/mesh/mesh_metrics.py b/backend/services/mesh/mesh_metrics.py new file mode 100644 index 00000000..e8883cb6 --- /dev/null +++ b/backend/services/mesh/mesh_metrics.py @@ -0,0 +1,25 @@ +"""Lightweight metrics for mesh protocol health signals.""" + +from __future__ import annotations + +import threading +import time + +_lock = threading.Lock() +_metrics: dict[str, int] = {} +_last_updated: float = 0.0 + + +def increment(name: str, count: int = 1) -> None: + global _last_updated + with _lock: + _metrics[name] = _metrics.get(name, 0) + count + _last_updated = time.time() + + +def snapshot() -> dict: + with _lock: + return { + "updated_at": _last_updated, + "counters": dict(_metrics), + } diff --git a/backend/services/mesh/mesh_oracle.py b/backend/services/mesh/mesh_oracle.py new file mode 100644 index 00000000..812c81ab --- /dev/null +++ b/backend/services/mesh/mesh_oracle.py @@ -0,0 +1,899 @@ +"""Oracle System — prediction-backed truth arbitration for the mesh. + +Oracle Rep is a separate reputation tier earned ONLY by: + 1. Correctly predicting outcomes on Kalshi/Polymarket-sourced markets + 2. Winning truth stakes on posts/comments + +Oracle Rep can be staked on posts to protect them from mob downvoting. +Other oracles can counter-stake. After the stake period (1-7 days), +whichever side has more oracle rep staked wins. Losers' rep is divided +proportionally among winners. + +Scoring formula for predictions: + oracle_rep_earned = 1.0 - probability_of_chosen_outcome / 100 + - Bet YES at 99% → earn 0.01 (trivial, everyone knew) + - Bet YES at 50% → earn 0.50 (genuine uncertainty, real insight) + - Bet YES at 10% → earn 0.90 (contrarian genius if correct) + +Designed for AI game theory: this mechanism works identically +whether participants are humans, AI agents, or a mix. + +Persistence: JSON files in backend/data/ (auto-saved on change). +""" + +import json +import time +import logging +import secrets +import threading +import atexit +from pathlib import Path +from typing import Optional + +logger = logging.getLogger("services.mesh_oracle") + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +ORACLE_FILE = DATA_DIR / "oracle_ledger.json" + +# ─── Constants ──────────────────────────────────────────────────────────── + +MIN_STAKE_DAYS = 1 # Minimum stake duration +MAX_STAKE_DAYS = 7 # Maximum stake duration +GRACE_PERIOD_HOURS = 24 # Counter-stakers get 24h after any new stake +ORACLE_DECAY_DAYS = 90 # Oracle rep decays over 90 days like regular rep + + +class OracleLedger: + """Oracle reputation ledger — predictions, stakes, and truth arbitration. + + Storage: + oracle_rep: {node_id: float} — current oracle rep balances + predictions: [{node_id, market_title, side, probability_at_bet, timestamp, resolved, correct, rep_earned}] + stakes: [{stake_id, message_id, poster_id, staker_id, side ("truth"|"false"), + amount, duration_days, created_at, expires_at, resolved}] + prediction_log: [{node_id, market_title, side, probability_at_bet, rep_earned, timestamp}] + """ + + def __init__(self): + self.oracle_rep: dict[str, float] = {} + self.predictions: list[dict] = [] + self.market_stakes: list[dict] = [] # Rep staked on prediction markets + self.stakes: list[dict] = [] # Truth stakes on posts (separate system) + self.prediction_log: list[dict] = [] # Public log of all predictions + self._dirty = False + self._save_lock = threading.Lock() + self._save_timer: threading.Timer | None = None + self._SAVE_INTERVAL = 5.0 + atexit.register(self._flush) + self._load() + + # ─── Persistence ────────────────────────────────────────────────── + + def _load(self): + if ORACLE_FILE.exists(): + try: + data = json.loads(ORACLE_FILE.read_text(encoding="utf-8")) + self.oracle_rep = data.get("oracle_rep", {}) + self.predictions = data.get("predictions", []) + self.market_stakes = data.get("market_stakes", []) + self.stakes = data.get("stakes", []) + self.prediction_log = data.get("prediction_log", []) + logger.info( + f"Loaded oracle ledger: {len(self.oracle_rep)} oracles, " + f"{len(self.predictions)} predictions, " + f"{len(self.market_stakes)} market stakes, {len(self.stakes)} truth stakes" + ) + except Exception as e: + logger.error(f"Failed to load oracle ledger: {e}") + + def _save(self): + """Mark dirty and schedule a coalesced disk write.""" + self._dirty = True + with self._save_lock: + if self._save_timer is None or not self._save_timer.is_alive(): + self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush) + self._save_timer.daemon = True + self._save_timer.start() + + def _flush(self): + """Actually write to disk (called by timer or atexit).""" + if not self._dirty: + return + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + data = { + "oracle_rep": self.oracle_rep, + "predictions": self.predictions, + "market_stakes": self.market_stakes, + "stakes": self.stakes, + "prediction_log": self.prediction_log, + } + ORACLE_FILE.write_text(json.dumps(data, indent=2), encoding="utf-8") + self._dirty = False + except Exception as e: + logger.error(f"Failed to save oracle ledger: {e}") + + # ─── Oracle Rep ─────────────────────────────────────────────────── + + def get_oracle_rep(self, node_id: str) -> float: + """Get current oracle rep for a node (excludes locked/staked amount).""" + total = self.oracle_rep.get(node_id, 0.0) + # Subtract locked truth stakes on posts + locked = sum( + s["amount"] + for s in self.stakes + if s["staker_id"] == node_id and not s.get("resolved", False) + ) + # Subtract locked market stakes + locked += sum( + s["amount"] + for s in self.market_stakes + if s["node_id"] == node_id and not s.get("resolved", False) + ) + return round(max(0, total - locked), 3) + + def get_total_oracle_rep(self, node_id: str) -> float: + """Get total oracle rep including locked stakes.""" + return round(self.oracle_rep.get(node_id, 0.0), 3) + + def _add_oracle_rep(self, node_id: str, amount: float): + """Add oracle rep to a node.""" + self.oracle_rep[node_id] = self.oracle_rep.get(node_id, 0.0) + amount + + def _remove_oracle_rep(self, node_id: str, amount: float): + """Remove oracle rep from a node (floor at 0).""" + self.oracle_rep[node_id] = max(0, self.oracle_rep.get(node_id, 0.0) - amount) + + # ─── Predictions ────────────────────────────────────────────────── + + def place_prediction( + self, node_id: str, market_title: str, side: str, probability_at_bet: float + ) -> tuple[bool, str]: + """Place a FREE prediction on a market outcome (no rep risked). + + Args: + node_id: Predictor's node ID + market_title: Title of the prediction market + side: "yes", "no", or any outcome name for multi-outcome markets + probability_at_bet: Current probability (0-100) of the chosen side + + Returns (success, detail) + """ + if not side or not side.strip(): + return False, "Side is required" + + if not (0 <= probability_at_bet <= 100): + return False, "Probability must be 0-100" + + # Check for duplicate predictions on same market + existing = [ + p + for p in self.predictions + if p["node_id"] == node_id + and p["market_title"] == market_title + and not p.get("resolved", False) + ] + if existing: + return ( + False, + f"You already have an active prediction on '{market_title}'. Your decision was FINAL.", + ) + + # Also check market stakes — can't free-pick AND stake on same market + existing_stake = [ + s + for s in self.market_stakes + if s["node_id"] == node_id + and s["market_title"] == market_title + and not s.get("resolved", False) + ] + if existing_stake: + return ( + False, + f"You already have a STAKED prediction on '{market_title}'. Your decision was FINAL.", + ) + + self.predictions.append( + { + "prediction_id": secrets.token_hex(6), + "node_id": node_id, + "market_title": market_title, + "side": side, + "probability_at_bet": probability_at_bet, + "timestamp": time.time(), + "resolved": False, + "correct": None, + "rep_earned": 0.0, + } + ) + self._save() + + # Potential rep = contrarianism score + potential = round(1.0 - probability_at_bet / 100, 3) + + logger.info( + f"FREE prediction: {node_id} picks '{side}' on '{market_title}' " + f"at {probability_at_bet}% (potential: {potential} oracle rep)" + ) + return True, ( + f"FREE PICK placed: {side.upper()} on '{market_title}' " + f"at {probability_at_bet}%. Potential oracle rep: {potential}. " + f"This decision is FINAL." + ) + + def resolve_market(self, market_title: str, outcome: str) -> tuple[int, int]: + """Resolve all FREE predictions on a market. + + Args: + market_title: Title of the market + outcome: "yes", "no", or any outcome name for multi-outcome markets + + Returns (winners, losers) counts + """ + if not outcome: + return 0, 0 + + outcome_lower = outcome.lower() + winners, losers = 0, 0 + now = time.time() + + for p in self.predictions: + if p["market_title"] != market_title or p.get("resolved", False): + continue + + p["resolved"] = True + correct = p["side"].lower() == outcome_lower + p["correct"] = correct + + if correct: + # Rep earned = contrarianism score + rep = round(1.0 - p["probability_at_bet"] / 100, 3) + rep = max(0.01, rep) # Minimum 0.01 even for easy bets + p["rep_earned"] = rep + self._add_oracle_rep(p["node_id"], rep) + winners += 1 + + self.prediction_log.append( + { + "node_id": p["node_id"], + "market_title": market_title, + "side": p["side"], + "outcome": outcome, + "probability_at_bet": p["probability_at_bet"], + "rep_earned": rep, + "timestamp": p["timestamp"], + "resolved_at": now, + } + ) + logger.info( + f"Oracle win: {p['node_id']} earned {rep} oracle rep " + f"on '{market_title}' ({p['side']} at {p['probability_at_bet']}%)" + ) + else: + p["rep_earned"] = 0.0 + losers += 1 + + self.prediction_log.append( + { + "node_id": p["node_id"], + "market_title": market_title, + "side": p["side"], + "outcome": outcome, + "probability_at_bet": p["probability_at_bet"], + "rep_earned": 0.0, + "timestamp": p["timestamp"], + "resolved_at": now, + } + ) + + self._save() + return winners, losers + + def get_active_markets(self) -> list[str]: + """Get list of market titles with unresolved predictions or stakes.""" + titles = set() + for p in self.predictions: + if not p.get("resolved", False): + titles.add(p["market_title"]) + for s in self.market_stakes: + if not s.get("resolved", False): + titles.add(s["market_title"]) + return list(titles) + + # ─── Market Stakes (prediction markets) ──────────────────────────── + + def place_market_stake( + self, node_id: str, market_title: str, side: str, amount: float, probability_at_bet: float + ) -> tuple[bool, str]: + """Stake oracle rep on a prediction market outcome. FINAL decision. + + Args: + node_id: Staker's node ID + market_title: Title of the prediction market + side: "yes", "no", or outcome name for multi-outcome markets + amount: How much oracle rep to risk + probability_at_bet: Current probability (0-100) of the chosen side + + Returns (success, detail) + """ + if not side or not side.strip(): + return False, "Side is required" + + if amount <= 0: + return False, "Stake amount must be positive" + + if not (0 <= probability_at_bet <= 100): + return False, "Probability must be 0-100" + + available = self.get_oracle_rep(node_id) + if available < amount: + return False, f"Insufficient oracle rep (have {available:.2f}, need {amount:.2f})" + + # Can't have both a free pick AND a stake on the same market + existing_free = [ + p + for p in self.predictions + if p["node_id"] == node_id + and p["market_title"] == market_title + and not p.get("resolved", False) + ] + if existing_free: + return ( + False, + f"You already have a FREE prediction on '{market_title}'. Your decision was FINAL.", + ) + + # Can't stake twice on the same market + existing_stake = [ + s + for s in self.market_stakes + if s["node_id"] == node_id + and s["market_title"] == market_title + and not s.get("resolved", False) + ] + if existing_stake: + return ( + False, + f"You already have a STAKED prediction on '{market_title}'. Your decision was FINAL.", + ) + + self.market_stakes.append( + { + "stake_id": secrets.token_hex(6), + "node_id": node_id, + "market_title": market_title, + "side": side, + "amount": amount, + "probability_at_bet": probability_at_bet, + "timestamp": time.time(), + "resolved": False, + "correct": None, + "rep_earned": 0.0, + } + ) + self._save() + + logger.info( + f"MARKET STAKE: {node_id} stakes {amount:.2f} rep on '{side}' " + f"for '{market_title}' at {probability_at_bet}%" + ) + return True, ( + f"STAKED {amount:.2f} oracle rep on {side.upper()} for '{market_title}' " + f"at {probability_at_bet}%. This decision is FINAL. " + f"If correct, you split the loser pool proportionally." + ) + + def resolve_market_stakes(self, market_title: str, outcome: str) -> dict: + """Resolve all market stakes for a concluded market. + + Winners split the loser pool proportionally to their stake. + If everyone picked the same side, stakes are returned (no profit, no loss). + + Returns summary dict. + """ + if not outcome: + return {"resolved": 0} + + outcome_lower = outcome.lower() + active = [ + s + for s in self.market_stakes + if s["market_title"] == market_title and not s.get("resolved", False) + ] + + if not active: + return {"resolved": 0} + + winners = [s for s in active if s["side"].lower() == outcome_lower] + losers = [s for s in active if s["side"].lower() != outcome_lower] + + winner_pool = sum(s["amount"] for s in winners) + loser_pool = sum(s["amount"] for s in losers) + + now = time.time() + + if not losers: + # Everyone picked the same side — return stakes, no profit + for s in active: + s["resolved"] = True + s["correct"] = True + s["rep_earned"] = 0.0 # No profit when no opposition + self._save() + logger.info( + f"Market stake resolution [{market_title}]: unanimous '{outcome}', " + f"{len(winners)} stakers get rep back (no loser pool)" + ) + return { + "resolved": len(active), + "winners": len(winners), + "losers": 0, + "winner_pool": winner_pool, + "loser_pool": 0, + "unanimous": True, + } + + # Losers lose their staked rep + for s in losers: + self._remove_oracle_rep(s["node_id"], s["amount"]) + s["resolved"] = True + s["correct"] = False + s["rep_earned"] = 0.0 + + self.prediction_log.append( + { + "node_id": s["node_id"], + "market_title": market_title, + "side": s["side"], + "outcome": outcome, + "probability_at_bet": s["probability_at_bet"], + "rep_earned": 0.0, + "staked": s["amount"], + "timestamp": s["timestamp"], + "resolved_at": now, + } + ) + + # Winners split loser pool proportionally + keep their own stake + for s in winners: + proportion = s["amount"] / winner_pool if winner_pool > 0 else 0 + winnings = round(loser_pool * proportion, 3) + s["resolved"] = True + s["correct"] = True + s["rep_earned"] = winnings + self._add_oracle_rep(s["node_id"], winnings) + + self.prediction_log.append( + { + "node_id": s["node_id"], + "market_title": market_title, + "side": s["side"], + "outcome": outcome, + "probability_at_bet": s["probability_at_bet"], + "rep_earned": winnings, + "staked": s["amount"], + "timestamp": s["timestamp"], + "resolved_at": now, + } + ) + + self._save() + logger.info( + f"Market stake resolution [{market_title}]: '{outcome}' wins. " + f"{len(winners)} winners split {loser_pool:.2f} rep from {len(losers)} losers" + ) + return { + "resolved": len(active), + "winners": len(winners), + "losers": len(losers), + "winner_pool": round(winner_pool, 3), + "loser_pool": round(loser_pool, 3), + } + + def get_market_consensus(self, market_title: str) -> dict: + """Get network consensus for a single market — picks + stakes per side.""" + sides: dict[str, dict] = {} + + # Count free predictions + for p in self.predictions: + if p["market_title"] != market_title or p.get("resolved", False): + continue + s = p["side"] + if s not in sides: + sides[s] = {"picks": 0, "staked": 0.0} + sides[s]["picks"] += 1 + + # Count market stakes + for st in self.market_stakes: + if st["market_title"] != market_title or st.get("resolved", False): + continue + s = st["side"] + if s not in sides: + sides[s] = {"picks": 0, "staked": 0.0} + sides[s]["picks"] += 1 + sides[s]["staked"] = round(sides[s]["staked"] + st["amount"], 3) + + total_picks = sum(v["picks"] for v in sides.values()) + total_staked = round(sum(v["staked"] for v in sides.values()), 3) + + return { + "market_title": market_title, + "total_picks": total_picks, + "total_staked": total_staked, + "sides": sides, + } + + def get_all_market_consensus(self) -> dict[str, dict]: + """Bulk consensus for all active markets. Returns {market_title: consensus_summary}.""" + titles = set() + for p in self.predictions: + if not p.get("resolved", False): + titles.add(p["market_title"]) + for s in self.market_stakes: + if not s.get("resolved", False): + titles.add(s["market_title"]) + + result = {} + for title in titles: + c = self.get_market_consensus(title) + result[title] = { + "total_picks": c["total_picks"], + "total_staked": c["total_staked"], + "sides": c["sides"], + } + return result + + # ─── Truth Stakes (posts/comments — separate system) ────────────── + + def place_stake( + self, + staker_id: str, + message_id: str, + poster_id: str, + side: str, + amount: float, + duration_days: int, + ) -> tuple[bool, str]: + """Stake oracle rep on a post's truthfulness. + + Args: + staker_id: Oracle staking their rep + message_id: The post/message being evaluated + poster_id: Who posted the original message + side: "truth" or "false" + amount: How much oracle rep to stake + duration_days: 1-7 days before resolution + + Returns (success, detail) + """ + if side not in ("truth", "false"): + return False, "Side must be 'truth' or 'false'" + + if not (MIN_STAKE_DAYS <= duration_days <= MAX_STAKE_DAYS): + return False, f"Duration must be {MIN_STAKE_DAYS}-{MAX_STAKE_DAYS} days" + + if amount <= 0: + return False, "Stake amount must be positive" + + available = self.get_oracle_rep(staker_id) + if available < amount: + return False, f"Insufficient oracle rep (have {available}, need {amount})" + + # Check if this staker already has an active stake on this message + existing = [ + s + for s in self.stakes + if s["staker_id"] == staker_id + and s["message_id"] == message_id + and not s.get("resolved", False) + ] + if existing: + return False, "You already have an active stake on this message" + + now = time.time() + expires = now + (duration_days * 86400) + + # Check if there are existing stakes — extend grace period + active_stakes = [ + s for s in self.stakes if s["message_id"] == message_id and not s.get("resolved", False) + ] + # If this is a counter-stake, ensure the expiry is at least GRACE_PERIOD_HOURS + # after the latest stake on the other side + for s in active_stakes: + if s["side"] != side: + min_expires = s.get("last_counter_at", s["created_at"]) + ( + GRACE_PERIOD_HOURS * 3600 + ) + if expires < min_expires: + expires = min_expires + + stake = { + "stake_id": secrets.token_hex(6), + "message_id": message_id, + "poster_id": poster_id, + "staker_id": staker_id, + "side": side, + "amount": amount, + "duration_days": duration_days, + "created_at": now, + "expires_at": expires, + "resolved": False, + "last_counter_at": now, + } + self.stakes.append(stake) + + # Update last_counter_at on opposing stakes (extends their grace period) + for s in active_stakes: + if s["side"] != side: + s["last_counter_at"] = now + + self._save() + + days_str = f"{duration_days} day{'s' if duration_days > 1 else ''}" + logger.info( + f"Oracle stake: {staker_id} stakes {amount} oracle rep " + f"as '{side}' on message {message_id} for {days_str}" + ) + return True, ( + f"Staked {amount} oracle rep as '{side.upper()}' on message " + f"{message_id} for {days_str}. Expires {time.strftime('%Y-%m-%d %H:%M', time.localtime(expires))}" + ) + + def resolve_expired_stakes(self) -> list[dict]: + """Resolve all expired stake contests. Called periodically. + + Returns list of resolution summaries. + """ + now = time.time() + resolutions = [] + + # Group active stakes by message_id + active_by_msg: dict[str, list[dict]] = {} + for s in self.stakes: + if not s.get("resolved", False): + active_by_msg.setdefault(s["message_id"], []).append(s) + + for msg_id, stakes in active_by_msg.items(): + # Check if ALL stakes for this message have expired + if not all(s["expires_at"] <= now for s in stakes): + continue # Some stakes haven't expired yet + + # Tally sides + truth_total = sum(s["amount"] for s in stakes if s["side"] == "truth") + false_total = sum(s["amount"] for s in stakes if s["side"] == "false") + + if truth_total == false_total: + # Tie — everyone gets their rep back, no resolution + for s in stakes: + s["resolved"] = True + resolutions.append( + { + "message_id": msg_id, + "outcome": "tie", + "truth_total": truth_total, + "false_total": false_total, + } + ) + continue + + winning_side = "truth" if truth_total > false_total else "false" + losing_total = false_total if winning_side == "truth" else truth_total + winning_total = truth_total if winning_side == "truth" else false_total + + winners = [s for s in stakes if s["side"] == winning_side] + losers = [s for s in stakes if s["side"] != winning_side] + + # Losers lose their staked rep + for s in losers: + self._remove_oracle_rep(s["staker_id"], s["amount"]) + s["resolved"] = True + + # Winners divide losers' rep proportionally + for s in winners: + proportion = s["amount"] / winning_total if winning_total > 0 else 0 + winnings = round(losing_total * proportion, 3) + self._add_oracle_rep(s["staker_id"], winnings) + s["resolved"] = True + + # Duration weight for the poster's reputation effect + max_duration = max(s["duration_days"] for s in stakes) + duration_label = ( + "resounding" if max_duration >= 7 else "contested" if max_duration >= 3 else "brief" + ) + + resolution = { + "message_id": msg_id, + "poster_id": stakes[0].get("poster_id", ""), + "outcome": winning_side, + "truth_total": round(truth_total, 3), + "false_total": round(false_total, 3), + "duration_label": duration_label, + "max_duration_days": max_duration, + "winners": [ + { + "node_id": s["staker_id"], + "staked": s["amount"], + "won": ( + round(losing_total * (s["amount"] / winning_total), 3) + if winning_total > 0 + else 0 + ), + } + for s in winners + ], + "losers": [ + { + "node_id": s["staker_id"], + "lost": s["amount"], + } + for s in losers + ], + } + resolutions.append(resolution) + logger.info( + f"Oracle resolution [{msg_id}]: {winning_side.upper()} wins " + f"({truth_total} vs {false_total}), {duration_label} verdict" + ) + + if resolutions: + self._save() + return resolutions + + def get_stakes_for_message(self, message_id: str) -> dict: + """Get all stakes on a message with totals.""" + active = [ + s for s in self.stakes if s["message_id"] == message_id and not s.get("resolved", False) + ] + truth_stakes = [s for s in active if s["side"] == "truth"] + false_stakes = [s for s in active if s["side"] == "false"] + + return { + "message_id": message_id, + "truth_total": round(sum(s["amount"] for s in truth_stakes), 3), + "false_total": round(sum(s["amount"] for s in false_stakes), 3), + "truth_stakers": [ + {"node_id": s["staker_id"], "amount": s["amount"], "expires": s["expires_at"]} + for s in truth_stakes + ], + "false_stakers": [ + {"node_id": s["staker_id"], "amount": s["amount"], "expires": s["expires_at"]} + for s in false_stakes + ], + "earliest_expiry": min((s["expires_at"] for s in active), default=0), + } + + # ─── Oracle Profile ─────────────────────────────────────────────── + + def get_oracle_profile(self, node_id: str) -> dict: + """Full oracle profile — rep, prediction history, active stakes.""" + total_rep = self.get_total_oracle_rep(node_id) + available_rep = self.get_oracle_rep(node_id) + + # Prediction stats + my_predictions = [p for p in self.prediction_log if p["node_id"] == node_id] + wins = [p for p in my_predictions if p["rep_earned"] > 0] + losses = [p for p in my_predictions if p["rep_earned"] == 0] + + # Active stakes + active_stakes = [ + { + "message_id": s["message_id"], + "side": s["side"], + "amount": s["amount"], + "expires": s["expires_at"], + } + for s in self.stakes + if s["staker_id"] == node_id and not s.get("resolved", False) + ] + + # Recent prediction log (last 20) + recent = sorted(my_predictions, key=lambda x: x.get("resolved_at", 0), reverse=True)[:20] + prediction_history = [ + { + "market": p["market_title"][:50], + "side": p["side"], + "probability": p["probability_at_bet"], + "outcome": p.get("outcome", "?"), + "rep_earned": p["rep_earned"], + "correct": p["rep_earned"] > 0, + "age": f"{int((time.time() - p.get('resolved_at', p['timestamp'])) / 86400)}d ago", + } + for p in recent + ] + + # Farming score — what % of bets were on >80% probability outcomes + if my_predictions: + easy_bets = sum( + 1 + for p in my_predictions + if (p["side"] == "yes" and p["probability_at_bet"] > 80) + or (p["side"] == "no" and p["probability_at_bet"] < 20) + ) + farming_pct = round(easy_bets / len(my_predictions) * 100) + else: + farming_pct = 0 + + return { + "node_id": node_id, + "oracle_rep": available_rep, + "oracle_rep_total": total_rep, + "oracle_rep_locked": round(total_rep - available_rep, 3), + "predictions_won": len(wins), + "predictions_lost": len(losses), + "win_rate": round(len(wins) / max(1, len(wins) + len(losses)) * 100), + "farming_pct": farming_pct, + "active_stakes": active_stakes, + "prediction_history": prediction_history, + } + + def get_active_predictions(self, node_id: str) -> list[dict]: + """Get a node's unresolved predictions (free picks + staked).""" + results = [] + now = time.time() + + # Free picks + for p in self.predictions: + if p["node_id"] != node_id or p.get("resolved", False): + continue + potential = round(1.0 - p["probability_at_bet"] / 100, 3) + days = int((now - p["timestamp"]) / 86400) + results.append( + { + "prediction_id": p["prediction_id"], + "market_title": p["market_title"], + "side": p["side"], + "probability_at_bet": p["probability_at_bet"], + "potential_rep": potential, + "staked": 0, + "mode": "free", + "placed": f"{days}d ago", + } + ) + + # Market stakes + for s in self.market_stakes: + if s["node_id"] != node_id or s.get("resolved", False): + continue + days = int((now - s["timestamp"]) / 86400) + results.append( + { + "prediction_id": s["stake_id"], + "market_title": s["market_title"], + "side": s["side"], + "probability_at_bet": s["probability_at_bet"], + "potential_rep": 0, # Depends on loser pool — unknown until resolution + "staked": s["amount"], + "mode": "staked", + "placed": f"{days}d ago", + } + ) + + return results + + # ─── Cleanup ────────────────────────────────────────────────────── + + def cleanup_old_data(self): + """Remove resolved predictions and market stakes older than decay window.""" + cutoff = time.time() - (ORACLE_DECAY_DAYS * 86400) + before_pred = len(self.predictions) + before_stakes = len(self.market_stakes) + self.predictions = [ + p for p in self.predictions if not p.get("resolved", False) or p["timestamp"] >= cutoff + ] + self.market_stakes = [ + s + for s in self.market_stakes + if not s.get("resolved", False) or s["timestamp"] >= cutoff + ] + # Trim prediction log + self.prediction_log = [ + p for p in self.prediction_log if p.get("resolved_at", p["timestamp"]) >= cutoff + ] + removed = (before_pred - len(self.predictions)) + (before_stakes - len(self.market_stakes)) + if removed: + self._save() + logger.info(f"Cleaned up {removed} old predictions/stakes") + + +# ─── Module-level singleton ────────────────────────────────────────────── + +oracle_ledger = OracleLedger() diff --git a/backend/services/mesh/mesh_peer_store.py b/backend/services/mesh/mesh_peer_store.py new file mode 100644 index 00000000..39b0ec70 --- /dev/null +++ b/backend/services/mesh/mesh_peer_store.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import json +import os +import tempfile +import time +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from services.mesh.mesh_crypto import normalize_peer_url + +BACKEND_DIR = Path(__file__).resolve().parents[2] +DATA_DIR = BACKEND_DIR / "data" +DEFAULT_PEER_STORE_PATH = DATA_DIR / "peer_store.json" +PEER_STORE_VERSION = 1 +ALLOWED_PEER_BUCKETS = {"bootstrap", "sync", "push"} +ALLOWED_PEER_SOURCES = {"bundle", "operator", "bootstrap_promoted", "runtime"} +ALLOWED_PEER_TRANSPORTS = {"clearnet", "onion"} +ALLOWED_PEER_ROLES = {"participant", "relay", "seed"} + + +class PeerStoreError(ValueError): + pass + + +def _atomic_write_text(target: Path, content: str) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=str(target.parent), suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, str(target)) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +@dataclass(frozen=True) +class PeerRecord: + bucket: str + source: str + peer_url: str + transport: str + role: str + label: str = "" + signer_id: str = "" + enabled: bool = True + added_at: int = 0 + updated_at: int = 0 + last_seen_at: int = 0 + last_sync_ok_at: int = 0 + last_push_ok_at: int = 0 + last_error: str = "" + failure_count: int = 0 + cooldown_until: int = 0 + metadata: dict[str, Any] = field(default_factory=dict) + + def record_key(self) -> str: + return f"{self.bucket}:{self.peer_url}" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _normalize_peer_record(data: dict[str, Any]) -> PeerRecord: + bucket = str(data.get("bucket", "") or "").strip().lower() + source = str(data.get("source", "") or "").strip().lower() + peer_url = str(data.get("peer_url", "") or "").strip() + transport = str(data.get("transport", "") or "").strip().lower() + role = str(data.get("role", "") or "").strip().lower() + label = str(data.get("label", "") or "").strip() + signer_id = str(data.get("signer_id", "") or "").strip() + enabled = bool(data.get("enabled", True)) + metadata = data.get("metadata", {}) + + if bucket not in ALLOWED_PEER_BUCKETS: + raise PeerStoreError(f"unsupported peer bucket: {bucket or 'missing'}") + if source not in ALLOWED_PEER_SOURCES: + raise PeerStoreError(f"unsupported peer source: {source or 'missing'}") + if transport not in ALLOWED_PEER_TRANSPORTS: + raise PeerStoreError(f"unsupported peer transport: {transport or 'missing'}") + if role not in ALLOWED_PEER_ROLES: + raise PeerStoreError(f"unsupported peer role: {role or 'missing'}") + + normalized = normalize_peer_url(peer_url) + if not normalized or normalized != peer_url: + raise PeerStoreError("peer_url must be normalized") + parsed = urlparse(normalized) + hostname = str(parsed.hostname or "").strip().lower() + if transport == "clearnet": + if parsed.scheme not in ("https", "http") or hostname.endswith(".onion"): + raise PeerStoreError("clearnet peers must use https:// (or http:// for LAN/testnet)") + elif transport == "onion": + if parsed.scheme != "http" or not hostname.endswith(".onion"): + raise PeerStoreError("onion peers must use http://*.onion") + + if not isinstance(metadata, dict): + raise PeerStoreError("peer metadata must be an object") + + return PeerRecord( + bucket=bucket, + source=source, + peer_url=normalized, + transport=transport, + role=role, + label=label, + signer_id=signer_id, + enabled=enabled, + added_at=int(data.get("added_at", 0) or 0), + updated_at=int(data.get("updated_at", 0) or 0), + last_seen_at=int(data.get("last_seen_at", 0) or 0), + last_sync_ok_at=int(data.get("last_sync_ok_at", 0) or 0), + last_push_ok_at=int(data.get("last_push_ok_at", 0) or 0), + last_error=str(data.get("last_error", "") or ""), + failure_count=int(data.get("failure_count", 0) or 0), + cooldown_until=int(data.get("cooldown_until", 0) or 0), + metadata=dict(metadata), + ) + + +def make_bootstrap_peer_record( + *, + peer_url: str, + transport: str, + role: str, + signer_id: str, + label: str = "", + now: float | None = None, +) -> PeerRecord: + timestamp = int(now if now is not None else time.time()) + return _normalize_peer_record( + { + "bucket": "bootstrap", + "source": "bundle", + "peer_url": peer_url, + "transport": transport, + "role": role, + "label": label, + "signer_id": signer_id, + "enabled": True, + "added_at": timestamp, + "updated_at": timestamp, + } + ) + + +def make_sync_peer_record( + *, + peer_url: str, + transport: str, + role: str = "participant", + source: str = "operator", + label: str = "", + signer_id: str = "", + now: float | None = None, +) -> PeerRecord: + timestamp = int(now if now is not None else time.time()) + return _normalize_peer_record( + { + "bucket": "sync", + "source": source, + "peer_url": peer_url, + "transport": transport, + "role": role, + "label": label, + "signer_id": signer_id, + "enabled": True, + "added_at": timestamp, + "updated_at": timestamp, + } + ) + + +def make_push_peer_record( + *, + peer_url: str, + transport: str, + role: str = "relay", + source: str = "operator", + label: str = "", + now: float | None = None, +) -> PeerRecord: + timestamp = int(now if now is not None else time.time()) + return _normalize_peer_record( + { + "bucket": "push", + "source": source, + "peer_url": peer_url, + "transport": transport, + "role": role, + "label": label, + "enabled": True, + "added_at": timestamp, + "updated_at": timestamp, + } + ) + + +class PeerStore: + def __init__(self, path: str | Path = DEFAULT_PEER_STORE_PATH): + self.path = Path(path) + self._records: dict[str, PeerRecord] = {} + + def load(self) -> list[PeerRecord]: + if not self.path.exists(): + self._records = {} + return [] + try: + raw = json.loads(self.path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise PeerStoreError("peer store is not valid JSON") from exc + + if not isinstance(raw, dict): + raise PeerStoreError("peer store root must be an object") + version = int(raw.get("version", 0) or 0) + if version != PEER_STORE_VERSION: + raise PeerStoreError(f"unsupported peer store version: {version}") + records_raw = raw.get("records", []) + if not isinstance(records_raw, list): + raise PeerStoreError("peer store records must be a list") + + records: dict[str, PeerRecord] = {} + for entry in records_raw: + if not isinstance(entry, dict): + raise PeerStoreError("peer store records must be objects") + record = _normalize_peer_record(entry) + records[record.record_key()] = record + self._records = records + return self.records() + + def save(self) -> None: + payload = { + "version": PEER_STORE_VERSION, + "records": [record.to_dict() for record in self.records()], + } + _atomic_write_text( + self.path, + json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False), + ) + + def records(self) -> list[PeerRecord]: + return sorted(self._records.values(), key=lambda item: (item.bucket, item.peer_url)) + + def records_for_bucket(self, bucket: str) -> list[PeerRecord]: + normalized_bucket = str(bucket or "").strip().lower() + return [record for record in self.records() if record.bucket == normalized_bucket] + + def upsert(self, record: PeerRecord) -> PeerRecord: + existing = self._records.get(record.record_key()) + if existing is None: + self._records[record.record_key()] = record + return record + + merged = PeerRecord( + bucket=record.bucket, + source=record.source, + peer_url=record.peer_url, + transport=record.transport, + role=record.role, + label=record.label or existing.label, + signer_id=record.signer_id or existing.signer_id, + enabled=record.enabled, + added_at=existing.added_at or record.added_at, + updated_at=max(existing.updated_at, record.updated_at), + last_seen_at=max(existing.last_seen_at, record.last_seen_at), + last_sync_ok_at=max(existing.last_sync_ok_at, record.last_sync_ok_at), + last_push_ok_at=max(existing.last_push_ok_at, record.last_push_ok_at), + last_error=record.last_error or existing.last_error, + failure_count=max(existing.failure_count, record.failure_count), + cooldown_until=max(existing.cooldown_until, record.cooldown_until), + metadata={**existing.metadata, **record.metadata}, + ) + self._records[record.record_key()] = merged + return merged + + def mark_seen(self, peer_url: str, bucket: str, *, now: float | None = None) -> PeerRecord: + record = self._require_record(peer_url, bucket) + timestamp = int(now if now is not None else time.time()) + updated = PeerRecord( + **{ + **record.to_dict(), + "last_seen_at": timestamp, + "updated_at": timestamp, + } + ) + self._records[updated.record_key()] = updated + return updated + + def mark_sync_success(self, peer_url: str, bucket: str = "sync", *, now: float | None = None) -> PeerRecord: + record = self._require_record(peer_url, bucket) + timestamp = int(now if now is not None else time.time()) + updated = PeerRecord( + **{ + **record.to_dict(), + "last_sync_ok_at": timestamp, + "last_error": "", + "failure_count": 0, + "cooldown_until": 0, + "updated_at": timestamp, + } + ) + self._records[updated.record_key()] = updated + return updated + + def mark_push_success(self, peer_url: str, bucket: str = "push", *, now: float | None = None) -> PeerRecord: + record = self._require_record(peer_url, bucket) + timestamp = int(now if now is not None else time.time()) + updated = PeerRecord( + **{ + **record.to_dict(), + "last_push_ok_at": timestamp, + "last_error": "", + "failure_count": 0, + "cooldown_until": 0, + "updated_at": timestamp, + } + ) + self._records[updated.record_key()] = updated + return updated + + def mark_failure( + self, + peer_url: str, + bucket: str, + *, + error: str, + cooldown_s: int = 0, + now: float | None = None, + ) -> PeerRecord: + record = self._require_record(peer_url, bucket) + timestamp = int(now if now is not None else time.time()) + updated = PeerRecord( + **{ + **record.to_dict(), + "last_error": str(error or "").strip(), + "failure_count": int(record.failure_count) + 1, + "cooldown_until": timestamp + max(0, int(cooldown_s or 0)), + "updated_at": timestamp, + } + ) + self._records[updated.record_key()] = updated + return updated + + def _require_record(self, peer_url: str, bucket: str) -> PeerRecord: + normalized_url = normalize_peer_url(peer_url) + key = f"{str(bucket or '').strip().lower()}:{normalized_url}" + if key not in self._records: + raise PeerStoreError(f"peer record not found: {key}") + return self._records[key] diff --git a/backend/services/mesh/mesh_privacy_logging.py b/backend/services/mesh/mesh_privacy_logging.py new file mode 100644 index 00000000..74dbd36a --- /dev/null +++ b/backend/services/mesh/mesh_privacy_logging.py @@ -0,0 +1,19 @@ +"""Helpers for privacy-aware operational logging. + +These helpers keep logs useful for debugging classes of failures without +recording stable private-plane identifiers verbatim. +""" + +from __future__ import annotations + +import hashlib + + +def privacy_log_label(value: str, *, label: str = "") -> str: + raw = str(value or "").strip() + if not raw: + return f"{label}#none" if label else "" + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] + if not label: + return digest + return f"{label}#{digest}" diff --git a/backend/services/mesh/mesh_protocol.py b/backend/services/mesh/mesh_protocol.py new file mode 100644 index 00000000..4a1df8db --- /dev/null +++ b/backend/services/mesh/mesh_protocol.py @@ -0,0 +1,250 @@ +"""Mesh protocol helpers for canonical payloads and versioning.""" + +from __future__ import annotations + +from typing import Any +PROTOCOL_VERSION = "infonet/2" +NETWORK_ID = "sb-testnet-0" + + +def _safe_int(val, default=0): + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _normalize_number(value: Any) -> int | float: + try: + num = float(value) + except Exception: + return 0 + if num.is_integer(): + return int(num) + return num + + +def normalize_message_payload(payload: dict[str, Any]) -> dict[str, Any]: + normalized = { + "message": str(payload.get("message", "")), + "destination": str(payload.get("destination", "")), + "channel": str(payload.get("channel", "LongFast")), + "priority": str(payload.get("priority", "normal")), + "ephemeral": bool(payload.get("ephemeral", False)), + } + transport_lock = str(payload.get("transport_lock", "") or "").strip().lower() + if transport_lock: + normalized["transport_lock"] = transport_lock + return normalized + + +def normalize_gate_message_payload(payload: dict[str, Any]) -> dict[str, Any]: + normalized = { + "gate": str(payload.get("gate", "")).strip().lower(), + "ciphertext": str(payload.get("ciphertext", "")), + "nonce": str(payload.get("nonce", payload.get("iv", ""))), + "sender_ref": str(payload.get("sender_ref", "")), + "format": str(payload.get("format", "mls1") or "mls1").strip().lower(), + } + epoch = _safe_int(payload.get("epoch", 0), 0) + if epoch > 0: + normalized["epoch"] = epoch + return normalized + + +def normalize_vote_payload(payload: dict[str, Any]) -> dict[str, Any]: + vote_val = _safe_int(payload.get("vote", 0), 0) + return { + "target_id": str(payload.get("target_id", "")), + "vote": vote_val, + "gate": str(payload.get("gate", "")), + } + + +def normalize_gate_create_payload(payload: dict[str, Any]) -> dict[str, Any]: + rules = payload.get("rules", {}) + if not isinstance(rules, dict): + rules = {} + return { + "gate_id": str(payload.get("gate_id", "")).lower(), + "display_name": str(payload.get("display_name", ""))[:64], + "rules": rules, + } + + +def normalize_prediction_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "market_title": str(payload.get("market_title", "")), + "side": str(payload.get("side", "")), + "stake_amount": _normalize_number(payload.get("stake_amount", 0.0)), + } + + +def normalize_stake_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "message_id": str(payload.get("message_id", "")), + "poster_id": str(payload.get("poster_id", "")), + "side": str(payload.get("side", "")), + "amount": _normalize_number(payload.get("amount", 0.0)), + "duration_days": _safe_int(payload.get("duration_days", 0), 0), + } + + +def normalize_dm_key_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "dh_pub_key": str(payload.get("dh_pub_key", "")), + "dh_algo": str(payload.get("dh_algo", "")), + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + } + + +def normalize_dm_message_payload(payload: dict[str, Any]) -> dict[str, Any]: + normalized = { + "recipient_id": str(payload.get("recipient_id", "")), + "delivery_class": str(payload.get("delivery_class", "")).lower(), + "recipient_token": str(payload.get("recipient_token", "")), + "ciphertext": str(payload.get("ciphertext", "")), + "msg_id": str(payload.get("msg_id", "")), + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + "format": str(payload.get("format", "dm1") or "dm1").strip().lower(), + } + session_welcome = payload.get("session_welcome") + if session_welcome: + normalized["session_welcome"] = str(session_welcome) + sender_seal = str(payload.get("sender_seal", "") or "") + if sender_seal: + normalized["sender_seal"] = sender_seal + relay_salt = str(payload.get("relay_salt", "") or "").strip().lower() + if relay_salt: + normalized["relay_salt"] = relay_salt + return normalized + + +def normalize_dm_message_payload_legacy(payload: dict[str, Any]) -> dict[str, Any]: + return { + "recipient_id": str(payload.get("recipient_id", "")), + "delivery_class": str(payload.get("delivery_class", "")).lower(), + "recipient_token": str(payload.get("recipient_token", "")), + "ciphertext": str(payload.get("ciphertext", "")), + "msg_id": str(payload.get("msg_id", "")), + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + } + + +def normalize_mailbox_claims(payload: dict[str, Any]) -> list[dict[str, str]]: + claims = payload.get("mailbox_claims", []) + if not isinstance(claims, list): + return [] + normalized: list[dict[str, str]] = [] + for claim in claims: + if not isinstance(claim, dict): + continue + normalized.append( + { + "type": str(claim.get("type", "")).lower(), + "token": str(claim.get("token", "")), + } + ) + return normalized + + +def normalize_dm_poll_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "mailbox_claims": normalize_mailbox_claims(payload), + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + "nonce": str(payload.get("nonce", "")), + } + + +def normalize_dm_count_payload(payload: dict[str, Any]) -> dict[str, Any]: + return normalize_dm_poll_payload(payload) + + +def normalize_dm_block_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "blocked_id": str(payload.get("blocked_id", "")), + "action": str(payload.get("action", "block")).lower(), + } + + +def normalize_dm_key_witness_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "target_id": str(payload.get("target_id", "")), + "dh_pub_key": str(payload.get("dh_pub_key", "")), + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + } + + +def normalize_trust_vouch_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "target_id": str(payload.get("target_id", "")), + "note": str(payload.get("note", ""))[:140], + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + } + + +def normalize_key_rotate_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "old_node_id": str(payload.get("old_node_id", "")), + "old_public_key": str(payload.get("old_public_key", "")), + "old_public_key_algo": str(payload.get("old_public_key_algo", "")), + "new_public_key": str(payload.get("new_public_key", "")), + "new_public_key_algo": str(payload.get("new_public_key_algo", "")), + "timestamp": _safe_int(payload.get("timestamp", 0), 0), + "old_signature": str(payload.get("old_signature", "")), + } + + +def normalize_key_revoke_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "revoked_public_key": str(payload.get("revoked_public_key", "")), + "revoked_public_key_algo": str(payload.get("revoked_public_key_algo", "")), + "revoked_at": _safe_int(payload.get("revoked_at", 0), 0), + "grace_until": _safe_int(payload.get("grace_until", 0), 0), + "reason": str(payload.get("reason", ""))[:140], + } + + +def normalize_abuse_report_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "target_id": str(payload.get("target_id", "")), + "reason": str(payload.get("reason", ""))[:280], + "gate": str(payload.get("gate", "")), + "evidence": str(payload.get("evidence", ""))[:256], + } + + +def normalize_payload(event_type: str, payload: dict[str, Any]) -> dict[str, Any]: + if event_type == "message": + return normalize_message_payload(payload) + if event_type == "gate_message": + return normalize_gate_message_payload(payload) + if event_type == "vote": + return normalize_vote_payload(payload) + if event_type == "gate_create": + return normalize_gate_create_payload(payload) + if event_type == "prediction": + return normalize_prediction_payload(payload) + if event_type == "stake": + return normalize_stake_payload(payload) + if event_type == "dm_key": + return normalize_dm_key_payload(payload) + if event_type == "dm_message": + return normalize_dm_message_payload(payload) + if event_type == "dm_poll": + return normalize_dm_poll_payload(payload) + if event_type == "dm_count": + return normalize_dm_count_payload(payload) + if event_type == "dm_block": + return normalize_dm_block_payload(payload) + if event_type == "dm_key_witness": + return normalize_dm_key_witness_payload(payload) + if event_type == "trust_vouch": + return normalize_trust_vouch_payload(payload) + if event_type == "key_rotate": + return normalize_key_rotate_payload(payload) + if event_type == "key_revoke": + return normalize_key_revoke_payload(payload) + if event_type == "abuse_report": + return normalize_abuse_report_payload(payload) + return payload diff --git a/backend/services/mesh/mesh_reputation.py b/backend/services/mesh/mesh_reputation.py new file mode 100644 index 00000000..a0df1c73 --- /dev/null +++ b/backend/services/mesh/mesh_reputation.py @@ -0,0 +1,985 @@ +"""Mesh Reputation Ledger — decentralized node trust scoring with gates. + +Every node maintains a local reputation ledger. Votes are weighted by voter +reputation and tenure (anti-Sybil). Scores decay linearly over a 2-year window. + +Gates are reputation-scoped communities. Entry requires meeting rep thresholds. +Getting downvoted below the threshold bars you automatically — no moderator needed. + +Persistence: JSON files in backend/data/ (auto-saved on change, loaded on start). +""" + +import math +import time +import logging +import os +import threading +import atexit +import hmac +import hashlib +from pathlib import Path +from typing import Optional + +from services.mesh.mesh_privacy_logging import privacy_log_label +from services.mesh.mesh_secure_storage import ( + read_domain_json, + read_secure_json, + write_domain_json, +) + +logger = logging.getLogger("services.mesh_reputation") + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +LEDGER_FILE = DATA_DIR / "reputation_ledger.json" +GATES_FILE = DATA_DIR / "gates.json" +LEDGER_DOMAIN = "reputation" +GATES_DOMAIN = "gates" + +# ─── Constants ──────────────────────────────────────────────────────────── + +VOTE_DECAY_MONTHS = 24 # Votes decay over 24 months. Recent votes weigh heaviest. +VOTE_DECAY_DAYS = VOTE_DECAY_MONTHS * 30 # ~720 days total window +MIN_REP_TO_VOTE = 3 # Minimum reputation to cast any vote (only enforced after bootstrap) +BOOTSTRAP_THRESHOLD = 1000 # Rep-to-vote rule kicks in after this many nodes join +MIN_REP_TO_CREATE_GATE = 10 # Minimum overall rep to create a gate +GATE_RATIFICATION_REP = ( + 50 # Cumulative member rep needed for a gate to be ratified (after bootstrap) +) +ALLOW_DYNAMIC_GATES = False +_VOTE_STORAGE_SALT_CACHE: bytes | None = None +_VOTE_STORAGE_SALT_WARNING_EMITTED = False + +DEFAULT_PRIVATE_GATES: dict[str, dict] = { + "infonet": { + "display_name": "Main Infonet", + "description": "Private network operations floor. Core testnet traffic, protocol notes, and live coordination stay here.", + "welcome": "WELCOME TO MAIN INFONET. Treat this as the protocol floor, not a public lobby.", + "sort_order": 10, + }, + "general-talk": { + "display_name": "General Talk", + "description": "Lower-friction private lounge for day-to-day chatter, intros, and community pulse checks.", + "welcome": "WELCOME TO GENERAL TALK. Keep it human, but remember the lane is still private and reputation-backed.", + "sort_order": 20, + }, + "gathered-intel": { + "display_name": "Gathered Intel", + "description": "Drop sourced observations, OSINT fragments, and operator notes worth preserving for later review.", + "welcome": "WELCOME TO GATHERED INTEL. Bring sources, timestamps, and enough context for someone else to verify you.", + "sort_order": 30, + }, + "tracked-planes": { + "display_name": "Tracked Planes", + "description": "Aviation watchers, route anomalies, military traffic, and callout chatter for flights worth tracking.", + "welcome": "WELCOME TO TRACKED PLANES. Call out the flight, route, why it matters, and what pattern you think you see.", + "sort_order": 40, + }, + "ukraine-front": { + "display_name": "Ukraine Front", + "description": "Focused room for Ukraine war developments, map observations, and source cross-checking.", + "welcome": "WELCOME TO UKRAINE FRONT. Keep reporting tight, sourced, and separated from wishcasting.", + "sort_order": 50, + }, + "iran-front": { + "display_name": "Iran Front", + "description": "Iran flashpoint monitoring, regional spillover, and escalation watch from a private-lane perspective.", + "welcome": "WELCOME TO IRAN FRONT. Track escalation, proxies, logistics, and what changes the risk picture.", + "sort_order": 60, + }, + "world-news": { + "display_name": "World News", + "description": "Big-picture geopolitical developments, breaking stories, and broader context outside the narrow fronts.", + "welcome": "WELCOME TO WORLD NEWS. Use this room when the story matters but does not fit a narrower gate.", + "sort_order": 70, + }, + "prediction-markets": { + "display_name": "Prediction Markets", + "description": "Discuss market signals, event contracts, and whether crowd pricing is tracking reality or pure narrative.", + "welcome": "WELCOME TO PREDICTION MARKETS. Bring the market angle and the narrative angle, then compare them honestly.", + "sort_order": 80, + }, + "finance": { + "display_name": "Finance", + "description": "Macro moves, defense names, rates, liquidity stress, and the parts of finance that steer the rest of the board.", + "welcome": "WELCOME TO FINANCE. Macro, defense names, liquidity stress, and market structure all belong here.", + "sort_order": 90, + }, + "cryptography": { + "display_name": "Cryptography", + "description": "Protocol design, primitives, breakage reports, and the sharper math behind the network.", + "welcome": "WELCOME TO CRYPTOGRAPHY. If you think something can be broken, this is where you try to prove it.", + "sort_order": 100, + }, + "cryptocurrencies": { + "display_name": "Cryptocurrencies", + "description": "Chain activity, privacy coin chatter, market structure, and crypto-adjacent threat intel.", + "welcome": "WELCOME TO CRYPTOCURRENCIES. Chain behavior, privacy tooling, and market weirdness all go on the table.", + "sort_order": 110, + }, + "meet-chat": { + "display_name": "Meet Chat", + "description": "Casual private hangout for getting to know the other operators behind the personas.", + "welcome": "WELCOME TO MEET CHAT. Lighten up a little and let the community feel like it has actual people in it.", + "sort_order": 120, + }, + "opsec-lab": { + "display_name": "OPSEC Lab", + "description": "Stress-test assumptions, try to break rep or persona boundaries, and document privacy failures without mercy.", + "welcome": "WELCOME TO OPSEC LAB. Be ruthless, document the leak, and assume everyone is smarter than the last audit.", + "sort_order": 130, + }, +} + + +def _blind_voter(voter_id: str, salt: bytes) -> str: + if not voter_id: + return "" + digest = hmac.new(salt, voter_id.encode("utf-8"), hashlib.sha256).hexdigest() + return f"{digest[:8]}…" + + +def _vote_storage_salt() -> bytes: + global _VOTE_STORAGE_SALT_CACHE, _VOTE_STORAGE_SALT_WARNING_EMITTED + if _VOTE_STORAGE_SALT_CACHE is not None: + return _VOTE_STORAGE_SALT_CACHE + try: + from services.config import get_settings + + secret = str(get_settings().MESH_PEER_PUSH_SECRET or "").strip() + except Exception: + secret = "" + if not secret and not _VOTE_STORAGE_SALT_WARNING_EMITTED: + logger.warning("MESH_PEER_PUSH_SECRET missing; falling back to local voter blinding salt") + _VOTE_STORAGE_SALT_WARNING_EMITTED = True + if secret: + _VOTE_STORAGE_SALT_CACHE = hmac.new( + secret.encode("utf-8"), + b"shadowbroker|reputation|voter-blind|v1", + hashlib.sha256, + ).digest() + else: + # Persist a stable salt to disk so blinded voter IDs survive restarts. + # Without this, duplicate-vote detection breaks on every restart + # because the blinded ID changes with a new random salt. + salt_path = DATA_DIR / "voter_blind_salt.bin" + try: + if salt_path.exists() and salt_path.stat().st_size == 32: + _VOTE_STORAGE_SALT_CACHE = salt_path.read_bytes() + else: + DATA_DIR.mkdir(parents=True, exist_ok=True) + new_salt = os.urandom(32) + salt_path.write_bytes(new_salt) + _VOTE_STORAGE_SALT_CACHE = new_salt + logger.info("Generated new persistent voter blinding salt") + except Exception as e: + logger.error(f"Failed to persist voter salt, falling back to random: {e}") + _VOTE_STORAGE_SALT_CACHE = os.urandom(32) + return _VOTE_STORAGE_SALT_CACHE + + +def _stored_voter_id(vote: dict) -> str: + blinded = str(vote.get("blinded_voter_id", "") or "").strip() + if blinded: + return blinded + raw = str(vote.get("voter_id", "") or "").strip() + if not raw: + return "" + return _blind_voter(raw, _vote_storage_salt()) + + +def _serialize_vote_record(vote: dict) -> dict: + blinded = _stored_voter_id(vote) + payload = dict(vote or {}) + payload.pop("voter_id", None) + if blinded: + payload["blinded_voter_id"] = blinded + return payload + + +# ─── Vote Record ────────────────────────────────────────────────────────── + + +class ReputationLedger: + """Local reputation ledger — each node maintains its own view. + + Storage format: + nodes: {node_id: {first_seen, public_key, agent}} + votes: [{voter_id, target_id, vote (+1/-1), gate (optional), timestamp}] + scores_cache: {node_id: {overall: int, gates: {gate_id: int}}} + """ + + def __init__(self): + self.nodes: dict[str, dict] = {} # {node_id: {first_seen, public_key, agent}} + self.votes: list[dict] = [] # [{voter_id, target_id, vote, gate, timestamp}] + self.vouches: list[dict] = [] # [{voucher_id, target_id, note, timestamp}] + self.aliases: dict[str, str] = {} # {new_node_id: old_node_id} + self._scores_dirty = True + self._scores_cache: dict[str, dict] = {} + self._dirty = False + self._save_lock = threading.Lock() + self._save_timer: threading.Timer | None = None + self._SAVE_INTERVAL = 5.0 + atexit.register(self._flush) + self._load() + + # ─── Persistence ────────────────────────────────────────────────── + + def _load(self): + """Load ledger from disk.""" + domain_path = DATA_DIR / LEDGER_DOMAIN / LEDGER_FILE.name + if not domain_path.exists() and LEDGER_FILE.exists(): + try: + legacy = read_secure_json( + LEDGER_FILE, + lambda: {"nodes": {}, "votes": [], "vouches": [], "aliases": {}}, + ) + write_domain_json(LEDGER_DOMAIN, LEDGER_FILE.name, legacy) + LEDGER_FILE.unlink(missing_ok=True) + except Exception as e: + logger.error(f"Failed to migrate reputation ledger: {e}") + try: + data = read_domain_json( + LEDGER_DOMAIN, + LEDGER_FILE.name, + lambda: {"nodes": {}, "votes": [], "vouches": [], "aliases": {}}, + ) + self.nodes = data.get("nodes", {}) + raw_votes = data.get("votes", []) + # Purge legacy __system__ cost votes — they stored raw voter + # identities as target_id, which is a privacy leak. + before = len(raw_votes) + self.votes = [v for v in raw_votes if not v.get("system_cost")] + purged = before - len(self.votes) + if purged: + self._dirty = True # re-save without the leaked records + logger.info(f"Purged {purged} legacy system_cost vote(s) with raw identity leak") + self.vouches = data.get("vouches", []) + self.aliases = data.get("aliases", {}) + self._scores_dirty = True + logger.info( + f"Loaded reputation ledger: {len(self.nodes)} nodes, {len(self.votes)} votes" + ) + except Exception as e: + logger.error(f"Failed to load reputation ledger: {e}") + + def _save(self): + """Mark dirty and schedule a coalesced disk write.""" + self._dirty = True + with self._save_lock: + if self._save_timer is None or not self._save_timer.is_alive(): + self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush) + self._save_timer.daemon = True + self._save_timer.start() + + def _flush(self): + """Actually write to disk (called by timer or atexit).""" + if not self._dirty: + return + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + data = { + "nodes": self.nodes, + "votes": [_serialize_vote_record(vote) for vote in self.votes], + "vouches": self.vouches, + "aliases": self.aliases, + } + write_domain_json(LEDGER_DOMAIN, LEDGER_FILE.name, data) + LEDGER_FILE.unlink(missing_ok=True) + self._dirty = False + except Exception as e: + logger.error(f"Failed to save reputation ledger: {e}") + + # ─── Node Registration ──────────────────────────────────────────── + + def register_node( + self, node_id: str, public_key: str = "", public_key_algo: str = "", agent: bool = False + ): + """Register a node if not already known. Updates public_key if provided.""" + if node_id not in self.nodes: + self.nodes[node_id] = { + "first_seen": time.time(), + "public_key": public_key, + "public_key_algo": public_key_algo, + "agent": agent, + } + self._save() + logger.info( + "Registered new node: %s", + privacy_log_label(node_id, label="node"), + ) + elif public_key and not self.nodes[node_id].get("public_key"): + self.nodes[node_id]["public_key"] = public_key + if public_key_algo: + self.nodes[node_id]["public_key_algo"] = public_key_algo + self._save() + + def link_identities(self, old_id: str, new_id: str) -> tuple[bool, str]: + """Link a new node_id to an old one for reputation continuity.""" + if not old_id or not new_id: + return False, "Missing old_id or new_id" + if old_id == new_id: + return False, "Old and new IDs must differ" + if new_id in self.aliases: + return False, f"{new_id} is already linked" + if old_id in self.aliases: + return False, f"{old_id} is already linked to {self.aliases[old_id]}" + if old_id in self.aliases.values(): + return False, f"{old_id} is already the source of a link" + + self.aliases[new_id] = old_id + self._scores_dirty = True + self._save() + logger.info( + "Linked identity: %s -> %s", + privacy_log_label(old_id, label="node"), + privacy_log_label(new_id, label="node"), + ) + return True, "linked" + + def get_node_age_days(self, node_id: str) -> float: + """Get node age in days.""" + node = self.nodes.get(node_id) + if not node: + return 0 + return (time.time() - node.get("first_seen", time.time())) / 86400 + + def is_agent(self, node_id: str) -> bool: + """Check if node is registered as an Agent (bot/AI).""" + return self.nodes.get(node_id, {}).get("agent", False) + + # ─── Voting ─────────────────────────────────────────────────────── + + def _compute_vote_weight(self, voter_id: str) -> float: + """Rep-weighted voting — your vote's power scales with reputation and tenure. + + Two factors combine: + rep_factor: log10(1 + |rep|) / log10(101) → 0 rep ≈ 0.0, 100 rep = 1.0 + tenure_factor: age_days / 720 → new = 0.0, 2yr+ = 1.0 + + Combined weight = max(0.1, rep_factor × tenure_factor), capped at 1.0. + Floor of 0.1 ensures every user's vote still counts for something. + + | Rep | Day 1 | 6 months | 1 year | 2 years | + |------|-------|----------|--------|---------| + | 0 | 0.10 | 0.10 | 0.10 | 0.10 | + | 5 | 0.10 | 0.10 | 0.20 | 0.39 | + | 25 | 0.10 | 0.18 | 0.35 | 0.70 | + | 50 | 0.10 | 0.21 | 0.42 | 0.85 | + | 100+ | 0.10 | 0.25 | 0.50 | 1.00 | + """ + # ── Rep factor: logarithmic scale, 100 rep = full power ── + rep = self.get_reputation(voter_id).get("overall", 0) + rep_factor = math.log10(1 + abs(rep)) / math.log10(101) # 0→0.0, 100→1.0 + + # ── Tenure factor: linear over the 2-year decay window ── + age_days = self.get_node_age_days(voter_id) + decay_window = float(VOTE_DECAY_DAYS) if VOTE_DECAY_DAYS > 0 else 720.0 + tenure_factor = min(1.0, age_days / decay_window) + + weight = rep_factor * tenure_factor + return max(0.1, min(1.0, weight)) + + def cast_vote( + self, voter_id: str, target_id: str, vote: int, gate: str = "" + ) -> tuple[bool, str, float]: + """Cast a vote. Returns (success, reason, weight). + + Rules: + - Self-votes allowed (costs -1 rep like any vote — net negative for voter) + - Must have rep >= MIN_REP_TO_VOTE (except first few bootstrap votes) + - One vote per voter per target per gate (can change direction) + - Vote value is weighted by voter's reputation and tenure + """ + if vote not in (1, -1): + return False, "Vote must be +1 or -1", 0.0 + + # Reputation burn: minimum rep to vote — only enforced after network bootstraps + network_size = len(self.nodes) + if network_size >= BOOTSTRAP_THRESHOLD: + voter_rep = self.get_reputation(voter_id).get("overall", 0) + if voter_rep < MIN_REP_TO_VOTE: + return ( + False, + f"Need {MIN_REP_TO_VOTE} reputation to vote (you have {voter_rep}). Network has {network_size} nodes — rep-to-vote is active.", + 0.0, + ) + + blinded_voter_id = _blind_voter(voter_id, _vote_storage_salt()) + existing_vote = next( + ( + v + for v in self.votes + if _stored_voter_id(v) == blinded_voter_id + and v["target_id"] == target_id + and v.get("gate", "") == gate + ), + None, + ) + existing_vote_value = None + if existing_vote is not None: + try: + existing_vote_value = int(existing_vote.get("vote", 0)) + except (TypeError, ValueError): + existing_vote_value = None + if existing_vote and existing_vote_value == vote: + direction = "up" if vote == 1 else "down" + gate_str = f" in gate '{gate}'" if gate else "" + return False, f"Vote already set to {direction} on {target_id}{gate_str}", 0.0 + + # Remove existing vote from this voter for this target in this gate + self.votes = [ + v + for v in self.votes + if not ( + _stored_voter_id(v) == blinded_voter_id + and v["target_id"] == target_id + and v.get("gate", "") == gate + ) + ] + + # Record the new vote + now = time.time() + weight = self._compute_vote_weight(voter_id) + is_direction_change = existing_vote is not None + self.votes.append( + { + "voter_id": voter_id, + "target_id": target_id, + "vote": vote, + "gate": gate, + "timestamp": now, + "weight": weight, + "agent_verify": self.is_agent(voter_id), + } + ) + + # Vote cost: costs the same as the vote's weight (if your vote only + # moves the score by 0.1, you only pay 0.1 — not a flat 1.0). + # Only on first vote, not direction changes. The target_id is the + # *blinded* voter ID so the root identity never touches disk. + if not is_direction_change: + self.votes.append( + { + "voter_id": "__system__", + "target_id": blinded_voter_id, + "vote": -1, + "gate": "", + "timestamp": now, + "weight": weight, + "agent_verify": False, + "vote_cost": True, + } + ) + + self._scores_dirty = True + self._save() + + direction = "up" if vote == 1 else "down" + gate_str = f" in gate '{gate}'" if gate else "" + logger.info( + "Vote: %s voted %s on %s%s", + privacy_log_label(voter_id, label="node"), + direction, + privacy_log_label(target_id, label="node"), + f" in {privacy_log_label(gate, label='gate')}" if gate else "", + ) + return True, f"Voted {direction} on {target_id}{gate_str}", weight + + # ─── Trust Vouches ──────────────────────────────────────────────── + + def add_vouch( + self, voucher_id: str, target_id: str, note: str = "", timestamp: float | None = None + ) -> tuple[bool, str]: + if not voucher_id or not target_id: + return False, "Missing voucher_id or target_id" + if voucher_id == target_id: + return False, "Cannot vouch for yourself" + ts = timestamp if timestamp is not None else time.time() + # Deduplicate vouches from same voucher to same target within 30 days + cutoff = ts - (30 * 86400) + for v in self.vouches: + if ( + v.get("voucher_id") == voucher_id + and v.get("target_id") == target_id + and float(v.get("timestamp", 0)) >= cutoff + ): + return False, "Duplicate vouch" + self.vouches.append( + { + "voucher_id": voucher_id, + "target_id": target_id, + "note": str(note)[:140], + "timestamp": float(ts), + } + ) + self._save() + return True, "vouched" + + def get_vouches(self, target_id: str, limit: int = 50) -> list[dict]: + if not target_id: + return [] + entries = [v for v in self.vouches if v.get("target_id") == target_id] + entries = sorted(entries, key=lambda v: v.get("timestamp", 0), reverse=True) + return entries[: max(1, limit)] + + # ─── Score Computation ──────────────────────────────────────────── + + def _recompute_scores(self): + """Recompute all scores with time-weighted decay. + + Votes decay linearly over VOTE_DECAY_MONTHS (24 months). + A vote cast today has full weight (1.0). A vote cast 12 months ago + has ~0.5 weight. A vote older than 24 months has 0 weight and is + skipped entirely. This means recent activity always matters more + than ancient history, but nothing ever fully disappears until 2 years + have passed. + """ + if not self._scores_dirty: + return + + now = time.time() + decay_seconds = VOTE_DECAY_DAYS * 86400 if VOTE_DECAY_DAYS > 0 else 0 + scores: dict[str, dict] = {} + + for v in self.votes: + age = now - v["timestamp"] + + # If decay is enabled, skip votes older than the window + if decay_seconds and age >= decay_seconds: + continue + + # Time-decay multiplier: 1.0 for brand new, 0.0 at the boundary. + # Recent months weigh heaviest. + if decay_seconds: + decay_factor = 1.0 - (age / decay_seconds) + else: + decay_factor = 1.0 + + target = v["target_id"] + if target not in scores: + scores[target] = {"overall": 0.0, "gates": {}, "upvotes": 0, "downvotes": 0} + + weighted = v["vote"] * v.get("weight", 1.0) * decay_factor + scores[target]["overall"] += weighted + + if v["vote"] > 0: + scores[target]["upvotes"] += 1 + else: + scores[target]["downvotes"] += 1 + + gate = v.get("gate", "") + if gate: + scores[target]["gates"].setdefault(gate, 0.0) + scores[target]["gates"][gate] += weighted + + # Round to 1 decimal place — weighted votes produce fractional scores + for nid in scores: + scores[nid]["overall"] = round(scores[nid]["overall"], 1) + for gid in scores[nid]["gates"]: + scores[nid]["gates"][gid] = round(scores[nid]["gates"][gid], 1) + + self._scores_cache = scores + self._scores_dirty = False + + def get_reputation(self, node_id: str) -> dict: + """Get reputation for a single node. + + Returns {overall: int, gates: {gate_id: int}, upvotes: int, downvotes: int} + + Scores are merged from three possible sources: + 1. Direct scores on ``node_id`` (votes targeting posts by this identity). + 2. Alias scores (if ``node_id`` is linked to an older identity). + 3. Blinded-wallet scores — vote costs are stored under the deterministic + HMAC-blinded form of the voter's root identity so the raw private key + never touches disk. When the caller supplies the raw node_id we can + recompute the blind and merge those costs in. + """ + self._recompute_scores() + _zero = lambda: {"overall": 0, "gates": {}, "upvotes": 0, "downvotes": 0} + base = self._scores_cache.get(node_id, _zero()) + + # Merge alias (old identity linked to this one) + alias = self.aliases.get(node_id) + if alias: + old = self._scores_cache.get(alias, _zero()) + base = self._merge_scores(base, old) + + # Merge blinded-wallet costs (vote-cost records target the blinded ID) + blinded = _blind_voter(node_id, _vote_storage_salt()) + if blinded and blinded != node_id: + wallet = self._scores_cache.get(blinded, _zero()) + if wallet["overall"] != 0 or wallet["upvotes"] != 0 or wallet["downvotes"] != 0: + base = self._merge_scores(base, wallet) + + return base + + @staticmethod + def _merge_scores(a: dict, b: dict) -> dict: + merged = { + "overall": a["overall"] + b["overall"], + "gates": {}, + "upvotes": a["upvotes"] + b["upvotes"], + "downvotes": a["downvotes"] + b["downvotes"], + } + gates = set(a.get("gates", {}).keys()) | set(b.get("gates", {}).keys()) + for g in gates: + merged["gates"][g] = a.get("gates", {}).get(g, 0) + b.get("gates", {}).get(g, 0) + return merged + + def get_all_reputations(self) -> dict[str, int]: + """Get overall reputation for all known nodes.""" + self._recompute_scores() + return {nid: s["overall"] for nid, s in self._scores_cache.items()} + + def get_reputation_log(self, node_id: str, *, detailed: bool = False) -> dict: + """Return reputation data for a node. + + Public callers receive a summary-only view. Rich breakdowns remain + available to authenticated audit tooling. + """ + cutoff = time.time() - (VOTE_DECAY_DAYS * 86400) + rep = self.get_reputation(node_id) + result = { + "node_id": node_id, + "overall": rep.get("overall", 0), + "upvotes": rep.get("upvotes", 0), + "downvotes": rep.get("downvotes", 0), + } + if not detailed: + return result + + alias = self.aliases.get(node_id) + target_ids = {node_id} + if alias: + target_ids.add(alias) + query_salt = os.urandom(8) + recent = [ + { + "voter": _blind_voter(_stored_voter_id(v), query_salt), + "vote": v["vote"], + "gate": "", + "weight": v.get("weight", 1.0), + "agent_verify": v.get("agent_verify", False), + "age": f"{int((time.time() - v['timestamp']) / 86400)}d ago", + } + for v in sorted(self.votes, key=lambda x: x["timestamp"], reverse=True) + if v["target_id"] in target_ids and v["timestamp"] >= cutoff + ][:20] + + result.update( + { + "gates": {}, + "recent_votes": recent, + "node_age_days": round(self.get_node_age_days(node_id), 1), + "is_agent": self.is_agent(node_id), + } + ) + return result + + # ─── DM Threshold ──────────────────────────────────────────────── + + def should_accept_message(self, sender_id: str, recipient_threshold: int) -> bool: + """Check if sender meets recipient's reputation threshold for DMs.""" + if recipient_threshold <= 0: + return True + sender_rep = self.get_reputation(sender_id).get("overall", 0) + return sender_rep >= recipient_threshold + + # ─── Cleanup ────────────────────────────────────────────────────── + + def cleanup_expired(self): + """Remove votes older than the 2-year decay window.""" + if VOTE_DECAY_DAYS <= 0: + return + cutoff = time.time() - (VOTE_DECAY_DAYS * 86400) + before = len(self.votes) + self.votes = [v for v in self.votes if v["timestamp"] >= cutoff] + after = len(self.votes) + if before != after: + self._scores_dirty = True + self._save() + logger.info(f"Cleaned up {before - after} expired votes") + + +# ─── Gate System ────────────────────────────────────────────────────────── + + +class GateManager: + """Self-governing reputation-gated communities. + + Anyone with rep >= 10 can create a gate. Entry requires meeting rep thresholds. + Getting downvoted below threshold bars you automatically. + """ + + def __init__(self, ledger: ReputationLedger): + self.ledger = ledger + self.gates: dict[str, dict] = {} + self._dirty = False + self._save_lock = threading.Lock() + self._save_timer: threading.Timer | None = None + self._SAVE_INTERVAL = 5.0 + atexit.register(self._flush) + self._load() + + def _load(self): + domain_path = DATA_DIR / GATES_DOMAIN / GATES_FILE.name + if not domain_path.exists() and GATES_FILE.exists(): + try: + legacy = read_secure_json(GATES_FILE, lambda: {}) + write_domain_json(GATES_DOMAIN, GATES_FILE.name, legacy) + GATES_FILE.unlink(missing_ok=True) + except Exception as e: + logger.error(f"Failed to migrate gates: {e}") + try: + self.gates = read_domain_json(GATES_DOMAIN, GATES_FILE.name, lambda: {}) + logger.info(f"Loaded {len(self.gates)} gates") + except Exception as e: + logger.error(f"Failed to load gates: {e}") + if self._apply_gate_catalog(): + self._save() + + def _apply_gate_catalog(self) -> bool: + """Seed fixed private launch gates and retire obsolete defaults.""" + changed = False + legacy_public_square = self.gates.get("public-square") + if isinstance(legacy_public_square, dict) and not legacy_public_square.get("fixed"): + self.gates.pop("public-square", None) + changed = True + + for gate_id, seed in DEFAULT_PRIVATE_GATES.items(): + gate = self.gates.get(gate_id) + if not isinstance(gate, dict): + self.gates[gate_id] = { + "creator_node_id": "!sb_seed", + "display_name": seed["display_name"], + "description": seed["description"], + "welcome": seed["welcome"], + "rules": { + "min_overall_rep": 0, + "min_gate_rep": {}, + }, + "created_at": time.time(), + "message_count": 0, + "fixed": True, + "sort_order": seed["sort_order"], + } + changed = True + continue + + for key in ("display_name", "description", "welcome", "sort_order"): + if gate.get(key) != seed[key]: + gate[key] = seed[key] + changed = True + if gate.get("fixed") is not True: + gate["fixed"] = True + changed = True + if "rules" not in gate or not isinstance(gate["rules"], dict): + gate["rules"] = {"min_overall_rep": 0, "min_gate_rep": {}} + changed = True + gate["rules"].setdefault("min_overall_rep", 0) + gate["rules"].setdefault("min_gate_rep", {}) + gate.setdefault("message_count", 0) + gate.setdefault("created_at", time.time()) + + return changed + + def _save(self): + """Mark dirty and schedule a coalesced disk write.""" + self._dirty = True + with self._save_lock: + if self._save_timer is None or not self._save_timer.is_alive(): + self._save_timer = threading.Timer(self._SAVE_INTERVAL, self._flush) + self._save_timer.daemon = True + self._save_timer.start() + + def _flush(self): + """Actually write to disk (called by timer or atexit).""" + if not self._dirty: + return + try: + DATA_DIR.mkdir(parents=True, exist_ok=True) + write_domain_json(GATES_DOMAIN, GATES_FILE.name, self.gates) + GATES_FILE.unlink(missing_ok=True) + self._dirty = False + except Exception as e: + logger.error(f"Failed to save gates: {e}") + + def create_gate( + self, + creator_id: str, + gate_id: str, + display_name: str, + min_overall_rep: int = 0, + min_gate_rep: Optional[dict] = None, + description: str = "", + ) -> tuple[bool, str]: + """Create a new gate. Rep gate disabled until network reaches critical mass.""" + + if not ALLOW_DYNAMIC_GATES: + return False, "Gate creation is disabled for the fixed private launch catalog" + + gate_id = gate_id.lower().strip() + if not gate_id or not gate_id.isalnum() and "-" not in gate_id: + return False, "Gate ID must be alphanumeric (hyphens allowed)" + if len(gate_id) > 32: + return False, "Gate ID too long (max 32 chars)" + if gate_id in self.gates: + return False, f"Gate '{gate_id}' already exists" + + self.gates[gate_id] = { + "creator_node_id": creator_id, + "display_name": display_name[:64], + "description": description[:240], + "rules": { + "min_overall_rep": min_overall_rep, + "min_gate_rep": min_gate_rep or {}, + }, + "created_at": time.time(), + "message_count": 0, + "fixed": False, + "sort_order": 1000, + } + self._save() + logger.info( + "Gate created: %s by %s", + privacy_log_label(gate_id, label="gate"), + privacy_log_label(creator_id, label="node"), + ) + return True, f"Gate '{gate_id}' created" + + def can_enter(self, node_id: str, gate_id: str) -> tuple[bool, str]: + """Check if a node meets the entry rules for a gate.""" + gate = self.gates.get(gate_id) + if not gate: + return False, f"Gate '{gate_id}' does not exist" + + rules = gate.get("rules", {}) + rep = self.ledger.get_reputation(node_id) + + # Check overall rep requirement + min_overall = rules.get("min_overall_rep", 0) + if rep.get("overall", 0) < min_overall: + return False, f"Need {min_overall} overall rep (you have {rep.get('overall', 0)})" + + # Check gate-specific rep requirements + for req_gate, req_min in rules.get("min_gate_rep", {}).items(): + gate_rep = rep.get("gates", {}).get(req_gate, 0) + if gate_rep < req_min: + return False, f"Need {req_min} rep in '{req_gate}' gate (you have {gate_rep})" + + return True, "Access granted" + + def list_gates(self) -> list[dict]: + """List all gates with metadata.""" + result = [] + for gid, gate in self.gates.items(): + result.append( + { + "gate_id": gid, + "display_name": gate.get("display_name", gid), + "description": gate.get("description", ""), + "welcome": gate.get("welcome", ""), + "rules": gate.get("rules", {}), + "created_at": gate.get("created_at", 0), + "fixed": bool(gate.get("fixed", False)), + "sort_order": int(gate.get("sort_order", 1000) or 1000), + } + ) + return sorted( + result, + key=lambda x: ( + 0 if x.get("fixed") else 1, + int(x.get("sort_order", 1000) or 1000), + -float(x.get("created_at", 0) or 0), + x.get("gate_id", ""), + ), + ) + + def get_gate(self, gate_id: str) -> Optional[dict]: + """Get gate details.""" + gate = self.gates.get(gate_id) + if not gate: + return None + public_gate = { + key: value + for key, value in gate.items() + if key not in {"creator_node_id", "message_count"} + } + return { + "gate_id": gate_id, + **public_gate, + } + + def record_message(self, gate_id: str): + """Increment message count for a gate.""" + if gate_id in self.gates: + self.gates[gate_id]["message_count"] = self.gates[gate_id].get("message_count", 0) + 1 + self._save() + + def is_ratified(self, gate_id: str) -> bool: + """Check if a gate is ratified (has permanent chain address). + + Before BOOTSTRAP_THRESHOLD nodes: all gates are ratified (early access). + After bootstrap: gates need cumulative member rep >= GATE_RATIFICATION_REP. + """ + if len(self.ledger.nodes) < BOOTSTRAP_THRESHOLD: + return True # Pre-bootstrap: all gates are ratified + + gate = self.gates.get(gate_id) + if not gate: + return False + + # Sum rep of all nodes that have gate-specific rep in this gate + all_reps = self.ledger.get_all_reputations() + self.ledger._recompute_scores() + cumulative = 0 + for nid, score_data in self.ledger._scores_cache.items(): + gate_rep = score_data.get("gates", {}).get(gate_id, 0) + if gate_rep > 0: + cumulative += gate_rep + + return cumulative >= GATE_RATIFICATION_REP + + def get_ratification_status(self, gate_id: str) -> dict: + """Get gate's ratification progress.""" + gate = self.gates.get(gate_id) + if not gate: + return {"ratified": False, "reason": "Gate not found"} + + network_size = len(self.ledger.nodes) + if network_size < BOOTSTRAP_THRESHOLD: + return { + "ratified": True, + "reason": f"Pre-bootstrap ({network_size}/{BOOTSTRAP_THRESHOLD} nodes) — all gates ratified", + "cumulative_rep": 0, + "required_rep": GATE_RATIFICATION_REP, + } + + # Compute cumulative gate rep + self.ledger._recompute_scores() + cumulative = 0 + contributors = 0 + for nid, score_data in self.ledger._scores_cache.items(): + gate_rep = score_data.get("gates", {}).get(gate_id, 0) + if gate_rep > 0: + cumulative += gate_rep + contributors += 1 + + ratified = cumulative >= GATE_RATIFICATION_REP + return { + "ratified": ratified, + "cumulative_rep": cumulative, + "required_rep": GATE_RATIFICATION_REP, + "contributors": contributors, + "reason": ( + "Ratified — permanent chain address" + if ratified + else f"Need {GATE_RATIFICATION_REP - cumulative} more cumulative rep" + ), + } + + +# ─── Module-level singletons ───────────────────────────────────────────── + +reputation_ledger = ReputationLedger() +gate_manager = GateManager(reputation_ledger) diff --git a/backend/services/mesh/mesh_rns.py b/backend/services/mesh/mesh_rns.py new file mode 100644 index 00000000..4af8841d --- /dev/null +++ b/backend/services/mesh/mesh_rns.py @@ -0,0 +1,1622 @@ +"""Reticulum (RNS) bridge for Infonet event propagation. + +Backend-hosted: the API server runs a Reticulum node and gossips signed events. +This module is optional and safely no-ops if Reticulum isn't installed. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import logging +import math +import os +import threading +import time +import uuid +from dataclasses import dataclass +from typing import Any + +from services.config import get_settings +from services.mesh.mesh_ibf import IBLT, build_iblt, minhash_sketch, minhash_similarity +from services.wormhole_settings import read_wormhole_settings + +logger = logging.getLogger("services.mesh_rns") + + +def _safe_int(val, default=0) -> int: + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _blind_mailbox_key(mailbox_key: str | bytes | None) -> str: + if isinstance(mailbox_key, bytes): + key_bytes = mailbox_key + else: + key_bytes = str(mailbox_key or "").encode("utf-8") + if not key_bytes: + return "" + return hmac.new(key_bytes, b"rns-mailbox-blind-v1", hashlib.sha256).hexdigest() + + +@dataclass +class RNSMessage: + msg_type: str + body: dict[str, Any] + meta: dict[str, Any] + + def encode(self) -> bytes: + return json.dumps( + {"type": self.msg_type, "body": self.body, "meta": self.meta}, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + +class RNSBridge: + def __init__(self) -> None: + self._enabled = False + self._ready = False + self._lock = threading.Lock() + self._dedupe: dict[str, float] = {} + self._last_churn = 0.0 + self._active_peers: list[str] = [] + self._reticulum = None + self._identity = None + self._destination = None + self._destinations_extra: list[dict[str, Any]] = [] + self._destination_created = 0.0 + self._last_identity_rotate = 0.0 + self._last_ibf_sync = 0.0 + self._ibf_thread: threading.Thread | None = None + self._peer_stats: dict[str, dict[str, float]] = {} + self._peer_lock = threading.Lock() + self._shard_cache: dict[str, dict[str, Any]] = {} + self._shard_lock = threading.Lock() + self._privacy_cache: dict[str, Any] = {"value": "default", "ts": 0.0} + self._batch_lock = threading.Lock() + self._batch_queue: list[dict] = [] + self._batch_timer: threading.Timer | None = None + self._cover_thread: threading.Thread | None = None + self._pending_sync: dict[str, dict[str, Any]] = {} + self._sync_lock = threading.Lock() + self._ibf_lock = threading.Lock() + self._ibf_fail_count = 0 + self._ibf_cooldown_until = 0.0 + self._dedupe_lock = threading.Lock() + self._dm_lock = threading.Lock() + self._dm_mailboxes: dict[str, list[dict[str, Any]]] = {} + + def enabled(self) -> bool: + return self._enabled and self._ready + + def status(self) -> dict: + settings = get_settings() + dest_age = 0 + if self._destination_created: + dest_age = max(0, int(time.time() - self._destination_created)) + cooldown_remaining = 0 + if self._ibf_cooldown_until: + cooldown_remaining = max(0, int(self._ibf_cooldown_until - time.time())) + return { + "enabled": bool(self._enabled), + "ready": bool(self._ready), + "local_hash": self._local_hash(), + "configured_peers": len(self._parse_peers()), + "active_peers": len(self._active_peers), + "dandelion_hops": settings.MESH_RNS_DANDELION_HOPS, + "ibf_interval_s": settings.MESH_RNS_IBF_INTERVAL_S, + "ibf_cooldown_s": cooldown_remaining, + "session_rotate_s": self._session_rotate_interval(), + "destination_age_s": dest_age, + "session_identities": len(self._destinations_extra) + (1 if self._destination else 0), + "private_dm_direct_ready": bool(self.enabled() and (self._active_peers or self._parse_peers())), + } + + def start(self) -> None: + settings = get_settings() + if not settings.MESH_RNS_ENABLED: + return + try: + import RNS # type: ignore + except Exception as exc: + logger.warning(f"RNS disabled: Reticulum import failed ({exc})") + return + + with self._lock: + if self._ready: + return + try: + self._reticulum = RNS.Reticulum() + identity = None + if settings.MESH_RNS_IDENTITY_PATH: + try: + identity = RNS.Identity.from_file(settings.MESH_RNS_IDENTITY_PATH) + except Exception as exc: + logger.warning(f"RNS identity load failed: {exc}") + if identity is None: + identity = RNS.Identity() + if settings.MESH_RNS_IDENTITY_PATH: + try: + identity.to_file(settings.MESH_RNS_IDENTITY_PATH) + except Exception as exc: + logger.warning(f"RNS identity save failed: {exc}") + + self._identity = identity + self._destination = self._create_destination(identity) + now = time.time() + self._destination_created = now + self._last_identity_rotate = now + self._destinations_extra = [] + + self._enabled = True + self._ready = True + logger.info("RNS bridge started") + + if settings.MESH_RNS_IBF_INTERVAL_S > 0: + self._ibf_thread = threading.Thread(target=self._ibf_sync_loop, daemon=True) + self._ibf_thread.start() + if self._cover_thread is None: + self._cover_thread = threading.Thread(target=self._cover_loop, daemon=True) + self._cover_thread.start() + except Exception as exc: + logger.warning(f"RNS disabled: init failed ({exc})") + self._enabled = False + self._ready = False + + def _prune_dedupe(self) -> None: + cutoff = time.time() - 300 + with self._dedupe_lock: + for key, ts in list(self._dedupe.items()): + if ts < cutoff: + del self._dedupe[key] + + def _session_rotate_interval(self) -> int: + settings = get_settings() + interval = int(settings.MESH_RNS_SESSION_ROTATE_S or 0) + if self._is_high_privacy() and interval <= 0: + interval = 600 + return max(0, int(interval)) + + def _rotation_enabled(self) -> bool: + settings = get_settings() + if settings.MESH_RNS_IDENTITY_PATH: + return False + return self._session_rotate_interval() > 0 + + def _create_destination(self, identity: Any) -> Any: + settings = get_settings() + import RNS # type: ignore + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + settings.MESH_RNS_APP_NAME, + settings.MESH_RNS_ASPECT, + ) + callback = getattr(destination, "set_packet_callback", None) + if callable(callback): + callback(self._on_packet) + else: + logger.warning("RNS destination has no packet callback; inbound disabled") + return destination + + def _prune_rotated_destinations(self, interval: int | None = None) -> None: + if not self._destinations_extra: + return + settings = get_settings() + interval = self._session_rotate_interval() if interval is None else int(interval) + grace = max( + 120, + int(interval) * 2, + int(settings.MESH_RNS_IBF_QUORUM_TIMEOUT_S or 0) * 2, + ) + cutoff = time.time() - grace + self._destinations_extra = [ + entry for entry in self._destinations_extra if entry.get("created", 0) >= cutoff + ] + + def _maybe_rotate_session(self, force: bool = False) -> None: + if not self.enabled(): + return + if not self._rotation_enabled(): + return + interval = self._session_rotate_interval() + if interval <= 0: + return + now = time.time() + if not force and (now - self._last_identity_rotate) < interval: + return + try: + import RNS # type: ignore + except Exception: + return + if self._destination is not None: + self._destinations_extra.append( + {"dest": self._destination, "created": self._destination_created or now} + ) + identity = RNS.Identity() + try: + destination = self._create_destination(identity) + except Exception as exc: + logger.warning(f"RNS session rotate failed: {exc}") + return + self._identity = identity + self._destination = destination + self._destination_created = now + self._last_identity_rotate = now + self._prune_rotated_destinations(interval) + logger.info("RNS session identity rotated") + + def _privacy_profile(self) -> str: + now = time.time() + if now - float(self._privacy_cache.get("ts", 0)) > 5: + try: + data = read_wormhole_settings() + profile = str(data.get("privacy_profile", "default") or "default").lower() + except Exception: + profile = "default" + self._privacy_cache = {"value": profile, "ts": now} + return str(self._privacy_cache.get("value", "default")) + + def _is_high_privacy(self) -> bool: + return self._privacy_profile() == "high" + + def _prune_shards(self) -> None: + ttl = max(5, int(get_settings().MESH_RNS_SHARD_TTL_S)) + cutoff = time.time() - ttl + with self._shard_lock: + for shard_id, entry in list(self._shard_cache.items()): + if entry.get("created", 0) < cutoff: + del self._shard_cache[shard_id] + + def _prune_sync_rounds(self) -> None: + settings = get_settings() + timeout = int(settings.MESH_RNS_IBF_QUORUM_TIMEOUT_S or 0) + if timeout <= 0: + return + timeout = max(3, timeout) + now = time.time() + merged_sets: list[list[dict]] = [] + with self._sync_lock: + for sync_id, entry in list(self._pending_sync.items()): + if now - float(entry.get("created", now)) < timeout: + continue + # Fallback: choose the largest agreement bucket, if any + buckets = entry.get("responses", {}) + best_hash = "" + best_count = 0 + for head_hash, bucket in buckets.items(): + count = int(bucket.get("count", 0) or 0) + if count > best_count: + best_count = count + best_hash = head_hash + if best_hash: + merged = self._merge_bucket_events(buckets.get(best_hash, {})) + if merged: + merged_sets.append(merged) + del self._pending_sync[sync_id] + for merged in merged_sets: + self._ingest_ordered(merged) + + def _ibf_in_cooldown(self) -> bool: + with self._ibf_lock: + return time.time() < self._ibf_cooldown_until + + def _note_ibf_failure(self) -> None: + settings = get_settings() + threshold = int(settings.MESH_RNS_IBF_FAIL_THRESHOLD or 0) + cooldown = int(settings.MESH_RNS_IBF_COOLDOWN_S or 0) + if threshold <= 0 or cooldown <= 0: + return + with self._ibf_lock: + self._ibf_fail_count += 1 + if self._ibf_fail_count >= threshold: + self._ibf_cooldown_until = time.time() + cooldown + self._ibf_fail_count = 0 + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("ibf_sync_failure") + except Exception: + pass + + def _note_ibf_success(self) -> None: + with self._ibf_lock: + self._ibf_fail_count = 0 + + @staticmethod + def _xor_bytes(a: bytes, b: bytes) -> bytes: + return bytes(x ^ y for x, y in zip(a, b)) + + def _rs_parity_shards(self, data: list[bytes], parity_shards: int) -> list[bytes] | None: + try: + import reedsolo # type: ignore + except Exception: + return None + if parity_shards <= 0 or not data: + return [] + data_shards = len(data) + if data_shards + parity_shards > 255: + logger.warning("RNS RS FEC requires data+parity <= 255; falling back to XOR") + return None + try: + rsc = reedsolo.RSCodec(parity_shards) + except Exception: + return None + shard_size = len(data[0]) + parity = [bytearray(shard_size) for _ in range(parity_shards)] + for pos in range(shard_size): + row = bytes(chunk[pos] for chunk in data) + encoded = rsc.encode(row) + parity_bytes = encoded[-parity_shards:] + for p in range(parity_shards): + parity[p][pos] = parity_bytes[p] + return [bytes(p) for p in parity] + + def _rs_recover_missing( + self, + data_map: dict[int, bytes], + parity_map: dict[int, bytes], + data_shards: int, + parity_shards: int, + shard_size: int, + ) -> dict[int, bytes] | None: + try: + import reedsolo # type: ignore + except Exception: + return None + missing = [i for i in range(data_shards) if i not in data_map] + if not missing or len(missing) > parity_shards: + return None + total = data_shards + parity_shards + if total > 255: + return None + try: + rsc = reedsolo.RSCodec(parity_shards) + except Exception: + return None + recovered: dict[int, bytearray] = {idx: bytearray(shard_size) for idx in missing} + for pos in range(shard_size): + codeword = bytearray(total) + erasures: list[int] = [] + for idx in range(total): + if idx < data_shards: + if idx in data_map: + codeword[idx] = data_map[idx][pos] + else: + codeword[idx] = 0 + erasures.append(idx) + else: + if idx in parity_map: + codeword[idx] = parity_map[idx][pos] + else: + codeword[idx] = 0 + erasures.append(idx) + if len(erasures) > parity_shards: + return None + decoded, _ecc, _err = rsc.decode(bytes(codeword), erase_pos=erasures) + for shard_idx in missing: + recovered[shard_idx][pos] = decoded[shard_idx] + return {idx: bytes(buf) for idx, buf in recovered.items()} + + def _split_payload(self, payload: bytes, data_shards: int) -> tuple[list[bytes], int]: + if data_shards <= 0: + return [payload], len(payload) + total_len = len(payload) + shard_size = int(math.ceil(total_len / float(data_shards))) + shards: list[bytes] = [] + for idx in range(data_shards): + start = idx * shard_size + end = start + shard_size + chunk = payload[start:end] + if len(chunk) < shard_size: + chunk = chunk + b"\x00" * (shard_size - len(chunk)) + shards.append(chunk) + return shards, total_len + + def _send_sharded_payload(self, payload: bytes, message_id: str) -> bool: + settings = get_settings() + data_shards = max(1, int(settings.MESH_RNS_SHARD_DATA_SHARDS)) + parity_shards = max(0, int(settings.MESH_RNS_SHARD_PARITY_SHARDS)) + fec = str(settings.MESH_RNS_FEC_CODEC or "xor").lower() + if fec not in ("xor", "rs"): + fec = "xor" + if self._is_high_privacy() and fec == "xor": + fec = "rs" + if fec == "xor" and parity_shards > 1: + parity_shards = 1 + payload_len = len(payload) + if payload_len <= 1024: + return False + if payload_len <= 4096: + data_shards = min(data_shards, 2) if data_shards else 2 + parity_shards = min(max(parity_shards, 1), 2) + peers = self._active_peers or self._select_peers(self._parse_peers()) + if not peers: + return False + import random + + peers = list(peers) + random.shuffle(peers) + + data, total_len = self._split_payload(payload, data_shards) + shard_size = len(data[0]) if data else len(payload) + parity_blobs: list[bytes] = [] + if parity_shards > 0 and data: + if fec == "rs": + parity_blobs = self._rs_parity_shards(data, parity_shards) or [] + if len(parity_blobs) != parity_shards: + fec = "xor" + parity_blobs = [] + if fec == "xor": + parity_blob = data[0] + for chunk in data[1:]: + parity_blob = self._xor_bytes(parity_blob, chunk) + parity_blobs = [parity_blob] + + shard_id = uuid.uuid4().hex + total = data_shards + len(parity_blobs) + shard_messages: list[bytes] = [] + + for idx, chunk in enumerate(data): + body = { + "shard_id": shard_id, + "index": idx, + "total": total, + "data_shards": data_shards, + "parity_shards": len(parity_blobs), + "size": shard_size, + "length": total_len, + "parity": False, + "fec": fec, + "data": base64.b64encode(chunk).decode("ascii"), + } + shard_messages.append( + RNSMessage( + msg_type="infonet_shard", + body=body, + meta={"message_id": f"shard:{shard_id}:{idx}", "ts": int(time.time())}, + ).encode() + ) + + for p, parity_blob in enumerate(parity_blobs): + idx = data_shards + p + body = { + "shard_id": shard_id, + "index": idx, + "total": total, + "data_shards": data_shards, + "parity_shards": len(parity_blobs), + "size": shard_size, + "length": total_len, + "parity": True, + "fec": fec, + "data": base64.b64encode(parity_blob).decode("ascii"), + } + shard_messages.append( + RNSMessage( + msg_type="infonet_shard", + body=body, + meta={"message_id": f"shard:{shard_id}:{idx}", "ts": int(time.time())}, + ).encode() + ) + + random.shuffle(shard_messages) + if any(len(msg) > settings.MESH_RNS_MAX_PAYLOAD for msg in shard_messages): + logger.warning("RNS shard payload too large; falling back to direct send") + return False + + for i, msg in enumerate(shard_messages): + peer = peers[i % len(peers)] + self._send_to_peer(peer, msg) + return True + + def _seen(self, message_id: str) -> bool: + self._prune_dedupe() + with self._dedupe_lock: + if message_id in self._dedupe: + return True + self._dedupe[message_id] = time.time() + return False + + def _parse_peers(self) -> list[str]: + settings = get_settings() + raw = settings.MESH_RNS_PEERS or "" + peers = [p.strip().lower() for p in raw.split(",") if p.strip()] + return peers[: settings.MESH_RNS_MAX_PEERS] + + def _peer_bucket(self, peer_hash: str) -> str: + prefix_len = max(1, int(get_settings().MESH_RNS_PEER_BUCKET_PREFIX)) + return peer_hash[:prefix_len] + + def _peer_in_cooldown(self, peer_hash: str) -> bool: + settings = get_settings() + with self._peer_lock: + stats = self._peer_stats.get(peer_hash) + if not stats: + return False + fails = int(stats.get("fail", 0)) + last_fail = float(stats.get("last_fail", 0)) + if fails < settings.MESH_RNS_PEER_FAIL_THRESHOLD: + return False + return (time.time() - last_fail) < settings.MESH_RNS_PEER_COOLDOWN_S + + def _select_peers(self, peers: list[str]) -> list[str]: + settings = get_settings() + if not peers: + return [] + import random + + random.shuffle(peers) + buckets: dict[str, int] = {} + selected: list[str] = [] + for peer in peers: + if self._peer_in_cooldown(peer): + continue + bucket = self._peer_bucket(peer) + if buckets.get(bucket, 0) >= settings.MESH_RNS_MAX_PEERS_PER_BUCKET: + continue + buckets[bucket] = buckets.get(bucket, 0) + 1 + selected.append(peer) + if len(selected) >= settings.MESH_RNS_MAX_PEERS: + break + return selected + + def _maybe_churn(self) -> None: + settings = get_settings() + if not settings.MESH_RNS_CHURN_INTERVAL_S: + return + now = time.time() + interval = settings.MESH_RNS_CHURN_INTERVAL_S + if self._is_high_privacy(): + interval = min(interval, 60) + if now - self._last_churn < interval: + return + peers = self._parse_peers() + self._active_peers = self._select_peers(peers) + self._last_churn = now + + def _pick_stem_peer(self) -> str | None: + self._maybe_churn() + peers = self._active_peers or self._select_peers(self._parse_peers()) + if not peers: + return None + import random + + return random.choice(peers) + + def _dandelion_hops(self) -> int: + settings = get_settings() + base = max(1, int(settings.MESH_RNS_DANDELION_HOPS)) + if not self._is_high_privacy(): + return base + peer_count = len(self._active_peers) or len(self._parse_peers()) + if peer_count <= 3: + return min(base, 1) + if peer_count <= 7: + return min(base, 2) if base >= 2 else base + return base + + def _send_to_peer(self, peer_hash: str, payload: bytes) -> bool: + settings = get_settings() + if not self._reticulum or not self._enabled: + return False + try: + import RNS # type: ignore + + dest = RNS.Destination( + None, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + settings.MESH_RNS_APP_NAME, + settings.MESH_RNS_ASPECT, + ) + # Best-effort: assign destination hash directly if supported + try: + dest.hash = bytes.fromhex(peer_hash) + except Exception: + pass + packet = RNS.Packet(dest, payload) + packet.send() + with self._peer_lock: + stats = self._peer_stats.get(peer_hash) or { + "fail": 0, + "success": 0, + "last_fail": 0, + } + stats["success"] = float(stats.get("success", 0)) + 1 + self._peer_stats[peer_hash] = stats + return True + except Exception as exc: + logger.debug(f"RNS send failed: {exc}") + with self._peer_lock: + stats = self._peer_stats.get(peer_hash) or { + "fail": 0, + "success": 0, + "last_fail": 0, + } + stats["fail"] = float(stats.get("fail", 0)) + 1 + stats["last_fail"] = time.time() + self._peer_stats[peer_hash] = stats + return False + + def _local_hash(self) -> str: + if not self._destination: + return "" + try: + return self._destination.hash.hex() + except Exception: + return "" + + def _make_message_id(self, prefix: str) -> str: + return f"{prefix}:{uuid.uuid4().hex}" + + def _send_message(self, peer_hash: str, msg_type: str, body: dict, meta: dict | None = None) -> bool: + settings = get_settings() + base_meta = { + "message_id": self._make_message_id(msg_type), + "reply_to": self._local_hash(), + "ts": int(time.time()), + } + if meta: + base_meta.update(meta) + payload = RNSMessage(msg_type=msg_type, body=body, meta=base_meta).encode() + if len(payload) > settings.MESH_RNS_MAX_PAYLOAD: + logger.warning(f"RNS payload too large for {msg_type}; dropped") + return False + return self._send_to_peer(peer_hash, payload) + + def _send_events(self, peer_hash: str, events: list[dict]) -> None: + settings = get_settings() + if not events: + return + limited = events[: settings.MESH_RNS_IBF_MAX_EVENTS] + while limited: + payload = RNSMessage( + msg_type="ibf_sync_events", + body={"events": limited}, + meta={ + "message_id": self._make_message_id("ibf_sync_events"), + "reply_to": self._local_hash(), + "ts": int(time.time()), + }, + ).encode() + if len(payload) <= settings.MESH_RNS_MAX_PAYLOAD: + self._send_to_peer(peer_hash, payload) + return + limited = limited[: max(1, len(limited) // 2)] + logger.warning("RNS payload too large for ibf_sync_events; dropped") + + def _recent_event_ids(self, window: int) -> list[str]: + if window <= 0: + return [] + try: + from services.mesh.mesh_hashchain import infonet + + events = infonet.events + tail = events[-window:] if len(events) > window else events[:] + out = [] + for evt in tail: + if isinstance(evt, dict): + if evt.get("event_type") == "gate_message": + continue + eid = evt.get("event_id", "") + if eid: + out.append(eid) + return out + except Exception: + return [] + + def _build_ibf_table(self, window: int, table_size: int) -> tuple[IBLT, list[int], int]: + event_ids = self._recent_event_ids(window) + keys = [] + for eid in event_ids: + try: + keys.append(bytes.fromhex(eid)) + except Exception: + continue + iblt = build_iblt(keys, table_size) + minhash = minhash_sketch(keys, k=get_settings().MESH_RNS_IBF_MINHASH_SIZE) + return iblt, minhash, len(keys) + + def _build_ibf_init_payload(self, sync_id: str) -> bytes | None: + settings = get_settings() + base_window = settings.MESH_RNS_IBF_WINDOW + jitter = max(0, int(settings.MESH_RNS_IBF_WINDOW_JITTER)) + if self._is_high_privacy() and jitter == 0: + jitter = 32 + if jitter: + import random + + window = max(32, base_window + random.randint(-jitter, jitter)) + else: + window = base_window + table_sizes = [ + settings.MESH_RNS_IBF_TABLE_SIZE, + max(16, settings.MESH_RNS_IBF_TABLE_SIZE // 2), + max(16, settings.MESH_RNS_IBF_TABLE_SIZE // 4), + ] + table_sizes = list(dict.fromkeys([s for s in table_sizes if s > 0])) + + for size in table_sizes: + iblt, minhash, key_count = self._build_ibf_table(window, size) + body = { + "window": window, + "table": iblt.to_compact_dict(), + "minhash": minhash, + "keys": key_count, + } + payload = RNSMessage( + msg_type="ibf_sync_init", + body=body, + meta={ + "message_id": self._make_message_id("ibf_sync_init"), + "reply_to": self._local_hash(), + "ts": int(time.time()), + "sync_id": sync_id, + }, + ).encode() + if len(payload) <= settings.MESH_RNS_MAX_PAYLOAD: + return payload + logger.warning("RNS payload too large for ibf_sync_init; dropped") + return None + + def _ibf_sync_loop(self) -> None: + settings = get_settings() + interval = max(10, settings.MESH_RNS_IBF_INTERVAL_S) + while True: + try: + if not self.enabled(): + time.sleep(interval) + continue + self._maybe_rotate_session() + if self._ibf_in_cooldown(): + time.sleep(interval) + continue + now = time.time() + if now - self._last_ibf_sync >= interval: + self._last_ibf_sync = now + self._send_ibf_sync_init() + self._prune_sync_rounds() + except Exception: + pass + time.sleep(interval) + + def _send_ibf_sync_init(self) -> None: + peers = self._select_sync_peers() + if not peers: + return + sync_id = uuid.uuid4().hex + payload = self._build_ibf_init_payload(sync_id) + if not payload: + return + with self._sync_lock: + self._pending_sync[sync_id] = { + "created": time.time(), + "expected": set(peers), + "quorum": max(1, (len(peers) // 2) + 1), + "responses": {}, + "responders": set(), + } + for peer in peers: + self._send_to_peer(peer, payload) + + def _ingest_ordered(self, events: list[dict]) -> None: + if not events: + return + try: + from services.mesh.mesh_hashchain import infonet + + by_prev: dict[str, dict] = {} + for evt in events: + if not isinstance(evt, dict): + continue + prev = evt.get("prev_hash", "") + eid = evt.get("event_id", "") + if not prev or not eid: + continue + if prev not in by_prev: + by_prev[prev] = evt + + ordered = [] + current = infonet.head_hash + while current in by_prev: + evt = by_prev[current] + ordered.append(evt) + current = evt.get("event_id", "") + if not current: + break + + if ordered: + infonet.ingest_events(ordered) + except Exception: + pass + + def _handle_ibf_sync_init(self, body: dict, meta: dict) -> None: + reply_to = str(meta.get("reply_to", "") or "") + if not reply_to: + return + sync_id = str(meta.get("sync_id", "") or "") + try: + remote_table = IBLT.from_compact_dict(body.get("table") or {}) + except Exception: + return + + window = _safe_int(body.get("window", 0) or 0) + if window <= 0: + return + window = min(window, get_settings().MESH_RNS_IBF_WINDOW) + remote_minhash = body.get("minhash") or [] + local_table, local_minhash, _keys = self._build_ibf_table(window, remote_table.size) + threshold = float(get_settings().MESH_RNS_IBF_MINHASH_THRESHOLD or 0.0) + if remote_minhash and local_minhash and threshold > 0: + similarity = minhash_similarity(remote_minhash, local_minhash) + if similarity < threshold: + self._send_message( + reply_to, + "ibf_sync_nak", + {"reason": "low_similarity", "similarity": similarity}, + ) + return + + diff = remote_table.subtract(local_table) + ok, plus, minus = diff.decode() + if not ok: + self._send_message( + reply_to, + "ibf_sync_nak", + {"reason": "decode_failed", "suggested_table": remote_table.size * 2}, + ) + return + + request_ids = [key.hex() for key in plus] + request_ids = request_ids[: get_settings().MESH_RNS_IBF_MAX_REQUEST_IDS] + + events_out: list[dict] = [] + if minus: + from services.mesh.mesh_hashchain import infonet + + for key in minus: + eid = key.hex() + evt = infonet.get_event(eid) + if evt and evt.get("event_type") != "gate_message": + events_out.append(evt) + if len(events_out) >= get_settings().MESH_RNS_IBF_MAX_EVENTS: + break + settings = get_settings() + request_ids = request_ids[: settings.MESH_RNS_IBF_MAX_REQUEST_IDS] + while True: + head_hash = "" + try: + from services.mesh.mesh_hashchain import infonet + + head_hash = infonet.head_hash + except Exception: + head_hash = "" + payload = RNSMessage( + msg_type="ibf_sync_delta", + body={"request": request_ids, "events": events_out}, + meta={ + "message_id": self._make_message_id("ibf_sync_delta"), + "reply_to": self._local_hash(), + "ts": int(time.time()), + "sync_id": sync_id, + "head_hash": head_hash, + }, + ).encode() + if len(payload) <= settings.MESH_RNS_MAX_PAYLOAD: + self._send_to_peer(reply_to, payload) + return + if events_out: + events_out = events_out[: max(1, len(events_out) // 2)] + continue + if request_ids: + request_ids = request_ids[: max(1, len(request_ids) // 2)] + continue + break + + def _handle_ibf_sync_delta(self, body: dict, meta: dict) -> None: + reply_to = str(meta.get("reply_to", "") or "") + events = body.get("events") or [] + if isinstance(events, list): + self._note_ibf_success() + self._ingest_with_quorum(events, meta) + + request_ids = body.get("request") or [] + if reply_to and isinstance(request_ids, list) and request_ids: + from services.mesh.mesh_hashchain import infonet + + events_out = [] + for eid in request_ids[: get_settings().MESH_RNS_IBF_MAX_REQUEST_IDS]: + evt = infonet.get_event(str(eid)) + if evt and evt.get("event_type") != "gate_message": + events_out.append(evt) + if events_out: + self._send_events(reply_to, events_out) + + def _handle_ibf_sync_events(self, body: dict) -> None: + events = body.get("events") or [] + if isinstance(events, list): + self._note_ibf_success() + self._ingest_ordered(events) + + def _handle_infonet_shard(self, body: dict) -> None: + self._prune_shards() + shard_id = str(body.get("shard_id", "") or "") + if not shard_id: + return + try: + index = _safe_int(body.get("index", 0) or 0) + total = _safe_int(body.get("total", 0) or 0) + data_shards = _safe_int(body.get("data_shards", 0) or 0) + parity_shards = _safe_int(body.get("parity_shards", 0) or 0) + size = _safe_int(body.get("size", 0) or 0) + length = _safe_int(body.get("length", 0) or 0) + parity = bool(body.get("parity", False)) + fec = str(body.get("fec", "xor") or "xor").lower() + blob = base64.b64decode(str(body.get("data", ""))) + except Exception: + return + + assembled: bytes | None = None + with self._shard_lock: + entry = self._shard_cache.get(shard_id) + if not entry: + entry = { + "created": time.time(), + "total": total, + "data_shards": data_shards, + "parity_shards": parity_shards, + "size": size, + "length": length, + "data": {}, + "parity": {}, + "fec": fec, + } + self._shard_cache[shard_id] = entry + + if parity: + entry["parity"][index] = blob + else: + entry["data"][index] = blob + + data_map: dict[int, bytes] = entry.get("data", {}) + parity_map: dict[int, bytes] = entry.get("parity", {}) + if data_shards > 0 and len(data_map) >= data_shards: + assembled = b"".join(data_map[i] for i in range(data_shards) if i in data_map) + assembled = assembled[:length] if length else assembled + del self._shard_cache[shard_id] + elif data_shards > 0: + fec = str(entry.get("fec", "xor") or "xor").lower() + if fec == "rs" and parity_shards > 0: + recovered = self._rs_recover_missing( + data_map, parity_map, data_shards, parity_shards, size + ) + if recovered: + data_map.update(recovered) + assembled = b"".join( + data_map[i] for i in range(data_shards) if i in data_map + ) + assembled = assembled[:length] if length else assembled + del self._shard_cache[shard_id] + elif fec == "xor" and parity_map and len(data_map) == data_shards - 1: + missing = [i for i in range(data_shards) if i not in data_map] + if len(missing) == 1: + parity_blob = next(iter(parity_map.values())) + recovered = parity_blob + for i in range(data_shards): + if i == missing[0]: + continue + recovered = self._xor_bytes(recovered, data_map[i]) + data_map[missing[0]] = recovered + assembled = b"".join( + data_map[i] for i in range(data_shards) if i in data_map + ) + assembled = assembled[:length] if length else assembled + del self._shard_cache[shard_id] + if assembled: + self._on_packet(assembled) + + def _send_diffuse(self, payload: bytes, exclude: str | None = None) -> int: + sent = 0 + peers = self._active_peers or self._select_peers(self._parse_peers()) + for peer in peers: + if exclude and peer == exclude: + continue + if self._send_to_peer(peer, payload): + sent += 1 + return sent + + def send_private_dm(self, *, mailbox_key: str, envelope: dict[str, Any]) -> bool: + if not self.enabled(): + return False + if not mailbox_key or not isinstance(envelope, dict): + return False + blinded_mailbox_key = _blind_mailbox_key(mailbox_key) + if not blinded_mailbox_key: + return False + message_id = str(envelope.get("msg_id", "") or self._make_message_id("private_dm")) + payload = RNSMessage( + msg_type="private_dm", + body={"mailbox_key": blinded_mailbox_key, "envelope": envelope}, + meta={ + "message_id": f"private_dm:{message_id}", + "dandelion": {"phase": "stem", "hops": 0, "max_hops": self._dandelion_hops()}, + }, + ).encode() + if len(payload) > get_settings().MESH_RNS_MAX_PAYLOAD: + logger.warning("RNS private DM payload too large; falling back to relay") + return False + stem_peer = self._pick_stem_peer() + if stem_peer: + if not self._send_to_peer(stem_peer, payload): + return False + + def _diffuse_dm() -> None: + diffuse = RNSMessage( + msg_type="private_dm", + body={"mailbox_key": blinded_mailbox_key, "envelope": envelope}, + meta={"message_id": f"private_dm:{message_id}", "dandelion": {"phase": "diffuse"}}, + ).encode() + self._send_diffuse(diffuse, exclude=stem_peer) + + delay_s = max(0, get_settings().MESH_RNS_DANDELION_DELAY_MS / 1000.0) + threading.Timer(delay_s, _diffuse_dm).start() + return True + return self._send_diffuse(payload) > 0 + + def _store_private_dm(self, mailbox_key: str, envelope: dict[str, Any]) -> None: + msg_id = str(envelope.get("msg_id", "") or "") + if not mailbox_key or not msg_id: + return + with self._dm_lock: + mailbox = self._dm_mailboxes.setdefault(mailbox_key, []) + if any(str(item.get("msg_id", "") or "") == msg_id for item in mailbox): + return + mailbox.append( + { + "sender_id": str(envelope.get("sender_id", "") or ""), + "ciphertext": str(envelope.get("ciphertext", "") or ""), + "timestamp": float(envelope.get("timestamp", 0) or time.time()), + "msg_id": msg_id, + "delivery_class": str(envelope.get("delivery_class", "shared") or "shared"), + "sender_seal": str(envelope.get("sender_seal", "") or ""), + "transport": "reticulum", + } + ) + + def collect_private_dm(self, mailbox_keys: list[str]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + with self._dm_lock: + for key in mailbox_keys: + blinded_key = _blind_mailbox_key(key) + if not blinded_key: + continue + mailbox = self._dm_mailboxes.pop(blinded_key, []) + for item in mailbox: + msg_id = str(item.get("msg_id", "") or "") + if not msg_id or msg_id in seen: + continue + seen.add(msg_id) + out.append(item) + return sorted(out, key=lambda item: float(item.get("timestamp", 0) or 0)) + + def count_private_dm(self, mailbox_keys: list[str]) -> int: + seen: set[str] = set() + with self._dm_lock: + for key in mailbox_keys: + blinded_key = _blind_mailbox_key(key) + if not blinded_key: + continue + for item in self._dm_mailboxes.get(blinded_key, []): + msg_id = str(item.get("msg_id", "") or "") + if msg_id: + seen.add(msg_id) + return len(seen) + + def private_dm_ids(self, mailbox_keys: list[str]) -> set[str]: + seen: set[str] = set() + with self._dm_lock: + for key in mailbox_keys: + blinded_key = _blind_mailbox_key(key) + if not blinded_key: + continue + for item in self._dm_mailboxes.get(blinded_key, []): + msg_id = str(item.get("msg_id", "") or "") + if msg_id: + seen.add(msg_id) + return seen + + def _publish_now(self, event: dict, message_id: str) -> None: + if not self.enabled(): + return + settings = get_settings() + priority = "" + payload_info = event.get("payload", {}) + if isinstance(payload_info, dict): + priority = str(payload_info.get("priority", "")).lower() + payload = RNSMessage( + msg_type="infonet_event", + body={"event": event}, + meta={ + "message_id": message_id, + "dandelion": { + "phase": "stem", + "hops": 0, + "max_hops": self._dandelion_hops(), + }, + }, + ).encode() + + if len(payload) > settings.MESH_RNS_MAX_PAYLOAD: + logger.warning("RNS payload too large; event not sent") + return + + if settings.MESH_RNS_SHARD_ENABLED or self._is_high_privacy(): + if priority in ("emergency", "high") and len(payload) <= settings.MESH_RNS_MAX_PAYLOAD: + pass + elif self._send_sharded_payload(payload, message_id): + return + + stem_peer = self._pick_stem_peer() + if stem_peer: + self._send_to_peer(stem_peer, payload) + + def _diffuse(): + diffuse_payload = RNSMessage( + msg_type="infonet_event", + body={"event": event}, + meta={"message_id": message_id, "dandelion": {"phase": "diffuse"}}, + ).encode() + self._send_diffuse(diffuse_payload, exclude=stem_peer) + + delay_s = max(0, settings.MESH_RNS_DANDELION_DELAY_MS / 1000.0) + threading.Timer(delay_s, _diffuse).start() + else: + self._send_diffuse(payload) + + def _flush_batch(self) -> None: + with self._batch_lock: + queued = list(self._batch_queue) + self._batch_queue.clear() + if self._batch_timer: + self._batch_timer.cancel() + self._batch_timer = None + for event in queued: + message_id = event.get("event_id", "") or self._make_message_id("event") + self._publish_now(event, message_id) + + def _queue_event(self, event: dict) -> None: + settings = get_settings() + max_batch = 25 + should_flush = False + with self._batch_lock: + self._batch_queue.append(event) + if len(self._batch_queue) >= max_batch: + should_flush = True + else: + if self._batch_timer is None: + delay = max(0, settings.MESH_RNS_BATCH_MS) / 1000.0 + timer = threading.Timer(delay, self._flush_batch) + timer.daemon = True + self._batch_timer = timer + timer.start() + if should_flush: + self._flush_batch() + + def publish_event(self, event: dict) -> None: + if not self.enabled(): + return + self._maybe_rotate_session() + settings = get_settings() + message_id = event.get("event_id", "") + if message_id and self._seen(message_id): + return + if self._is_high_privacy() and settings.MESH_RNS_BATCH_MS > 0: + self._queue_event(event) + return + self._publish_now(event, message_id or self._make_message_id("event")) + + def publish_gate_event(self, gate_id: str, event: dict) -> None: + """Publish a gate message on the private plane using the current signer-carried v1 envelope.""" + if not self.enabled(): + return + self._maybe_rotate_session() + local_event_id = str(event.get("event_id", "") or "") + if local_event_id and self._seen(local_event_id): + return + + payload_info = event.get("payload") if isinstance(event, dict) else {} + if not isinstance(payload_info, dict): + payload_info = {} + from services.mesh.mesh_hashchain import build_gate_wire_ref + + safe_event = { + "event_type": "gate_message", + "timestamp": event.get("timestamp", 0), + "payload": { + "ciphertext": str(payload_info.get("ciphertext", "") or ""), + "format": str(payload_info.get("format", "") or ""), + }, + } + nonce = str(payload_info.get("nonce", "") or "") + sender_ref = str(payload_info.get("sender_ref", "") or "") + epoch = int(payload_info.get("epoch", 0) or 0) + if nonce: + safe_event["payload"]["nonce"] = nonce + if sender_ref: + safe_event["payload"]["sender_ref"] = sender_ref + if epoch > 0: + safe_event["payload"]["epoch"] = epoch + for field_name in ( + "event_id", + "node_id", + "sequence", + "signature", + "public_key", + "public_key_algo", + "protocol_version", + ): + value = event.get(field_name, "") + if value not in ("", None): + safe_event[field_name] = value + gate_ref = build_gate_wire_ref(str(payload_info.get("gate", "") or gate_id), safe_event) + if not gate_ref: + logger.warning("RNS private gate forwarding requires MESH_PEER_PUSH_SECRET; event not sent") + return + safe_event["payload"]["gate_ref"] = gate_ref + wire_message_id = self._make_message_id("gate") + payload = RNSMessage( + msg_type="gate_event", + body={"event": safe_event}, + meta={ + "message_id": wire_message_id, + "dandelion": { + "phase": "stem", + "hops": 0, + "max_hops": self._dandelion_hops(), + }, + }, + ).encode() + if len(payload) > get_settings().MESH_RNS_MAX_PAYLOAD: + logger.warning("RNS gate payload too large; event not sent") + return + stem_peer = self._pick_stem_peer() + if stem_peer: + self._send_to_peer(stem_peer, payload) + + def _diffuse_gate() -> None: + diffuse_payload = RNSMessage( + msg_type="gate_event", + body={"event": safe_event}, + meta={"message_id": wire_message_id, "dandelion": {"phase": "diffuse"}}, + ).encode() + self._send_diffuse(diffuse_payload, exclude=stem_peer) + + delay_s = max(0, get_settings().MESH_RNS_DANDELION_DELAY_MS / 1000.0) + threading.Timer(delay_s, _diffuse_gate).start() + return + self._send_diffuse(payload) + + def _cover_interval(self) -> float: + settings = get_settings() + interval = float(settings.MESH_RNS_COVER_INTERVAL_S or 0) + if self._is_high_privacy() and interval <= 0: + interval = 15.0 + if self._batch_queue: + qlen = len(self._batch_queue) + if qlen >= 25: + interval *= 3 + elif qlen >= 10: + interval *= 2 + return interval + + def _send_cover_traffic(self) -> None: + settings = get_settings() + size = max(16, int(settings.MESH_RNS_COVER_SIZE)) + payload = os.urandom(size) + msg = RNSMessage( + msg_type="cover_traffic", + body={"pad": base64.b64encode(payload).decode("ascii"), "size": size}, + meta={"message_id": self._make_message_id("cover"), "ts": int(time.time())}, + ).encode() + if len(msg) > settings.MESH_RNS_MAX_PAYLOAD: + return + peer = self._pick_stem_peer() + if peer: + self._send_to_peer(peer, msg) + + def _cover_loop(self) -> None: + import random + + while True: + try: + if not self.enabled() or not self._is_high_privacy(): + time.sleep(3) + continue + interval = self._cover_interval() + if interval <= 0: + time.sleep(5) + continue + self._send_cover_traffic() + jitter = random.uniform(0.7, 1.3) + time.sleep(interval * jitter) + except Exception: + time.sleep(5) + + def _peer_score(self, peer_hash: str) -> float: + with self._peer_lock: + stats = self._peer_stats.get(peer_hash, {}) + success = float(stats.get("success", 0)) + fail = float(stats.get("fail", 0)) + return success - (fail * 2) + + def _select_sync_peers(self) -> list[str]: + settings = get_settings() + peers = self._active_peers or self._select_peers(self._parse_peers()) + if not peers: + return [] + scored = sorted(peers, key=self._peer_score, reverse=True) + max_peers = max(1, int(settings.MESH_RNS_IBF_SYNC_PEERS)) + return scored[: max_peers] + + def _merge_bucket_events(self, bucket: dict[str, Any]) -> list[dict]: + events = [] + seen = set() + for evt_list in bucket.get("events", []): + if not isinstance(evt_list, list): + continue + for evt in evt_list: + if not isinstance(evt, dict): + continue + eid = evt.get("event_id", "") + if eid and eid in seen: + continue + if eid: + seen.add(eid) + events.append(evt) + return events + + def _ingest_with_quorum(self, events: list[dict], meta: dict) -> None: + sync_id = str(meta.get("sync_id", "") or "") + head_hash = str(meta.get("head_hash", "") or "") + if not sync_id or not head_hash: + self._ingest_ordered(events) + return + merged: list[dict] | None = None + quorum = 1 + bucket_count = 0 + bucket_events: list[list[dict]] | None = None + with self._sync_lock: + entry = self._pending_sync.get(sync_id) + if not entry: + self._ingest_ordered(events) + return + peer_id = str(meta.get("reply_to", "") or "") + responders = entry.get("responders", set()) + if peer_id and peer_id in responders: + return + if peer_id: + responders.add(peer_id) + buckets = entry.get("responses", {}) + bucket = buckets.get(head_hash) or {"count": 0, "events": []} + bucket["count"] = int(bucket.get("count", 0) or 0) + 1 + bucket["events"].append(events) + buckets[head_hash] = bucket + entry["responses"] = buckets + entry["responders"] = responders + quorum = int(entry.get("quorum", 1) or 1) + bucket_count = int(bucket.get("count", 0) or 0) + if bucket_count >= quorum: + bucket_events = list(bucket.get("events", [])) + del self._pending_sync[sync_id] + if bucket_events is not None: + merged = self._merge_bucket_events({"events": bucket_events}) + if merged: + try: + from services.mesh.mesh_hashchain import infonet + + if head_hash and head_hash != infonet.head_hash: + applied, reason = infonet.apply_fork( + merged, head_hash, bucket_count, quorum + ) + if not applied: + logger.info(f"Fork rejected: {reason}") + try: + from services.mesh.mesh_metrics import increment as metrics_inc + + metrics_inc("fork_rejected") + except Exception: + pass + else: + logger.info("Fork applied by quorum") + else: + self._ingest_ordered(merged) + except Exception: + self._ingest_ordered(merged) + else: + self._prune_sync_rounds() + + def _on_packet(self, data: bytes, packet: Any = None) -> None: + settings = get_settings() + try: + msg = json.loads(data.decode("utf-8")) + except Exception: + return + msg_type = msg.get("type", "") + meta = msg.get("meta", {}) or {} + message_id = meta.get("message_id", "") + if message_id and self._seen(message_id): + return + + if msg_type == "ibf_sync_init": + body = msg.get("body") or {} + if isinstance(body, dict): + self._handle_ibf_sync_init(body, meta) + return + if msg_type == "ibf_sync_delta": + body = msg.get("body") or {} + if isinstance(body, dict): + self._handle_ibf_sync_delta(body, meta) + return + if msg_type == "ibf_sync_events": + body = msg.get("body") or {} + if isinstance(body, dict): + self._handle_ibf_sync_events(body) + return + if msg_type == "ibf_sync_nak": + self._note_ibf_failure() + return + if msg_type == "infonet_shard": + body = msg.get("body") or {} + if isinstance(body, dict): + self._handle_infonet_shard(body) + return + if msg_type == "cover_traffic": + return + if msg_type == "private_dm": + body = msg.get("body") or {} + if not isinstance(body, dict): + return + mailbox_key = str(body.get("mailbox_key", "") or "") + envelope = body.get("envelope") or {} + if not mailbox_key or not isinstance(envelope, dict): + return + + dandelion = meta.get("dandelion", {}) or {} + phase = dandelion.get("phase", "diffuse") + hops = int(dandelion.get("hops", 0) or 0) + max_hops = int(dandelion.get("max_hops", settings.MESH_RNS_DANDELION_HOPS) or 0) + + if phase == "stem" and hops < max_hops: + peer = self._pick_stem_peer() + if peer: + next_meta = { + "message_id": message_id, + "dandelion": {"phase": "stem", "hops": hops + 1, "max_hops": max_hops}, + } + forward = RNSMessage( + msg_type="private_dm", + body={"mailbox_key": mailbox_key, "envelope": envelope}, + meta=next_meta, + ).encode() + self._send_to_peer(peer, forward) + elif phase == "stem": + diffuse = RNSMessage( + msg_type="private_dm", + body={"mailbox_key": mailbox_key, "envelope": envelope}, + meta={"message_id": message_id, "dandelion": {"phase": "diffuse"}}, + ).encode() + self._send_diffuse(diffuse) + + self._store_private_dm(mailbox_key, envelope) + return + + if msg_type == "infonet_event": + event = (msg.get("body") or {}).get("event") + if not isinstance(event, dict): + return + + dandelion = meta.get("dandelion", {}) or {} + phase = dandelion.get("phase", "diffuse") + hops = int(dandelion.get("hops", 0) or 0) + max_hops = int(dandelion.get("max_hops", settings.MESH_RNS_DANDELION_HOPS) or 0) + + if phase == "stem" and hops < max_hops: + peer = self._pick_stem_peer() + if peer: + next_meta = { + "message_id": message_id, + "dandelion": {"phase": "stem", "hops": hops + 1, "max_hops": max_hops}, + } + forward = RNSMessage( + msg_type="infonet_event", + body={"event": event}, + meta=next_meta, + ).encode() + self._send_to_peer(peer, forward) + elif phase == "stem": + diffuse = RNSMessage( + msg_type="infonet_event", + body={"event": event}, + meta={"message_id": message_id, "dandelion": {"phase": "diffuse"}}, + ).encode() + self._send_diffuse(diffuse) + + # Ingest locally + try: + from services.mesh.mesh_hashchain import infonet + + infonet.ingest_events([event]) + except Exception: + pass + return + + if msg_type == "gate_event": + body = msg.get("body") or {} + if not isinstance(body, dict): + return + event = body.get("event") + if not isinstance(event, dict): + return + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + gate_id = str(payload.get("gate", "") or "").strip().lower() + if not gate_id: + try: + from services.mesh.mesh_hashchain import resolve_gate_wire_ref + + gate_id = resolve_gate_wire_ref(str(payload.get("gate_ref", "") or ""), event) + except Exception: + gate_id = "" + if not gate_id: + # Non-members can still forward opaque gate events even if they + # cannot resolve the local gate identifier. + gate_id = "" + + dandelion = meta.get("dandelion", {}) or {} + phase = dandelion.get("phase", "diffuse") + hops = int(dandelion.get("hops", 0) or 0) + max_hops = int(dandelion.get("max_hops", settings.MESH_RNS_DANDELION_HOPS) or 0) + + if phase == "stem" and hops < max_hops: + peer = self._pick_stem_peer() + if peer: + next_meta = { + "message_id": message_id, + "dandelion": {"phase": "stem", "hops": hops + 1, "max_hops": max_hops}, + } + forward = RNSMessage( + msg_type="gate_event", + body={"event": event}, + meta=next_meta, + ).encode() + self._send_to_peer(peer, forward) + elif phase == "stem": + diffuse = RNSMessage( + msg_type="gate_event", + body={"event": event}, + meta={"message_id": message_id, "dandelion": {"phase": "diffuse"}}, + ).encode() + self._send_diffuse(diffuse) + + if gate_id: + try: + from services.mesh.mesh_hashchain import gate_store + + event_for_store = dict(event) + payload_for_store = payload.copy() + payload_for_store["gate"] = gate_id + event_for_store["payload"] = payload_for_store + gate_store.ingest_peer_events(gate_id, [event_for_store]) + except Exception: + pass + + +rns_bridge = RNSBridge() diff --git a/backend/services/mesh/mesh_router.py b/backend/services/mesh/mesh_router.py new file mode 100644 index 00000000..07b77a12 --- /dev/null +++ b/backend/services/mesh/mesh_router.py @@ -0,0 +1,1087 @@ +"""Mesh Router — policy-driven multi-transport message routing. + +Routes messages through the optimal transport based on: + - Payload size (LoRa < 200 bytes, APRS < 67 chars, WiFi/Internet unlimited) + - Urgency (EMERGENCY → all available transports simultaneously) + - Destination type (APRS callsign → APRS, Meshtastic node → MQTT, etc.) + - Node reachability (what transports can reach the target?) + +Transports: + - APRS-IS: Two-way text to ham radio operators (max 67 chars, needs callsign+passcode) + - Meshtastic: MQTT publish to LoRa mesh (max ~200 bytes, public LongFast channel) + - Internet: Future — Reticulum, direct TCP, WebSocket relay + +The router doesn't care about the transport — it cares about getting the +message from A to B as efficiently as possible. +""" + +import json +import time +import logging +import hashlib +import hmac +import secrets +from dataclasses import dataclass, field, asdict +from enum import Enum +from typing import Optional +from collections import deque +from urllib.parse import urlparse +from services.mesh.mesh_crypto import _derive_peer_key, normalize_peer_url +from services.mesh.meshtastic_topics import normalize_root + +logger = logging.getLogger("services.mesh_router") + +DEDUP_TTL_SECONDS = 300 +DEDUP_MAX_ENTRIES = 5000 +_TRANSPORT_PAD_BUCKETS = (1024, 2048, 4096, 8192, 16384, 32768) + + +def _peer_audit_label(peer_url: str) -> str: + normalized = normalize_peer_url(peer_url) + if not normalized: + return "peer:unknown" + parsed = urlparse(normalized) + scheme = parsed.scheme or "peer" + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:10] + return f"{scheme}:{digest}" + + +def peer_transport_kind(peer_url: str) -> str: + normalized = normalize_peer_url(peer_url) + parsed = urlparse(normalized) + hostname = str(parsed.hostname or "").strip().lower() + if parsed.scheme == "http" and hostname.endswith(".onion"): + return "onion" + if parsed.scheme == "https" and hostname: + return "clearnet" + # Allow plain http for LAN / testnet peers (not .onion) + if parsed.scheme == "http" and hostname: + return "clearnet" + return "" + + +def parse_configured_relay_peers(raw: str) -> list[str]: + peers: list[str] = [] + seen: set[str] = set() + for candidate in str(raw or "").split(","): + url = normalize_peer_url(candidate) + transport = peer_transport_kind(url) + if not url or not transport or url in seen: + if str(candidate or "").strip(): + logger.warning( + "Ignoring peer URL (must be https:// or http://*.onion): %s", + str(candidate).strip()[:80], + ) + continue + seen.add(url) + peers.append(url) + return peers + + +def configured_relay_peer_urls() -> list[str]: + from services.config import get_settings + + raw = str(get_settings().MESH_RELAY_PEERS or "").strip() + return parse_configured_relay_peers(raw) + + +def _store_peer_urls(bucket: str, *, transport: str | None = None) -> list[str]: + try: + from services.mesh.mesh_peer_store import DEFAULT_PEER_STORE_PATH, PeerStore + + store = PeerStore(DEFAULT_PEER_STORE_PATH) + records = store.load() + except Exception: + return [] + + seen: set[str] = set() + urls: list[str] = [] + for record in records: + if record.bucket != bucket or not record.enabled: + continue + if transport and record.transport != transport: + continue + if record.peer_url in seen: + continue + seen.add(record.peer_url) + urls.append(record.peer_url) + return urls + + +def authenticated_push_peer_urls(*, transport: str | None = None) -> list[str]: + from_store = _store_peer_urls("push", transport=transport) + if from_store: + return from_store + configured = configured_relay_peer_urls() + if transport: + return [url for url in configured if peer_transport_kind(url) == transport] + return configured + + +def active_sync_peer_urls() -> list[str]: + from_store = _store_peer_urls("sync") + if from_store: + return from_store + return configured_relay_peer_urls() + + +def _high_privacy_profile_blocks_clearnet_fallback() -> bool: + # Explicit clearnet-fallback policy takes precedence over privacy-profile. + try: + from services.config import get_settings + + if str(get_settings().MESH_PRIVATE_CLEARNET_FALLBACK or "").strip().lower() == "block": + return True + except Exception: + pass + try: + from services.wormhole_settings import read_wormhole_settings + + settings = read_wormhole_settings() + return str(settings.get("privacy_profile", "default") or "default").strip().lower() == "high" + except Exception: + return False + + +def _pad_transport_payload(raw_json_bytes: bytes) -> bytes: + """Pad serialized JSON payload to a fixed-size bucket.""" + raw_len = len(raw_json_bytes) + for bucket in _TRANSPORT_PAD_BUCKETS: + if raw_len <= bucket: + return raw_json_bytes + (b" " * (bucket - raw_len)) + target = (((raw_len - 1) // _TRANSPORT_PAD_BUCKETS[-1]) + 1) * _TRANSPORT_PAD_BUCKETS[-1] + return raw_json_bytes + (b" " * (target - raw_len)) + +# ─── Message Envelope ────────────────────────────────────────────────────── + + +class Priority(str, Enum): + EMERGENCY = "emergency" # SOS — broadcast on ALL transports simultaneously + HIGH = "high" # Time-sensitive — prefer fastest available + NORMAL = "normal" # Standard routing — optimize for efficiency + LOW = "low" # Batch-able — wait for optimal conditions + + +class PayloadType(str, Enum): + TEXT = "text" # Short text message (< 200 bytes ideal for LoRa) + POSITION = "position" # GPS coordinates + metadata + TELEMETRY = "telemetry" # Sensor data, battery, environment + FILE = "file" # Binary payload — requires high-bandwidth transport + COMMAND = "command" # Control message (channel join, ack, etc.) + + +@dataclass +class MeshEnvelope: + """Canonical message format that all transports share. + + Every message in the system is wrapped in this envelope regardless of + which transport carries it. This is the "lingua franca" of the mesh. + """ + + # Identity + sender_id: str # Node ID or callsign of sender + destination: str # Target node ID, callsign, or "broadcast" + channel: str = "LongFast" # Channel name (LongFast, Shadowbroker, etc.) + + # Routing metadata + priority: Priority = Priority.NORMAL + payload_type: PayloadType = PayloadType.TEXT + ttl: int = 3 # Max hops before discard + trust_tier: str = "public_degraded" # public_degraded | private_transitional | private_strong + + # Payload + payload: str = "" # The actual message content + payload_bytes: int = 0 # Computed size for routing decisions + + # Provenance + message_id: str = "" # Unique ID (generated if empty) + timestamp: float = 0.0 # Unix timestamp (generated if 0) + signature: str = "" # Integrity-only hash, not a cryptographic authentication signature + + # Retention + ephemeral: bool = False # If True, auto-purge after 24h + + # Routing result (filled by router) + routed_via: str = "" # Which transport was used + route_reason: str = "" # Why this transport was chosen + + def __post_init__(self): + if not self.message_id: + self.message_id = secrets.token_hex(8) + if not self.timestamp: + self.timestamp = time.time() + if not self.payload_bytes: + self.payload_bytes = len(self.payload.encode("utf-8")) + if not self.signature: + h = hashlib.sha256( + f"{self.sender_id}:{self.destination}:{self.payload}:{self.timestamp}".encode() + ) + self.signature = h.hexdigest()[:16] + + def to_dict(self) -> dict: + return asdict(self) + + +# ─── Transport Adapters ──────────────────────────────────────────────────── + + +class TransportResult: + """Result of a transport send attempt.""" + + def __init__(self, ok: bool, transport: str, detail: str = ""): + self.ok = ok + self.transport = transport + self.detail = detail + + def to_dict(self) -> dict: + return {"ok": self.ok, "transport": self.transport, "detail": self.detail} + + +def _private_transport_outcomes(results: list[TransportResult]) -> list[dict[str, object]]: + return [{"transport": result.transport, "ok": bool(result.ok)} for result in results] + + +class APRSTransport: + """APRS-IS transport — sends text messages to ham radio callsigns.""" + + NAME = "aprs" + MAX_PAYLOAD = 67 # APRS message length limit + + def can_reach(self, envelope: MeshEnvelope) -> bool: + """APRS can reach targets that look like ham callsigns.""" + dest = envelope.destination.upper() + # Ham callsigns: 1-2 letters + digit + 1-3 letters, optional -SSID + if dest == "broadcast": + return False # APRS doesn't support broadcast to all + # Simple heuristic: contains a digit and is short + return ( + any(c.isdigit() for c in dest) + and len(dest.split("-")[0]) <= 6 + and envelope.payload_bytes <= self.MAX_PAYLOAD + ) + + def send(self, envelope: MeshEnvelope, credentials: dict) -> TransportResult: + """Send via APRS-IS. Requires callsign + passcode in credentials.""" + from services.sigint_bridge import send_aprs_message + + callsign = credentials.get("aprs_callsign", "") + passcode = credentials.get("aprs_passcode", "") + if not callsign or not passcode: + return TransportResult(False, self.NAME, "APRS requires callsign + passcode") + + result = send_aprs_message(callsign, passcode, envelope.destination, envelope.payload) + return TransportResult(result["ok"], self.NAME, result["detail"]) + + +class MeshtasticTransport: + """Meshtastic MQTT transport — publishes messages to LoRa mesh via MQTT broker.""" + + NAME = "meshtastic" + MAX_PAYLOAD = 200 # LoRa practical payload limit + BROKER = "mqtt.meshtastic.org" + PORT = 1883 + + @staticmethod + def _mqtt_creds() -> tuple[str, str]: + try: + from services.config import get_settings + + s = get_settings() + return ( + str(s.MESH_MQTT_USER or "meshdev"), + str(s.MESH_MQTT_PASS or "large4cats"), + ) + except Exception: + return ("meshdev", "large4cats") + + def can_reach(self, envelope: MeshEnvelope) -> bool: + """Meshtastic can reach mesh nodes and supports broadcast.""" + # Meshtastic can broadcast to a channel or DM a node ID + return envelope.payload_bytes <= self.MAX_PAYLOAD + + # Default LongFast PSK (firmware-hardcoded for PSK=0x01) + DEFAULT_KEY = bytes( + [ + 0xD4, + 0xF1, + 0xBB, + 0x3A, + 0x20, + 0x29, + 0x07, + 0x59, + 0xF0, + 0xBC, + 0xFF, + 0xAB, + 0xCF, + 0x4E, + 0x69, + 0x01, + ] + ) + + @staticmethod + def _stable_node_id(sender_id: str) -> int: + """Derive a stable 32-bit node id from sender_id.""" + digest = hashlib.sha256(sender_id.encode("utf-8")).digest() + return int.from_bytes(digest[:4], "big") + + @staticmethod + def mesh_address_for_sender(sender_id: str) -> str: + """Return the synthetic public mesh address used for MQTT-originated sends.""" + return f"!{MeshtasticTransport._stable_node_id(sender_id):08x}" + + @staticmethod + def _parse_node_id(destination: str) -> Optional[int]: + """Parse a Meshtastic-style node address like !a0cc7a80.""" + dest = (destination or "").strip().lower() + if dest.startswith("!"): + dest = dest[1:] + if len(dest) != 8 or any(c not in "0123456789abcdef" for c in dest): + return None + try: + return int(dest, 16) + except ValueError: + return None + + def send(self, envelope: MeshEnvelope, credentials: dict) -> TransportResult: + """Publish protobuf-encoded, AES-encrypted message to Meshtastic MQTT.""" + try: + import paho.mqtt.client as mqtt + import struct + import random + from meshtastic import mesh_pb2, mqtt_pb2, portnums_pb2 + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + except ImportError as e: + return TransportResult(False, self.NAME, f"Missing dependency: {e}") + + try: + raw_root = credentials.get("mesh_root") or credentials.get("mesh_region", "US") + region = normalize_root(str(raw_root or "US")) or "US" + channel = envelope.channel or "LongFast" + + # Build Data payload + data_msg = mesh_pb2.Data() + data_msg.portnum = portnums_pb2.PortNum.TEXT_MESSAGE_APP + data_msg.payload = envelope.payload.encode("utf-8") + plaintext = data_msg.SerializeToString() + + # Generate IDs + packet_id = random.randint(1, 0xFFFFFFFF) + from_node = self._stable_node_id(envelope.sender_id) + direct_node = self._parse_node_id(envelope.destination) + to_node = direct_node if direct_node is not None else 0xFFFFFFFF + + # Encrypt (AES-128-CTR) + nonce = struct.pack(" {target}]: {envelope.payload[:50]}") + return TransportResult( + True, + self.NAME, + ( + f"Published direct to !{to_node:08x} via {region}/{channel}" + if direct_node is not None + else f"Published to {region}/{channel} ({len(payload)}B protobuf)" + ), + ) + except Exception as e: + return TransportResult(False, self.NAME, f"MQTT error: {e}") + + +class _PeerPushTransportMixin: + def __init__(self): + self._peer_failures: dict[str, int] = {} + self._peer_cooldown_until: dict[str, float] = {} + self._consecutive_total_failures: int = 0 + + def _get_peers(self) -> list[str]: + if getattr(self, "NAME", "") == "tor_arti": + return authenticated_push_peer_urls(transport="onion") + return authenticated_push_peer_urls(transport="clearnet") + + def _is_peer_cooled_down(self, peer_url: str) -> bool: + expiry = self._peer_cooldown_until.get(peer_url, 0.0) + return time.time() < expiry + + def _record_peer_failure(self, peer_url: str): + from services.config import get_settings + + settings = get_settings() + self._peer_failures[peer_url] = self._peer_failures.get(peer_url, 0) + 1 + if self._peer_failures[peer_url] >= int(settings.MESH_RELAY_MAX_FAILURES or 3): + cooldown_s = int(settings.MESH_RELAY_FAILURE_COOLDOWN_S or 120) + self._peer_cooldown_until[peer_url] = time.time() + cooldown_s + logger.warning( + "Peer %s exceeded failure threshold — cooling down for %ss", + peer_url, + cooldown_s, + ) + + def _reset_peer_failures(self, peer_url: str): + self._peer_failures.pop(peer_url, None) + self._peer_cooldown_until.pop(peer_url, None) + + def _build_peer_push_request(self, envelope: MeshEnvelope, push_source: str) -> tuple[str, bytes]: + evt_dict = envelope.to_dict() + payload_candidate = envelope.payload + if isinstance(payload_candidate, str): + try: + decoded = json.loads(payload_candidate) + except Exception: + decoded = None + if isinstance(decoded, dict) and decoded.get("event_type"): + evt_dict = decoded + + if evt_dict.get("event_type") == "gate_message": + from services.mesh.mesh_hashchain import build_gate_wire_ref + + payload_info = evt_dict.get("payload") if isinstance(evt_dict.get("payload"), dict) else {} + gate_id = str(payload_info.get("gate", "") or "").strip().lower() + safe_evt = { + "event_type": "gate_message", + "timestamp": evt_dict.get("timestamp", 0), + "payload": { + "ciphertext": str(payload_info.get("ciphertext", "") or ""), + "format": str(payload_info.get("format", "") or ""), + }, + } + gate_ref = build_gate_wire_ref(gate_id, safe_evt) + if not gate_ref: + raise ValueError("private gate forwarding requires MESH_PEER_PUSH_SECRET") + safe_evt["payload"]["gate_ref"] = gate_ref + nonce = str(payload_info.get("nonce", "") or "") + sender_ref = str(payload_info.get("sender_ref", "") or "") + epoch = int(payload_info.get("epoch", 0) or 0) + if nonce: + safe_evt["payload"]["nonce"] = nonce + if sender_ref: + safe_evt["payload"]["sender_ref"] = sender_ref + if epoch > 0: + safe_evt["payload"]["epoch"] = epoch + for field_name in ( + "event_id", + "node_id", + "sequence", + "signature", + "public_key", + "public_key_algo", + "protocol_version", + ): + value = evt_dict.get(field_name, "") + if value not in ("", None): + safe_evt[field_name] = value + payload = {"events": [safe_evt], "push_source": push_source} + return "/api/mesh/gate/peer-push", _pad_transport_payload( + json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + + payload = {"events": [evt_dict], "push_source": push_source} + return "/api/mesh/infonet/peer-push", _pad_transport_payload( + json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + ) + + +class InternetTransport(_PeerPushTransportMixin): + """Clearnet relay transport — pushes events to peers over plain HTTPS/HTTP.""" + + NAME = "internet" + + def __init__(self): + super().__init__() + + def can_reach(self, envelope: MeshEnvelope) -> bool: + return bool(self._get_peers()) + + def send(self, envelope: MeshEnvelope, credentials: dict) -> TransportResult: + import requests as _requests + from services.config import get_settings + + settings = get_settings() + peers = self._get_peers() + if not peers: + return TransportResult(False, self.NAME, "No relay peers configured") + + timeout = int(settings.MESH_RELAY_PUSH_TIMEOUT_S or 10) + try: + endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME) + except ValueError as exc: + return TransportResult(False, self.NAME, str(exc)) + secret = str(settings.MESH_PEER_PUSH_SECRET or "").strip() + + delivered = 0 + last_error = "" + for peer_url in peers: + if self._is_peer_cooled_down(peer_url): + continue + try: + normalized_peer_url = normalize_peer_url(peer_url) + headers = {"Content-Type": "application/json"} + if secret: + peer_key = _derive_peer_key(secret, normalized_peer_url) + if not peer_key: + raise ValueError("invalid peer URL for HMAC derivation") + headers["X-Peer-Url"] = normalized_peer_url + headers["X-Peer-HMAC"] = hmac.new( + peer_key, + padded, + hashlib.sha256, + ).hexdigest() + url = f"{peer_url}{endpoint_path}" + resp = _requests.post( + url, + data=padded, + timeout=timeout, + headers=headers, + ) + ok = resp.status_code == 200 + logger.info( + "TRANSPORT_AUDIT_PEER peer=%s transport=%s ok=%s detail=%s", + _peer_audit_label(peer_url), + self.NAME, + ok, + f"HTTP {resp.status_code}", + ) + if ok: + self._reset_peer_failures(peer_url) + delivered += 1 + else: + last_error = f"{peer_url}: HTTP {resp.status_code}" + self._record_peer_failure(peer_url) + except Exception as exc: + last_error = f"{peer_url}: {type(exc).__name__}" + logger.info( + "TRANSPORT_AUDIT_PEER peer=%s transport=%s ok=%s detail=%s", + _peer_audit_label(peer_url), + self.NAME, + False, + type(exc).__name__, + ) + self._record_peer_failure(peer_url) + + if delivered > 0: + self._consecutive_total_failures = 0 + return TransportResult( + True, self.NAME, f"Delivered to {delivered}/{len(peers)} peers via clearnet" + ) + + self._consecutive_total_failures += 1 + return TransportResult(False, self.NAME, f"All peers failed — last: {last_error}") + + +class TorArtiTransport(_PeerPushTransportMixin): + """Tor/Arti transport — forwards peer pushes through the local SOCKS5 proxy.""" + + NAME = "tor_arti" + + def __init__(self): + super().__init__() + + def can_reach(self, envelope: MeshEnvelope) -> bool: + from services.config import get_settings + from services.wormhole_supervisor import _check_arti_ready + + settings = get_settings() + return bool(settings.MESH_ARTI_ENABLED) and _check_arti_ready() and bool(self._get_peers()) + + def send(self, envelope: MeshEnvelope, credentials: dict) -> TransportResult: + import requests as _requests + from services.config import get_settings + + settings = get_settings() + peers = self._get_peers() + if not peers: + return TransportResult(False, self.NAME, "No relay peers configured") + + socks_port = int(settings.MESH_ARTI_SOCKS_PORT or 9050) + timeout = int(settings.MESH_RELAY_PUSH_TIMEOUT_S or 10) + proxy = f"socks5h://127.0.0.1:{socks_port}" + proxies = {"http": proxy, "https": proxy} + + try: + endpoint_path, padded = self._build_peer_push_request(envelope, self.NAME) + except ValueError as exc: + return TransportResult(False, self.NAME, str(exc)) + secret = str(settings.MESH_PEER_PUSH_SECRET or "").strip() + + delivered = 0 + last_error = "" + for peer_url in peers: + if self._is_peer_cooled_down(peer_url): + continue + try: + normalized_peer_url = normalize_peer_url(peer_url) + headers = {"Content-Type": "application/json"} + if secret: + peer_key = _derive_peer_key(secret, normalized_peer_url) + if not peer_key: + raise ValueError("invalid peer URL for HMAC derivation") + headers["X-Peer-Url"] = normalized_peer_url + headers["X-Peer-HMAC"] = hmac.new( + peer_key, + padded, + hashlib.sha256, + ).hexdigest() + url = f"{peer_url}{endpoint_path}" + resp = _requests.post( + url, + data=padded, + proxies=proxies, + timeout=timeout, + headers=headers, + ) + ok = resp.status_code == 200 + logger.info( + "TRANSPORT_AUDIT_PEER peer=%s transport=%s ok=%s detail=%s", + _peer_audit_label(peer_url), + self.NAME, + ok, + f"HTTP {resp.status_code}", + ) + if ok: + self._reset_peer_failures(peer_url) + delivered += 1 + else: + last_error = f"{peer_url}: HTTP {resp.status_code}" + self._record_peer_failure(peer_url) + except Exception as exc: + last_error = f"{peer_url}: {type(exc).__name__}" + logger.info( + "TRANSPORT_AUDIT_PEER peer=%s transport=%s ok=%s detail=%s", + _peer_audit_label(peer_url), + self.NAME, + False, + type(exc).__name__, + ) + self._record_peer_failure(peer_url) + + if delivered > 0: + self._consecutive_total_failures = 0 + return TransportResult(True, self.NAME, f"Delivered to {delivered}/{len(peers)} peers via Tor") + + self._consecutive_total_failures += 1 + if self._consecutive_total_failures >= int(settings.MESH_RELAY_MAX_FAILURES or 3): + logger.warning( + "TRANSPORT_DEGRADED: tor_arti has failed %d consecutive sends — will re-check on next supervisor refresh", + self._consecutive_total_failures, + ) + return TransportResult(False, self.NAME, f"All peers failed — last: {last_error}") + + +# ─── Conditional Gate Router ─────────────────────────────────────────────── + + +class CircuitBreaker: + """Automatic RF safety valve — prevents flooding external radio networks. + + Tracks outbound message counts per transport per 10-minute window. + Soft limit: log warning, reject low-priority sends. + Hard limit: disable transport entirely for a cooldown period. + """ + + def __init__( + self, + transport_name: str, + soft_limit: int, + hard_limit: int, + cooldown_seconds: int = 1800, + window_seconds: int = 600, + ): + self.transport_name = transport_name + self.soft_limit = soft_limit + self.hard_limit = hard_limit + self.cooldown_seconds = cooldown_seconds + self.window_seconds = window_seconds + self.send_times: deque[float] = deque() + self.air_gapped_until: float = 0.0 + + def _prune_window(self): + """Remove timestamps older than the sliding window.""" + cutoff = time.time() - self.window_seconds + while self.send_times and self.send_times[0] < cutoff: + self.send_times.popleft() + + def is_air_gapped(self) -> bool: + """Check if transport is currently disabled.""" + if self.air_gapped_until and time.time() < self.air_gapped_until: + return True + if self.air_gapped_until and time.time() >= self.air_gapped_until: + self.air_gapped_until = 0.0 # Cooldown expired + return False + + def check_and_record(self, priority: "Priority") -> tuple[bool, str]: + """Check if a send is allowed and record it. + + Returns (allowed: bool, reason: str). + """ + if self.is_air_gapped(): + remaining = int(self.air_gapped_until - time.time()) + return False, ( + f"{self.transport_name} CIRCUIT BREAKER: RF injection suspended " + f"({remaining}s remaining) — too many outbound messages" + ) + + self._prune_window() + count = len(self.send_times) + + # Hard limit → air-gap the transport + if count >= self.hard_limit: + self.air_gapped_until = time.time() + self.cooldown_seconds + logger.warning( + f"CIRCUIT BREAKER [{self.transport_name}]: HARD LIMIT {self.hard_limit} reached — " + f"transport disabled for {self.cooldown_seconds}s" + ) + return False, ( + f"{self.transport_name} temporarily suspended (network protection, " + f"{self.cooldown_seconds}s cooldown). Message will be rerouted." + ) + + # Soft limit → reject non-emergency, non-high priority + if count >= self.soft_limit and priority not in (Priority.EMERGENCY, Priority.HIGH): + logger.warning( + f"CIRCUIT BREAKER [{self.transport_name}]: Soft limit {self.soft_limit} reached — " + f"rejecting low-priority send ({count}/{self.hard_limit})" + ) + return False, ( + f"{self.transport_name} approaching rate limit " + f"({count}/{self.hard_limit}). Only high-priority messages accepted." + ) + + # Allowed — record the send + self.send_times.append(time.time()) + return True, "" + + def get_status(self) -> dict: + """Return current circuit breaker status for diagnostics.""" + self._prune_window() + return { + "transport": self.transport_name, + "window_count": len(self.send_times), + "soft_limit": self.soft_limit, + "hard_limit": self.hard_limit, + "air_gapped": self.is_air_gapped(), + "air_gapped_remaining": ( + max(0, int(self.air_gapped_until - time.time())) if self.air_gapped_until else 0 + ), + } + + +class MeshRouter: + """Policy-driven router that picks the optimal transport for each message. + + Gate logic: + 1. EMERGENCY → blast on ALL available transports simultaneously + 2. Small text (< 67 chars) to APRS callsign → APRS-IS + 3. Small text (< 200 bytes) to mesh or broadcast → Meshtastic MQTT + 4. Large payload → Internet relay (future WiFi mesh / Reticulum) + 5. Fallback → try each transport in capability order + + Circuit breakers protect external radio networks from being flooded. + """ + + def __init__(self): + self.aprs = APRSTransport() + self.meshtastic = MeshtasticTransport() + self.tor_arti = TorArtiTransport() + self.internet = InternetTransport() + self.transports = [self.aprs, self.meshtastic, self.tor_arti, self.internet] + # Message log for audit trail / provenance + self.message_log: deque[dict] = deque(maxlen=500) + self._dedupe: dict[str, float] = {} + # Circuit breakers — protect external networks + self.breakers = { + "aprs": CircuitBreaker("APRS", soft_limit=20, hard_limit=50, cooldown_seconds=1800), + "meshtastic": CircuitBreaker( + "Meshtastic", soft_limit=60, hard_limit=150, cooldown_seconds=900 + ), + } + + def prune_message_log(self, now: float | None = None) -> None: + from services.config import get_settings + + ttl_s = int(getattr(get_settings(), "MESH_PRIVATE_LOG_TTL_S", 900) or 0) + if ttl_s <= 0 or not self.message_log: + return + cutoff = float(now if now is not None else time.time()) - float(ttl_s) + filtered: list[dict] = [] + changed = False + for entry in self.message_log: + tier_str = str((entry or {}).get("trust_tier", "") or "").strip().lower() + if tier_str.startswith("private_"): + ts = float((entry or {}).get("timestamp", 0) or 0.0) + if ts > 0 and ts < cutoff: + changed = True + continue + filtered.append(entry) + if changed: + self.message_log = deque(filtered, maxlen=self.message_log.maxlen) + + def _dedupe_key(self, envelope: MeshEnvelope) -> str: + base = f"{envelope.sender_id}:{envelope.destination}:{envelope.payload}" + return hashlib.sha256(base.encode("utf-8")).hexdigest() + + def _prune_dedupe(self, now: float): + cutoff = now - DEDUP_TTL_SECONDS + for key, ts in list(self._dedupe.items()): + if ts < cutoff: + del self._dedupe[key] + if len(self._dedupe) > DEDUP_MAX_ENTRIES: + # Drop oldest entries if we exceeded max + for key, _ in sorted(self._dedupe.items(), key=lambda kv: kv[1])[ + : len(self._dedupe) - DEDUP_MAX_ENTRIES + ]: + del self._dedupe[key] + + def _is_duplicate(self, envelope: MeshEnvelope) -> bool: + now = time.time() + self._prune_dedupe(now) + key = self._dedupe_key(envelope) + if key in self._dedupe: + return True + self._dedupe[key] = now + return False + + def route(self, envelope: MeshEnvelope, credentials: dict) -> list[TransportResult]: + """Route a message through the optimal transport(s). + + Returns list of TransportResult (multiple for EMERGENCY broadcast). + """ + results: list[TransportResult] = [] + private_tier = str(envelope.trust_tier or "public_degraded").strip().lower().startswith( + "private_" + ) + + if self._is_duplicate(envelope): + envelope.route_reason = "Duplicate suppressed (loop protection)" + results.append(TransportResult(False, "dedupe", "Duplicate message suppressed")) + self._log(envelope, results) + return results + + # ─── Gate 1: EMERGENCY → broadcast on ALL transports ─────────── + if envelope.priority == Priority.EMERGENCY: + envelope.route_reason = "EMERGENCY — broadcasting on all available transports" + tier_str = str(envelope.trust_tier or "public_degraded").strip().lower() + for transport in self.transports: + if private_tier and transport.NAME in {"aprs", "meshtastic"}: + continue + if tier_str == "private_strong" and transport.NAME == "internet": + continue + if transport.can_reach(envelope): + r = transport.send(envelope, credentials) + results.append(r) + if r.ok: + envelope.routed_via += f"{transport.NAME}," + self._log(envelope, results) + return results + + # ─── Gate 2: APRS callsign target → APRS-IS ─────────────────── + if not private_tier and self.aprs.can_reach(envelope): + # Check circuit breaker before sending + cb_ok, cb_reason = self.breakers["aprs"].check_and_record(envelope.priority) + if not cb_ok: + results.append(TransportResult(False, self.aprs.NAME, cb_reason)) + # Fall through to Gate 3 instead of failing + else: + envelope.route_reason = "Target is APRS callsign, payload fits APRS limit" + r = self.aprs.send(envelope, credentials) + if r.ok: + envelope.routed_via = self.aprs.NAME + results.append(r) + self._log(envelope, results) + return results + # APRS failed (no credentials?) — fall through to next gate + results.append(r) + + # ─── Gate 3: Small payload → Meshtastic LoRa ────────────────── + if not private_tier and self.meshtastic.can_reach(envelope): + # Check circuit breaker before sending + cb_ok, cb_reason = self.breakers["meshtastic"].check_and_record(envelope.priority) + if not cb_ok: + results.append(TransportResult(False, self.meshtastic.NAME, cb_reason)) + # Fall through to Gate 4 + else: + if self.meshtastic._parse_node_id(envelope.destination) is not None: + envelope.route_reason = ( + "Target is Meshtastic node ID, routing as public node-targeted message via Meshtastic MQTT" + ) + else: + envelope.route_reason = "Payload fits LoRa, routing via Meshtastic MQTT" + r = self.meshtastic.send(envelope, credentials) + if r.ok: + envelope.routed_via = self.meshtastic.NAME + results.append(r) + self._log(envelope, results) + return results + results.append(r) + + # ─── Gate 4: Large payload or fallback → Internet relay ─────── + tier_str = str(envelope.trust_tier or "public_degraded").strip().lower() + + if tier_str == "private_strong": + # private_strong MUST use Tor — no clearnet fallback + if self.tor_arti.can_reach(envelope): + envelope.route_reason = "PRIVATE_STRONG — Tor required, no clearnet fallback" + tor_result = self.tor_arti.send(envelope, credentials) + results.append(tor_result) + if tor_result.ok: + envelope.routed_via = self.tor_arti.NAME + self._log(envelope, results) + return results + envelope.route_reason = ( + "PRIVATE_STRONG — Tor unavailable or failed, refusing clearnet fallback" + ) + results.append( + TransportResult( + False, + "policy", + "private_strong requires Tor — clearnet fallback refused", + ) + ) + self._log(envelope, results) + return results + + elif private_tier: + # private_transitional — prefer Tor, but allow clearnet fallback + if self.tor_arti.can_reach(envelope): + envelope.route_reason = "PRIVATE payload prefers tor_arti when available" + tor_result = self.tor_arti.send(envelope, credentials) + results.append(tor_result) + if tor_result.ok: + envelope.routed_via = self.tor_arti.NAME + self._log(envelope, results) + return results + if _high_privacy_profile_blocks_clearnet_fallback(): + envelope.route_reason = ( + "HIGH PRIVACY profile refuses clearnet fallback for private traffic" + ) + results.append( + TransportResult( + False, + "policy", + "high privacy profile requires hidden/private transport — clearnet fallback refused", + ) + ) + self._log(envelope, results) + return results + + envelope.route_reason = ( + "Payload too large for radio or radio transports failed — internet relay" + ) + if private_tier: + logger.warning( + "[mesh] Transport degradation: message sent via clearnet, expected private transport" + ) + r = self.internet.send(envelope, credentials) + envelope.routed_via = self.internet.NAME + results.append(r) + self._log(envelope, results) + return results + + def _log(self, envelope: MeshEnvelope, results: list[TransportResult]): + """Record message in audit log for provenance tracking. + + Private-tier messages get redacted logs — no sender, destination, + signature, or payload preview. Only routing metadata is logged. + """ + tier_str = str(envelope.trust_tier or "public_degraded").strip().lower() + is_private = tier_str.startswith("private_") + + self.prune_message_log() + + entry = { + "priority": envelope.priority.value, + "routed_via": envelope.routed_via, + "route_reason": envelope.route_reason, + "timestamp": envelope.timestamp, + "trust_tier": tier_str, + } + if is_private: + entry["transport_outcomes"] = _private_transport_outcomes(results) + else: + entry["message_id"] = envelope.message_id + entry["channel"] = envelope.channel + entry["payload_type"] = envelope.payload_type.value + entry["payload_bytes"] = envelope.payload_bytes + entry["results"] = [r.to_dict() for r in results] + entry["sender"] = envelope.sender_id + entry["destination"] = envelope.destination + entry["payload_preview"] = envelope.payload[:50] + entry["signature"] = envelope.signature + + self.message_log.append(entry) + any_ok = any(r.ok for r in results) + level = "info" if any_ok else "warning" + if is_private: + getattr(logger, level)( + "TRANSPORT_AUDIT tier=%s transports=%s ok=%s reason=%s", + tier_str, + ",".join(r.transport for r in results), + ",".join(str(r.ok) for r in results), + envelope.route_reason, + ) + else: + getattr(logger, level)( + "TRANSPORT_AUDIT msg_id=%s tier=%s transports=%s ok=%s destination=%s reason=%s", + envelope.message_id, + tier_str, + ",".join(r.transport for r in results), + ",".join(str(r.ok) for r in results), + envelope.destination, + envelope.route_reason, + ) + + +# Module-level singleton +mesh_router = MeshRouter() diff --git a/backend/services/mesh/mesh_schema.py b/backend/services/mesh/mesh_schema.py new file mode 100644 index 00000000..467740a8 --- /dev/null +++ b/backend/services/mesh/mesh_schema.py @@ -0,0 +1,399 @@ +"""Central schema registry for mesh protocol events.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable + +from services.mesh.mesh_protocol import normalize_payload, PROTOCOL_VERSION, NETWORK_ID + + +def _safe_int(val, default=0): + try: + return int(val) + except (TypeError, ValueError): + return default + + +@dataclass(frozen=True) +class EventSchema: + event_type: str + required_fields: tuple[str, ...] + optional_fields: tuple[str, ...] + validate: Callable[[dict[str, Any]], tuple[bool, str]] + + def validate_payload(self, payload: dict[str, Any]) -> tuple[bool, str]: + return self.validate(payload) + + +def _require_fields(payload: dict[str, Any], fields: tuple[str, ...]) -> tuple[bool, str]: + for key in fields: + if key not in payload: + return False, f"Missing field: {key}" + return True, "ok" + + +def _validate_message(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields( + payload, ("message", "destination", "channel", "priority", "ephemeral") + ) + if not ok: + return ok, reason + if payload.get("priority") not in ("normal", "high", "emergency", "low"): + return False, "Invalid priority" + if not isinstance(payload.get("ephemeral"), bool): + return False, "ephemeral must be boolean" + return True, "ok" + + +def _validate_gate_message(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("gate", "ciphertext", "nonce", "sender_ref")) + if not ok: + return ok, reason + if "message" in payload: + return False, "plaintext gate message field is not allowed" + gate = str(payload.get("gate", "")).strip().lower() + if not gate: + return False, "gate cannot be empty" + if "epoch" in payload: + epoch = _safe_int(payload.get("epoch", 0) or 0, 0) + if epoch <= 0: + return False, "epoch must be a positive integer" + elif ( + not str(payload.get("ciphertext", "")).strip() + and not str(payload.get("nonce", "")).strip() + and not str(payload.get("sender_ref", "")).strip() + ): + return False, "epoch must be a positive integer" + if not str(payload.get("ciphertext", "")).strip(): + return False, "ciphertext cannot be empty" + if not str(payload.get("nonce", "")).strip(): + return False, "nonce cannot be empty" + if not str(payload.get("sender_ref", "")).strip(): + return False, "sender_ref cannot be empty" + payload_format = str(payload.get("format", "mls1") or "mls1").strip().lower() + if payload_format != "mls1": + return False, "Unsupported gate message format" + return True, "ok" + + +def _validate_vote(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("target_id", "vote", "gate")) + if not ok: + return ok, reason + if payload.get("vote") not in (-1, 1): + return False, "Invalid vote" + return True, "ok" + + +def _validate_gate_create(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("gate_id", "display_name", "rules")) + if not ok: + return ok, reason + if not isinstance(payload.get("rules"), dict): + return False, "rules must be an object" + return True, "ok" + + +def _validate_prediction(payload: dict[str, Any]) -> tuple[bool, str]: + return _require_fields(payload, ("market_title", "side", "stake_amount")) + + +def _validate_stake(payload: dict[str, Any]) -> tuple[bool, str]: + return _require_fields(payload, ("message_id", "poster_id", "side", "amount", "duration_days")) + + +def _validate_dm_block(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("blocked_id", "action")) + if not ok: + return ok, reason + if payload.get("action") not in ("block", "unblock"): + return False, "Invalid action" + return True, "ok" + + +def _validate_dm_key(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("dh_pub_key", "dh_algo", "timestamp")) + if not ok: + return ok, reason + if payload.get("dh_algo") not in ("X25519", "ECDH", "ECDH_P256"): + return False, "Invalid dh_algo" + return True, "ok" + + +def _validate_dm_message(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields( + payload, ("recipient_id", "delivery_class", "recipient_token", "ciphertext", "msg_id", "timestamp") + ) + if not ok: + return ok, reason + delivery_class = str(payload.get("delivery_class", "")).lower() + if delivery_class not in ("request", "shared"): + return False, "Invalid delivery_class" + if delivery_class == "shared" and not str(payload.get("recipient_token", "")).strip(): + return False, "recipient_token required for shared delivery" + dm_format = str(payload.get("format", "mls1") or "mls1").strip().lower() + if dm_format not in ("mls1", "dm1"): + return False, f"Unknown DM format: {dm_format}" + return True, "ok" + + +def _validate_mailbox_claims(claims: Any) -> tuple[bool, str]: + if not isinstance(claims, list) or not claims: + return False, "mailbox_claims must be a non-empty list" + for claim in claims: + if not isinstance(claim, dict): + return False, "mailbox_claims entries must be objects" + claim_type = str(claim.get("type", "")).lower() + if claim_type not in ("self", "requests", "shared"): + return False, "Invalid mailbox claim type" + if not str(claim.get("token", "")).strip(): + return False, f"{claim_type} mailbox claims require token" + return True, "ok" + + +def _validate_dm_poll(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("mailbox_claims", "timestamp", "nonce")) + if not ok: + return ok, reason + return _validate_mailbox_claims(payload.get("mailbox_claims")) + + +def _validate_dm_count(payload: dict[str, Any]) -> tuple[bool, str]: + return _validate_dm_poll(payload) + + +def _validate_key_rotate(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields( + payload, + ( + "old_node_id", + "old_public_key", + "old_public_key_algo", + "new_public_key", + "new_public_key_algo", + "timestamp", + "old_signature", + ), + ) + if not ok: + return ok, reason + return True, "ok" + + +def _validate_key_revoke(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields( + payload, + ( + "revoked_public_key", + "revoked_public_key_algo", + "revoked_at", + "grace_until", + "reason", + ), + ) + if not ok: + return ok, reason + revoked_at = _safe_int(payload.get("revoked_at", 0) or 0, 0) + grace_until = _safe_int(payload.get("grace_until", 0) or 0, 0) + if revoked_at <= 0: + return False, "revoked_at must be a positive timestamp" + if grace_until < revoked_at: + return False, "grace_until must be >= revoked_at" + return True, "ok" + + +def _validate_abuse_report(payload: dict[str, Any]) -> tuple[bool, str]: + ok, reason = _require_fields(payload, ("target_id", "reason")) + if not ok: + return ok, reason + if not str(payload.get("reason", "")).strip(): + return False, "reason cannot be empty" + return True, "ok" + + +SCHEMA_REGISTRY: dict[str, EventSchema] = { + "message": EventSchema( + event_type="message", + required_fields=("message", "destination", "channel", "priority", "ephemeral"), + optional_fields=(), + validate=_validate_message, + ), + "gate_message": EventSchema( + event_type="gate_message", + required_fields=("gate", "ciphertext", "nonce", "sender_ref"), + optional_fields=("format",), + validate=_validate_gate_message, + ), + "vote": EventSchema( + event_type="vote", + required_fields=("target_id", "vote", "gate"), + optional_fields=(), + validate=_validate_vote, + ), + "gate_create": EventSchema( + event_type="gate_create", + required_fields=("gate_id", "display_name", "rules"), + optional_fields=(), + validate=_validate_gate_create, + ), + "prediction": EventSchema( + event_type="prediction", + required_fields=("market_title", "side", "stake_amount"), + optional_fields=(), + validate=_validate_prediction, + ), + "stake": EventSchema( + event_type="stake", + required_fields=("message_id", "poster_id", "side", "amount", "duration_days"), + optional_fields=(), + validate=_validate_stake, + ), + "dm_block": EventSchema( + event_type="dm_block", + required_fields=("blocked_id", "action"), + optional_fields=(), + validate=_validate_dm_block, + ), + "dm_key": EventSchema( + event_type="dm_key", + required_fields=("dh_pub_key", "dh_algo", "timestamp"), + optional_fields=(), + validate=_validate_dm_key, + ), + "dm_message": EventSchema( + event_type="dm_message", + required_fields=("recipient_id", "delivery_class", "recipient_token", "ciphertext", "msg_id", "timestamp"), + optional_fields=(), + validate=_validate_dm_message, + ), + "dm_poll": EventSchema( + event_type="dm_poll", + required_fields=("mailbox_claims", "timestamp", "nonce"), + optional_fields=(), + validate=_validate_dm_poll, + ), + "dm_count": EventSchema( + event_type="dm_count", + required_fields=("mailbox_claims", "timestamp", "nonce"), + optional_fields=(), + validate=_validate_dm_count, + ), + "key_rotate": EventSchema( + event_type="key_rotate", + required_fields=( + "old_node_id", + "old_public_key", + "old_public_key_algo", + "new_public_key", + "new_public_key_algo", + "timestamp", + "old_signature", + ), + optional_fields=(), + validate=_validate_key_rotate, + ), + "key_revoke": EventSchema( + event_type="key_revoke", + required_fields=( + "revoked_public_key", + "revoked_public_key_algo", + "revoked_at", + "grace_until", + "reason", + ), + optional_fields=(), + validate=_validate_key_revoke, + ), + "abuse_report": EventSchema( + event_type="abuse_report", + required_fields=("target_id", "reason"), + optional_fields=("gate", "evidence"), + validate=_validate_abuse_report, + ), +} + + +PUBLIC_LEDGER_EVENT_TYPES: frozenset[str] = frozenset( + { + "message", + "vote", + "gate_create", + "gate_message", + "prediction", + "stake", + "key_rotate", + "key_revoke", + "abuse_report", + } +) + +_PUBLIC_LEDGER_FORBIDDEN_FIELDS: frozenset[str] = frozenset( + { + "ip", + "ip_address", + "origin_ip", + "source_ip", + "client_ip", + "host", + "hostname", + "origin", + "originator", + "originator_hint", + "routing_hint", + "route", + "route_hint", + "route_reason", + "routed_via", + "transport", + "transport_handle", + "transport_lock", + "recipient_id", + "recipient_token", + "delivery_class", + "mailbox_claims", + "dh_pub_key", + "sender_token", + } +) + + +def get_schema(event_type: str) -> EventSchema | None: + return SCHEMA_REGISTRY.get(event_type) + + +def validate_event_payload(event_type: str, payload: dict[str, Any]) -> tuple[bool, str]: + schema = get_schema(event_type) + if not schema: + return False, "Unknown event_type" + normalized = normalize_payload(event_type, payload) + if normalized != payload: + return False, "Payload is not normalized" + if event_type not in ("message", "gate_message") and "ephemeral" in payload: + return False, "ephemeral not allowed for this event type" + return schema.validate_payload(payload) + + +def validate_public_ledger_payload(event_type: str, payload: dict[str, Any]) -> tuple[bool, str]: + if event_type not in PUBLIC_LEDGER_EVENT_TYPES: + return False, f"{event_type} is not allowed on the public ledger" + forbidden = sorted( + key + for key in payload.keys() + if str(key or "").strip().lower() in _PUBLIC_LEDGER_FORBIDDEN_FIELDS + ) + if forbidden: + return False, f"public ledger payload contains forbidden fields: {', '.join(forbidden)}" + if event_type == "message": + destination = str(payload.get("destination", "") or "").strip().lower() + if destination and destination != "broadcast": + return False, "public ledger message destination must be broadcast" + return True, "ok" + + +def validate_protocol_fields(protocol_version: str, network_id: str) -> tuple[bool, str]: + if protocol_version != PROTOCOL_VERSION: + return False, "Unsupported protocol_version" + if network_id != NETWORK_ID: + return False, "network_id mismatch" + return True, "ok" diff --git a/backend/services/mesh/mesh_secure_storage.py b/backend/services/mesh/mesh_secure_storage.py new file mode 100644 index 00000000..cd02e9ff --- /dev/null +++ b/backend/services/mesh/mesh_secure_storage.py @@ -0,0 +1,577 @@ +"""Secure local storage helpers for Wormhole-owned state. + +Windows uses DPAPI to protect local key envelopes. Root secure-json payloads +still use a dedicated master key, while domain-scoped payloads now use +independent per-domain keys so compromise of one domain key does not +automatically collapse every other Wormhole compartment. Non-Windows platforms +can fall back to raw local key files only when tests are running or an +explicit development/CI opt-in is set until native keyrings are added in the +desktop phase. +""" + +from __future__ import annotations + +import base64 +import ctypes +import hashlib +import hmac +import json +import os +import re +import tempfile +import time +from pathlib import Path +from typing import Any, Callable, TypeVar + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +MASTER_KEY_FILE = DATA_DIR / "wormhole_secure_store.key" + +_ENVELOPE_KIND = "sb_secure_json" +_ENVELOPE_VERSION = 1 +_MASTER_KIND = "sb_secure_master_key" +_MASTER_VERSION = 1 +_DOMAIN_KEY_KIND = "sb_secure_domain_key" +_DOMAIN_KEY_VERSION = 1 +_MASTER_KEY_CACHE: tuple[str, bytes] | None = None +_DOMAIN_KEY_CACHE: dict[str, tuple[str, bytes]] = {} + +T = TypeVar("T") + + +class SecureStorageError(RuntimeError): + """Raised when secure local storage cannot be read or written safely.""" + + +def _atomic_write_text(target: Path, content: str, encoding: str = "utf-8") -> None: + """Write content atomically via temp file + os.replace().""" + parent = target.parent + parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=str(parent), suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding=encoding) as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + last_exc: Exception | None = None + for _ in range(5): + try: + os.replace(tmp_path, str(target)) + last_exc = None + break + except PermissionError as exc: + last_exc = exc + time.sleep(0.02) + if last_exc is not None: + raise last_exc + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _stable_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _envelope_aad(path: Path) -> bytes: + return f"shadowbroker|secure-json|v{_ENVELOPE_VERSION}|{path.name}".encode("utf-8") + + +def _master_aad() -> bytes: + return f"shadowbroker|master-key|v{_MASTER_VERSION}".encode("utf-8") + + +def _domain_key_aad(domain: str) -> bytes: + return f"shadowbroker|domain-key|v{_DOMAIN_KEY_VERSION}|{domain}".encode("utf-8") + + +def _storage_root(base_dir: str | Path | None = None) -> Path: + return Path(base_dir).resolve() if base_dir is not None else DATA_DIR.resolve() + + +def _domain_key_dir(base_dir: str | Path | None = None) -> Path: + return _storage_root(base_dir) / "_domain_keys" + + +def _normalize_domain_name(domain: str) -> str: + domain_name = str(domain or "").strip().lower() + if not domain_name: + raise SecureStorageError("domain name required for domain-scoped storage") + if not re.fullmatch(r"[a-z0-9_]+", domain_name): + raise SecureStorageError(f"invalid domain name: {domain_name!r}") + return domain_name + + +def _domain_aad(domain: str, filename: str) -> bytes: + return f"shadowbroker|domain-json|v{_ENVELOPE_VERSION}|{domain}|{filename}".encode("utf-8") + + +def _master_envelope_for_windows(protected_key: bytes, *, provider: str) -> dict[str, Any]: + return { + "kind": _MASTER_KIND, + "version": _MASTER_VERSION, + "provider": provider, + "protected_key": _b64(protected_key), + } + + +def _master_envelope_for_fallback(raw_key: bytes) -> dict[str, Any]: + return { + "kind": _MASTER_KIND, + "version": _MASTER_VERSION, + "provider": "raw", + "key": _b64(raw_key), + } + + +def _domain_key_envelope_for_windows( + domain: str, + protected_key: bytes, + *, + provider: str, +) -> dict[str, Any]: + return { + "kind": _DOMAIN_KEY_KIND, + "version": _DOMAIN_KEY_VERSION, + "provider": provider, + "domain": domain, + "protected_key": _b64(protected_key), + } + + +def _domain_key_envelope_for_fallback(domain: str, raw_key: bytes) -> dict[str, Any]: + return { + "kind": _DOMAIN_KEY_KIND, + "version": _DOMAIN_KEY_VERSION, + "provider": "raw", + "domain": domain, + "key": _b64(raw_key), + } + + +def _secure_envelope(path: Path, nonce: bytes, ciphertext: bytes) -> dict[str, Any]: + return { + "kind": _ENVELOPE_KIND, + "version": _ENVELOPE_VERSION, + "path": path.name, + "nonce": _b64(nonce), + "ciphertext": _b64(ciphertext), + } + + +def _is_secure_envelope(value: Any) -> bool: + return ( + isinstance(value, dict) + and str(value.get("kind", "") or "") == _ENVELOPE_KIND + and int(value.get("version", 0) or 0) == _ENVELOPE_VERSION + and "nonce" in value + and "ciphertext" in value + ) + + +def _is_windows() -> bool: + return os.name == "nt" + + +def _raw_fallback_allowed() -> bool: + if _is_windows(): + return False + if os.environ.get("PYTEST_CURRENT_TEST"): + return True + try: + from services.config import get_settings + + settings = get_settings() + if bool(getattr(settings, "MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK", False)): + return True + except Exception: + pass + return False + + +if _is_windows(): + from ctypes import wintypes + + class _DATA_BLOB(ctypes.Structure): + _fields_ = [("cbData", wintypes.DWORD), ("pbData", ctypes.POINTER(ctypes.c_byte))] + + + _crypt32 = ctypes.windll.crypt32 + _kernel32 = ctypes.windll.kernel32 + _CRYPTPROTECT_UI_FORBIDDEN = 0x1 + _CRYPTPROTECT_LOCAL_MACHINE = 0x4 + + _crypt32.CryptProtectData.argtypes = [ + ctypes.POINTER(_DATA_BLOB), + wintypes.LPCWSTR, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(_DATA_BLOB), + ] + _crypt32.CryptProtectData.restype = wintypes.BOOL + _crypt32.CryptUnprotectData.argtypes = [ + ctypes.POINTER(_DATA_BLOB), + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, + wintypes.DWORD, + ctypes.POINTER(_DATA_BLOB), + ] + _crypt32.CryptUnprotectData.restype = wintypes.BOOL + _kernel32.LocalFree.argtypes = [ctypes.c_void_p] + _kernel32.LocalFree.restype = ctypes.c_void_p + + + def _blob_from_bytes(data: bytes) -> tuple[_DATA_BLOB, ctypes.Array[ctypes.c_char]]: + buf = ctypes.create_string_buffer(data, len(data)) + blob = _DATA_BLOB(len(data), ctypes.cast(buf, ctypes.POINTER(ctypes.c_byte))) + return blob, buf + + + def _bytes_from_blob(blob: _DATA_BLOB) -> bytes: + return ctypes.string_at(blob.pbData, blob.cbData) + + + def _dpapi_protect(data: bytes, *, machine_scope: bool) -> bytes: + in_blob, in_buf = _blob_from_bytes(data) + out_blob = _DATA_BLOB() + flags = _CRYPTPROTECT_UI_FORBIDDEN + if machine_scope: + flags |= _CRYPTPROTECT_LOCAL_MACHINE + if not _crypt32.CryptProtectData( + ctypes.byref(in_blob), + "ShadowBroker Wormhole", + None, + None, + None, + flags, + ctypes.byref(out_blob), + ): + raise ctypes.WinError() + try: + _ = in_buf # Keep the backing buffer alive for the API call. + return _bytes_from_blob(out_blob) + finally: + if out_blob.pbData: + _kernel32.LocalFree(out_blob.pbData) + + + def _dpapi_unprotect(data: bytes) -> bytes: + in_blob, in_buf = _blob_from_bytes(data) + out_blob = _DATA_BLOB() + if not _crypt32.CryptUnprotectData( + ctypes.byref(in_blob), + None, + None, + None, + None, + _CRYPTPROTECT_UI_FORBIDDEN, + ctypes.byref(out_blob), + ): + raise ctypes.WinError() + try: + _ = in_buf # Keep the backing buffer alive for the API call. + return _bytes_from_blob(out_blob) + finally: + if out_blob.pbData: + _kernel32.LocalFree(out_blob.pbData) + + +else: + + def _dpapi_protect(data: bytes, *, machine_scope: bool) -> bytes: + raise SecureStorageError("DPAPI is only available on Windows") + + + def _dpapi_unprotect(data: bytes) -> bytes: + raise SecureStorageError("DPAPI is only available on Windows") + + +def _load_master_key() -> bytes: + global _MASTER_KEY_CACHE + DATA_DIR.mkdir(parents=True, exist_ok=True) + cache_key = str(MASTER_KEY_FILE.resolve()) + if _MASTER_KEY_CACHE and _MASTER_KEY_CACHE[0] == cache_key: + return _MASTER_KEY_CACHE[1] + if not MASTER_KEY_FILE.exists(): + raw_key = os.urandom(32) + if _is_windows(): + envelope = _master_envelope_for_windows( + _dpapi_protect(raw_key, machine_scope=True), + provider="dpapi-machine", + ) + else: + if not _raw_fallback_allowed(): + raise SecureStorageError( + "Non-Windows secure storage requires a native keyring or explicit raw fallback opt-in" + ) + envelope = _master_envelope_for_fallback(raw_key) + _atomic_write_text(MASTER_KEY_FILE, json.dumps(envelope, indent=2), encoding="utf-8") + _MASTER_KEY_CACHE = (cache_key, raw_key) + return raw_key + + try: + payload = json.loads(MASTER_KEY_FILE.read_text(encoding="utf-8")) + except Exception as exc: + raise SecureStorageError(f"Failed to load secure storage master key: {exc}") from exc + if not isinstance(payload, dict) or payload.get("kind") != _MASTER_KIND: + raise SecureStorageError("Malformed secure storage master key envelope") + provider = str(payload.get("provider", "") or "").lower() + if provider in {"dpapi", "dpapi-user", "dpapi-machine"}: + try: + raw_key = _dpapi_unprotect(_unb64(payload.get("protected_key"))) + _MASTER_KEY_CACHE = (cache_key, raw_key) + return raw_key + except Exception as exc: + raise SecureStorageError(f"Failed to unwrap DPAPI master key: {exc}") from exc + if provider == "raw": + if not _raw_fallback_allowed(): + raise SecureStorageError( + "Raw secure-storage envelopes are disabled outside debug/test unless explicitly opted in" + ) + raw_key = _unb64(payload.get("key")) + _MASTER_KEY_CACHE = (cache_key, raw_key) + return raw_key + raise SecureStorageError(f"Unsupported secure storage provider: {provider}") + + +def _domain_key_file(domain: str, *, base_dir: str | Path | None = None) -> Path: + domain_name = _normalize_domain_name(domain) + return (_domain_key_dir(base_dir) / f"{domain_name}.key").resolve() + + +def _load_domain_key( + domain: str, + *, + create_if_missing: bool = True, + base_dir: str | Path | None = None, +) -> bytes: + domain_name = _normalize_domain_name(domain) + root = _storage_root(base_dir) + root.mkdir(parents=True, exist_ok=True) + key_file = _domain_key_file(domain_name, base_dir=base_dir) + cache_key = str(key_file) + cache_slot = f"{root}::{domain_name}" + cached = _DOMAIN_KEY_CACHE.get(cache_slot) + if cached and cached[0] == cache_key: + return cached[1] + if not key_file.exists(): + if not create_if_missing: + raise SecureStorageError(f"Domain key not found for {domain_name}") + raw_key = os.urandom(32) + if _is_windows(): + envelope = _domain_key_envelope_for_windows( + domain_name, + _dpapi_protect(raw_key, machine_scope=True), + provider="dpapi-machine", + ) + else: + if not _raw_fallback_allowed(): + raise SecureStorageError( + "Non-Windows secure storage requires a native keyring or explicit raw fallback opt-in" + ) + envelope = _domain_key_envelope_for_fallback(domain_name, raw_key) + _atomic_write_text(key_file, json.dumps(envelope, indent=2), encoding="utf-8") + _DOMAIN_KEY_CACHE[cache_slot] = (cache_key, raw_key) + return raw_key + + try: + payload = json.loads(key_file.read_text(encoding="utf-8")) + except Exception as exc: + raise SecureStorageError(f"Failed to load domain key for {domain_name}: {exc}") from exc + if not isinstance(payload, dict) or payload.get("kind") != _DOMAIN_KEY_KIND: + raise SecureStorageError(f"Malformed domain key envelope for {domain_name}") + if str(payload.get("domain", "") or "").strip().lower() != domain_name: + raise SecureStorageError(f"Domain key envelope mismatch for {domain_name}") + provider = str(payload.get("provider", "") or "").lower() + if provider in {"dpapi", "dpapi-user", "dpapi-machine"}: + try: + raw_key = _dpapi_unprotect(_unb64(payload.get("protected_key"))) + _DOMAIN_KEY_CACHE[cache_slot] = (cache_key, raw_key) + return raw_key + except Exception as exc: + raise SecureStorageError(f"Failed to unwrap domain key for {domain_name}: {exc}") from exc + if provider == "raw": + if not _raw_fallback_allowed(): + raise SecureStorageError( + "Raw secure-storage envelopes are disabled outside debug/test unless explicitly opted in" + ) + raw_key = _unb64(payload.get("key")) + _DOMAIN_KEY_CACHE[cache_slot] = (cache_key, raw_key) + return raw_key + raise SecureStorageError(f"Unsupported domain key provider for {domain_name}: {provider}") + + +def _derive_legacy_domain_key(domain: str) -> bytes: + domain_name = _normalize_domain_name(domain) + return hmac.new( + _load_master_key(), + domain_name.encode("utf-8"), + hashlib.sha256, + ).digest() + + +def _domain_file_path(domain: str, filename: str, *, base_dir: str | Path | None = None) -> Path: + domain_name = _normalize_domain_name(domain) + file_name = str(filename or "").strip() + if not file_name: + raise SecureStorageError("filename required for domain-scoped storage") + if not re.fullmatch(r"[a-z0-9_.]+", file_name): + raise SecureStorageError(f"invalid filename: {file_name!r}") + root = _storage_root(base_dir) + resolved = (root / domain_name / file_name).resolve() + if not str(resolved).startswith(str(root)): + raise SecureStorageError("domain storage path traversal rejected") + return resolved + + +def write_secure_json(path: str | Path, payload: Any) -> None: + file_path = Path(path) + file_path.parent.mkdir(parents=True, exist_ok=True) + master_key = _load_master_key() + nonce = os.urandom(12) + ciphertext = AESGCM(master_key).encrypt(nonce, _stable_json(payload), _envelope_aad(file_path)) + envelope = _secure_envelope(file_path, nonce, ciphertext) + _atomic_write_text(file_path, json.dumps(envelope, indent=2), encoding="utf-8") + + +def read_secure_json(path: str | Path, default_factory: Callable[[], T]) -> T: + file_path = Path(path) + if not file_path.exists(): + return default_factory() + + try: + raw = json.loads(file_path.read_text(encoding="utf-8")) + except Exception as exc: + raise SecureStorageError(f"Failed to parse secure JSON {file_path.name}: {exc}") from exc + + if _is_secure_envelope(raw): + master_key = _load_master_key() + try: + plaintext = AESGCM(master_key).decrypt( + _unb64(raw.get("nonce")), + _unb64(raw.get("ciphertext")), + _envelope_aad(file_path), + ) + except Exception as exc: + raise SecureStorageError(f"Failed to decrypt secure JSON {file_path.name}: {exc}") from exc + try: + return json.loads(plaintext.decode("utf-8")) + except Exception as exc: + raise SecureStorageError( + f"Failed to decode secure JSON payload {file_path.name}: {exc}" + ) from exc + + # Legacy plaintext JSON: migrate in place on first successful read. + migrated = raw if isinstance(raw, (dict, list)) else default_factory() + write_secure_json(file_path, migrated) + return migrated + + +def write_domain_json( + domain: str, + filename: str, + payload: Any, + *, + base_dir: str | Path | None = None, +) -> Path: + file_path = _domain_file_path(domain, filename, base_dir=base_dir) + file_path.parent.mkdir(parents=True, exist_ok=True) + nonce = os.urandom(12) + domain_name = _normalize_domain_name(domain) + ciphertext = AESGCM(_load_domain_key(domain_name, base_dir=base_dir)).encrypt( + nonce, + _stable_json(payload), + _domain_aad(domain_name, file_path.name), + ) + envelope = _secure_envelope(file_path, nonce, ciphertext) + _atomic_write_text(file_path, json.dumps(envelope, indent=2), encoding="utf-8") + return file_path + + +def read_domain_json( + domain: str, + filename: str, + default_factory: Callable[[], T], + *, + base_dir: str | Path | None = None, +) -> T: + file_path = _domain_file_path(domain, filename, base_dir=base_dir) + domain_name = _normalize_domain_name(domain) + if not file_path.exists(): + return default_factory() + try: + raw = json.loads(file_path.read_text(encoding="utf-8")) + except Exception as exc: + raise SecureStorageError(f"Failed to parse domain JSON {file_path.name}: {exc}") from exc + + if _is_secure_envelope(raw): + aad = _domain_aad(domain_name, file_path.name) + plaintext: bytes | None = None + used_legacy_key = False + used_master_key = False + try: + current_key = _load_domain_key(domain_name, create_if_missing=False, base_dir=base_dir) + except SecureStorageError: + current_key = None + if current_key is not None: + try: + plaintext = AESGCM(current_key).decrypt( + _unb64(raw.get("nonce")), + _unb64(raw.get("ciphertext")), + aad, + ) + except Exception: + plaintext = None + if plaintext is None: + try: + plaintext = AESGCM(_derive_legacy_domain_key(domain_name)).decrypt( + _unb64(raw.get("nonce")), + _unb64(raw.get("ciphertext")), + aad, + ) + used_legacy_key = True + except Exception as exc: + try: + plaintext = AESGCM(_load_master_key()).decrypt( + _unb64(raw.get("nonce")), + _unb64(raw.get("ciphertext")), + _envelope_aad(file_path), + ) + used_master_key = True + except Exception: + raise SecureStorageError( + f"Failed to decrypt domain JSON {file_path.name}: {exc}" + ) from exc + try: + decoded = json.loads(plaintext.decode("utf-8")) + except Exception as exc: + raise SecureStorageError( + f"Failed to decode domain JSON payload {file_path.name}: {exc}" + ) from exc + if used_legacy_key or used_master_key: + write_domain_json(domain_name, file_path.name, decoded, base_dir=base_dir) + return decoded + + migrated = raw if isinstance(raw, (dict, list)) else default_factory() + write_domain_json(domain_name, file_path.name, migrated, base_dir=base_dir) + return migrated diff --git a/backend/services/mesh/mesh_wormhole_contacts.py b/backend/services/mesh/mesh_wormhole_contacts.py new file mode 100644 index 00000000..db0f1f95 --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_contacts.py @@ -0,0 +1,225 @@ +"""Wormhole-owned DM contact and alias graph state.""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from services.mesh.mesh_secure_storage import read_secure_json, write_secure_json + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +CONTACTS_FILE = DATA_DIR / "wormhole_dm_contacts.json" + + +def _default_contact() -> dict[str, Any]: + return { + "alias": "", + "blocked": False, + "dhPubKey": "", + "dhAlgo": "", + "sharedAlias": "", + "previousSharedAliases": [], + "pendingSharedAlias": "", + "sharedAliasGraceUntil": 0, + "sharedAliasRotatedAt": 0, + "verify_inband": False, + "verify_registry": False, + "verified": False, + "verify_mismatch": False, + "verified_at": 0, + "remotePrekeyFingerprint": "", + "remotePrekeyObservedFingerprint": "", + "remotePrekeyPinnedAt": 0, + "remotePrekeyLastSeenAt": 0, + "remotePrekeySequence": 0, + "remotePrekeySignedAt": 0, + "remotePrekeyMismatch": False, + "witness_count": 0, + "witness_checked_at": 0, + "vouch_count": 0, + "vouch_checked_at": 0, + "updated_at": 0, + } + + +def _normalize_contact(value: dict[str, Any] | None) -> dict[str, Any]: + current = _default_contact() + current.update(value or {}) + current["alias"] = str(current.get("alias", "") or "") + current["blocked"] = bool(current.get("blocked")) + current["dhPubKey"] = str(current.get("dhPubKey", "") or "") + current["dhAlgo"] = str(current.get("dhAlgo", "") or "") + current["sharedAlias"] = str(current.get("sharedAlias", "") or "") + current["previousSharedAliases"] = [ + str(item or "") for item in list(current.get("previousSharedAliases") or []) if str(item or "").strip() + ][-8:] + current["pendingSharedAlias"] = str(current.get("pendingSharedAlias", "") or "") + current["remotePrekeyFingerprint"] = str(current.get("remotePrekeyFingerprint", "") or "") + current["remotePrekeyObservedFingerprint"] = str(current.get("remotePrekeyObservedFingerprint", "") or "") + for key in ( + "sharedAliasGraceUntil", + "sharedAliasRotatedAt", + "verified_at", + "remotePrekeyPinnedAt", + "remotePrekeyLastSeenAt", + "remotePrekeySequence", + "remotePrekeySignedAt", + "witness_count", + "witness_checked_at", + "vouch_count", + "vouch_checked_at", + "updated_at", + ): + current[key] = int(current.get(key, 0) or 0) + for key in ( + "verify_inband", + "verify_registry", + "verified", + "verify_mismatch", + "remotePrekeyMismatch", + ): + current[key] = bool(current.get(key)) + return current + + +def _merge_alias_history(*aliases: str, limit: int = 8) -> list[str]: + unique: set[str] = set() + ordered: list[str] = [] + for alias in aliases: + value = str(alias or "").strip() + if not value or value in unique: + continue + unique.add(value) + ordered.append(value) + if len(ordered) >= limit: + break + return ordered + + +def _promote_pending_alias_if_due(contact: dict[str, Any]) -> tuple[dict[str, Any], bool]: + current = _normalize_contact(contact) + pending = str(current.get("pendingSharedAlias", "") or "").strip() + grace_until = int(current.get("sharedAliasGraceUntil", 0) or 0) + if not pending or grace_until <= 0 or grace_until > int(time.time() * 1000): + return current, False + active = str(current.get("sharedAlias", "") or "").strip() + promoted = dict(current) + promoted["sharedAlias"] = pending or active + promoted["pendingSharedAlias"] = "" + promoted["sharedAliasGraceUntil"] = 0 + promoted["sharedAliasRotatedAt"] = int(time.time() * 1000) + promoted["previousSharedAliases"] = _merge_alias_history( + active, + *list(current.get("previousSharedAliases") or []), + ) + return _normalize_contact(promoted), True + + +def _read_contacts() -> dict[str, dict[str, Any]]: + try: + raw = read_secure_json(CONTACTS_FILE, lambda: {}) + except Exception: + import logging + logging.getLogger(__name__).warning( + "Contacts file could not be decrypted — starting with empty contacts" + ) + CONTACTS_FILE.unlink(missing_ok=True) + return {} + if not isinstance(raw, dict): + return {} + contacts: dict[str, dict[str, Any]] = {} + changed = False + for peer_id, value in raw.items(): + key = str(peer_id or "").strip() + if not key: + continue + normalized, promoted = _promote_pending_alias_if_due(value if isinstance(value, dict) else {}) + contacts[key] = normalized + changed = changed or promoted + if changed: + _write_contacts(contacts) + return contacts + + +def _write_contacts(contacts: dict[str, dict[str, Any]]) -> None: + DATA_DIR.mkdir(parents=True, exist_ok=True) + payload = { + str(peer_id): _normalize_contact(contact) + for peer_id, contact in contacts.items() + if str(peer_id or "").strip() + } + write_secure_json(CONTACTS_FILE, payload) + + +def list_wormhole_dm_contacts() -> dict[str, dict[str, Any]]: + return _read_contacts() + + +def upsert_wormhole_dm_contact(peer_id: str, updates: dict[str, Any]) -> dict[str, Any]: + peer_id = str(peer_id or "").strip() + if not peer_id: + raise ValueError("peer_id required") + contacts = _read_contacts() + merged = _normalize_contact({**contacts.get(peer_id, _default_contact()), **dict(updates or {})}) + merged["updated_at"] = int(time.time()) + contacts[peer_id] = merged + _write_contacts(contacts) + return merged + + +def delete_wormhole_dm_contact(peer_id: str) -> bool: + peer_id = str(peer_id or "").strip() + if not peer_id: + return False + contacts = _read_contacts() + if peer_id not in contacts: + return False + del contacts[peer_id] + _write_contacts(contacts) + return True + + +def observe_remote_prekey_identity( + peer_id: str, + *, + fingerprint: str, + sequence: int = 0, + signed_at: int = 0, +) -> dict[str, Any]: + peer_key = str(peer_id or "").strip() + candidate = str(fingerprint or "").strip().lower() + if not peer_key: + raise ValueError("peer_id required") + if not candidate: + raise ValueError("fingerprint required") + + contacts = _read_contacts() + current = _normalize_contact(contacts.get(peer_key)) + now = int(time.time()) + pinned = str(current.get("remotePrekeyFingerprint", "") or "").strip().lower() + + current["remotePrekeyObservedFingerprint"] = candidate + current["remotePrekeyLastSeenAt"] = now + current["remotePrekeySequence"] = int(sequence or 0) + current["remotePrekeySignedAt"] = int(signed_at or 0) + + trust_changed = False + if not pinned: + current["remotePrekeyFingerprint"] = candidate + current["remotePrekeyPinnedAt"] = now + current["remotePrekeyMismatch"] = False + pinned = candidate + else: + trust_changed = pinned != candidate + current["remotePrekeyMismatch"] = trust_changed + + current["updated_at"] = int(time.time()) + contacts[peer_key] = _normalize_contact(current) + _write_contacts(contacts) + return { + "ok": True, + "peer_id": peer_key, + "trust_changed": trust_changed, + "contact": contacts[peer_key], + } diff --git a/backend/services/mesh/mesh_wormhole_dead_drop.py b/backend/services/mesh/mesh_wormhole_dead_drop.py new file mode 100644 index 00000000..e1573f9f --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_dead_drop.py @@ -0,0 +1,416 @@ +"""Wormhole-owned dead-drop token derivation helpers. + +These helpers move mailbox token derivation off the browser when Wormhole is the +secure trust anchor. The browser supplies only peer identifiers and peer DH +public keys; Wormhole derives the shared secret locally and returns mailbox +tokens for the current and previous epochs. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import secrets +import time +from typing import Any + +from cryptography.hazmat.primitives.asymmetric import x25519 + +from services.mesh.mesh_wormhole_identity import bootstrap_wormhole_identity, read_wormhole_identity +from services.mesh.mesh_wormhole_contacts import list_wormhole_dm_contacts, upsert_wormhole_dm_contact +from services.wormhole_settings import read_wormhole_settings + +DEFAULT_DM_EPOCH_SECONDS = 6 * 60 * 60 +HIGH_PRIVACY_DM_EPOCH_SECONDS = 2 * 60 * 60 +SAS_PREFIXES = [ + "amber", + "apex", + "atlas", + "birch", + "cinder", + "cobalt", + "delta", + "ember", + "falcon", + "frost", + "glint", + "harbor", + "juno", + "kepler", + "lumen", + "nova", +] +SAS_SUFFIXES = [ + "anchor", + "arrow", + "bloom", + "cabin", + "cedar", + "cipher", + "comet", + "field", + "grove", + "harvest", + "meadow", + "mesa", + "orbit", + "signal", + "summit", + "thunder", +] +SAS_WORDS = [f"{prefix}-{suffix}" for prefix in SAS_PREFIXES for suffix in SAS_SUFFIXES] +DM_CONSENT_PREFIX = "DM_CONSENT:" +PAIRWISE_ALIAS_PREFIX = "dmx_" + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def build_contact_offer(*, dh_pub_key: str, dh_algo: str, geo_hint: str = "") -> str: + return ( + f"{DM_CONSENT_PREFIX}" + + json.dumps( + { + "kind": "contact_offer", + "dh_pub_key": str(dh_pub_key or ""), + "dh_algo": str(dh_algo or ""), + "geo_hint": str(geo_hint or ""), + }, + separators=(",", ":"), + ) + ) + + +def build_contact_accept(*, shared_alias: str) -> str: + return ( + f"{DM_CONSENT_PREFIX}" + + json.dumps( + { + "kind": "contact_accept", + "shared_alias": str(shared_alias or ""), + }, + separators=(",", ":"), + ) + ) + + +def build_contact_deny(*, reason: str = "") -> str: + return ( + f"{DM_CONSENT_PREFIX}" + + json.dumps( + { + "kind": "contact_deny", + "reason": str(reason or ""), + }, + separators=(",", ":"), + ) + ) + + +def parse_contact_consent(message: str) -> dict[str, Any] | None: + text = str(message or "").strip() + if not text.startswith(DM_CONSENT_PREFIX): + return None + try: + payload = json.loads(text[len(DM_CONSENT_PREFIX) :]) + except Exception: + return None + kind = str(payload.get("kind", "") or "").strip().lower() + if kind == "contact_offer": + dh_pub_key = str(payload.get("dh_pub_key", "") or "").strip() + if not dh_pub_key: + return None + return { + "kind": kind, + "dh_pub_key": dh_pub_key, + "dh_algo": str(payload.get("dh_algo", "") or "").strip() or "X25519", + "geo_hint": str(payload.get("geo_hint", "") or "").strip(), + } + if kind == "contact_accept": + shared_alias = str(payload.get("shared_alias", "") or "").strip() + if not shared_alias: + return None + return {"kind": kind, "shared_alias": shared_alias} + if kind == "contact_deny": + return { + "kind": kind, + "reason": str(payload.get("reason", "") or "").strip(), + } + return None + + +def _new_pairwise_alias() -> str: + return f"{PAIRWISE_ALIAS_PREFIX}{secrets.token_hex(12)}" + + +def _merge_alias_history(*aliases: str, limit: int = 8) -> list[str]: + unique: set[str] = set() + ordered: list[str] = [] + for alias in aliases: + value = str(alias or "").strip() + if not value or value in unique: + continue + unique.add(value) + ordered.append(value) + if len(ordered) >= limit: + break + return ordered + + +def issue_pairwise_dm_alias(*, peer_id: str, peer_dh_pub: str = "") -> dict[str, Any]: + peer_id = str(peer_id or "").strip() + peer_dh_pub = str(peer_dh_pub or "").strip() + if not peer_id: + return {"ok": False, "detail": "peer_id required"} + + from services.mesh.mesh_wormhole_persona import ( + bootstrap_wormhole_persona_state, + get_dm_identity, + ) + + bootstrap_wormhole_persona_state() + dm_identity = get_dm_identity() + current = dict(list_wormhole_dm_contacts().get(peer_id) or {}) + previous_alias = str(current.get("sharedAlias", "") or "").strip() + shared_alias = _new_pairwise_alias() + while shared_alias == previous_alias: + shared_alias = _new_pairwise_alias() + + rotated_at_ms = int(time.time() * 1000) + contact_updates: dict[str, Any] = { + "sharedAlias": shared_alias, + "pendingSharedAlias": "", + "sharedAliasGraceUntil": 0, + "sharedAliasRotatedAt": rotated_at_ms, + "previousSharedAliases": _merge_alias_history( + previous_alias, + *list(current.get("previousSharedAliases") or []), + ), + } + if peer_dh_pub: + contact_updates["dhPubKey"] = peer_dh_pub + elif str(current.get("dhPubKey", "") or "").strip(): + contact_updates["dhPubKey"] = str(current.get("dhPubKey", "") or "").strip() + if str(current.get("dhAlgo", "") or "").strip(): + contact_updates["dhAlgo"] = str(current.get("dhAlgo", "") or "").strip() + + contact = upsert_wormhole_dm_contact(peer_id, contact_updates) + return { + "ok": True, + "peer_id": peer_id, + "shared_alias": shared_alias, + "replaced_alias": previous_alias, + "identity_scope": "dm_alias", + "dm_identity_id": str(dm_identity.get("node_id", "") or ""), + "contact": contact, + } + + +def rotate_pairwise_dm_alias( + *, + peer_id: str, + peer_dh_pub: str = "", + grace_ms: int = 45_000, +) -> dict[str, Any]: + peer_id = str(peer_id or "").strip() + peer_dh_pub = str(peer_dh_pub or "").strip() + if not peer_id: + return {"ok": False, "detail": "peer_id required"} + + from services.mesh.mesh_wormhole_persona import ( + bootstrap_wormhole_persona_state, + get_dm_identity, + ) + + bootstrap_wormhole_persona_state() + dm_identity = get_dm_identity() + current = dict(list_wormhole_dm_contacts().get(peer_id) or {}) + active_alias = str(current.get("sharedAlias", "") or "").strip() + if not active_alias: + return issue_pairwise_dm_alias(peer_id=peer_id, peer_dh_pub=peer_dh_pub) + + now_ms = int(time.time() * 1000) + pending_alias = str(current.get("pendingSharedAlias", "") or "").strip() + grace_until = int(current.get("sharedAliasGraceUntil", 0) or 0) + if pending_alias and grace_until > now_ms: + return { + "ok": True, + "peer_id": peer_id, + "active_alias": active_alias, + "pending_alias": pending_alias, + "grace_until": grace_until, + "identity_scope": "dm_alias", + "dm_identity_id": str(dm_identity.get("node_id", "") or ""), + "contact": current, + "rotated": False, + } + + next_alias = _new_pairwise_alias() + reserved = { + active_alias, + pending_alias, + *[str(item or "").strip() for item in list(current.get("previousSharedAliases") or [])], + } + while next_alias in reserved: + next_alias = _new_pairwise_alias() + + clamped_grace_ms = max(5_000, min(int(grace_ms or 45_000), 5 * 60 * 1000)) + next_grace_until = now_ms + clamped_grace_ms + contact_updates: dict[str, Any] = { + "pendingSharedAlias": next_alias, + "sharedAliasGraceUntil": next_grace_until, + "sharedAliasRotatedAt": now_ms, + "previousSharedAliases": _merge_alias_history( + active_alias, + pending_alias, + *list(current.get("previousSharedAliases") or []), + ), + } + if peer_dh_pub: + contact_updates["dhPubKey"] = peer_dh_pub + elif str(current.get("dhPubKey", "") or "").strip(): + contact_updates["dhPubKey"] = str(current.get("dhPubKey", "") or "").strip() + if str(current.get("dhAlgo", "") or "").strip(): + contact_updates["dhAlgo"] = str(current.get("dhAlgo", "") or "").strip() + + contact = upsert_wormhole_dm_contact(peer_id, contact_updates) + return { + "ok": True, + "peer_id": peer_id, + "active_alias": active_alias, + "pending_alias": next_alias, + "grace_until": next_grace_until, + "identity_scope": "dm_alias", + "dm_identity_id": str(dm_identity.get("node_id", "") or ""), + "contact": contact, + "rotated": True, + } + + +def mailbox_epoch_seconds() -> int: + try: + settings = read_wormhole_settings() + if str(settings.get("privacy_profile", "default") or "default").lower() == "high": + return HIGH_PRIVACY_DM_EPOCH_SECONDS + except Exception: + pass + return DEFAULT_DM_EPOCH_SECONDS + + +def current_mailbox_epoch(ts_seconds: int | None = None) -> int: + now = int(ts_seconds) if ts_seconds is not None else int(time.time()) + return now // mailbox_epoch_seconds() + + +def _derive_shared_secret(my_private_b64: str, peer_public_b64: str) -> bytes: + priv = x25519.X25519PrivateKey.from_private_bytes(_unb64(my_private_b64)) + pub = x25519.X25519PublicKey.from_public_bytes(_unb64(peer_public_b64)) + return priv.exchange(pub) + + +def _token_for(secret: bytes, peer_id: str, my_node_id: str, epoch: int) -> str: + ids = "|".join(sorted([str(my_node_id or ""), str(peer_id or "")])) + message = f"sb_dd|v1|{int(epoch)}|{ids}".encode("utf-8") + return hmac.new(secret, message, hashlib.sha256).hexdigest() + + +def _sas_words_from_digest(digest: bytes, count: int) -> list[str]: + out: list[str] = [] + acc = 0 + acc_bits = 0 + for byte in digest: + acc = (acc << 8) | byte + acc_bits += 8 + while acc_bits >= 8 and len(out) < count: + idx = (acc >> (acc_bits - 8)) & 0xFF + out.append(SAS_WORDS[idx]) + acc_bits -= 8 + if len(out) >= count: + break + return out + + +def derive_dead_drop_token_pair(*, peer_id: str, peer_dh_pub: str) -> dict[str, Any]: + peer_id = str(peer_id or "").strip() + peer_dh_pub = str(peer_dh_pub or "").strip() + if not peer_id or not peer_dh_pub: + return {"ok": False, "detail": "peer_id and peer_dh_pub required"} + + identity = read_wormhole_identity() + if not identity.get("bootstrapped"): + bootstrap_wormhole_identity() + identity = read_wormhole_identity() + + my_private = str(identity.get("dh_private_key", "") or "") + my_node_id = str(identity.get("node_id", "") or "") + if not my_private or not my_node_id: + return {"ok": False, "detail": "Wormhole DH identity unavailable"} + + try: + secret = _derive_shared_secret(my_private, peer_dh_pub) + except Exception as exc: + return {"ok": False, "detail": str(exc) or "dead_drop_secret_failed"} + + epoch = current_mailbox_epoch() + return { + "ok": True, + "peer_id": peer_id, + "epoch": epoch, + "current": _token_for(secret, peer_id, my_node_id, epoch), + "previous": _token_for(secret, peer_id, my_node_id, epoch - 1), + } + + +def derive_dead_drop_tokens_for_contacts(*, contacts: list[dict[str, Any]], limit: int = 24) -> dict[str, Any]: + results: list[dict[str, Any]] = [] + for item in contacts[: max(1, min(int(limit or 24), 64))]: + peer_id = str((item or {}).get("peer_id", "") or "").strip() + peer_dh_pub = str((item or {}).get("peer_dh_pub", "") or "").strip() + if not peer_id or not peer_dh_pub: + continue + pair = derive_dead_drop_token_pair(peer_id=peer_id, peer_dh_pub=peer_dh_pub) + if pair.get("ok"): + results.append( + { + "peer_id": peer_id, + "current": str(pair.get("current", "") or ""), + "previous": str(pair.get("previous", "") or ""), + "epoch": int(pair.get("epoch", 0) or 0), + } + ) + return {"ok": True, "tokens": results} + + +def derive_sas_phrase(*, peer_id: str, peer_dh_pub: str, words: int = 8) -> dict[str, Any]: + peer_id = str(peer_id or "").strip() + peer_dh_pub = str(peer_dh_pub or "").strip() + word_count = max(2, min(int(words or 8), 16)) + if not peer_id or not peer_dh_pub: + return {"ok": False, "detail": "peer_id and peer_dh_pub required"} + + identity = read_wormhole_identity() + if not identity.get("bootstrapped"): + bootstrap_wormhole_identity() + identity = read_wormhole_identity() + + my_private = str(identity.get("dh_private_key", "") or "") + my_node_id = str(identity.get("node_id", "") or "") + if not my_private or not my_node_id: + return {"ok": False, "detail": "Wormhole DH identity unavailable"} + + try: + secret = _derive_shared_secret(my_private, peer_dh_pub) + except Exception as exc: + return {"ok": False, "detail": str(exc) or "sas_secret_failed"} + + ids = "|".join(sorted([my_node_id, peer_id])) + digest = hmac.new(secret, f"sb_sas|v1|{ids}".encode("utf-8"), hashlib.sha256).digest() + phrase = " ".join(_sas_words_from_digest(digest, word_count)) + return {"ok": True, "peer_id": peer_id, "phrase": phrase, "words": word_count} diff --git a/backend/services/mesh/mesh_wormhole_identity.py b/backend/services/mesh/mesh_wormhole_identity.py new file mode 100644 index 00000000..716fe020 --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_identity.py @@ -0,0 +1,231 @@ +"""Wormhole-managed DM identity wrappers. + +This module preserves the legacy DM identity API while sourcing its state from +the Wormhole persona manager. Public transport identity stays separate, and DM +operations now use the dedicated DM alias compartment. +""" + +from __future__ import annotations + +import base64 +import hmac +import hashlib +import time +from typing import Any + +from services.mesh.mesh_protocol import PROTOCOL_VERSION +from services.mesh.mesh_wormhole_persona import ( + bootstrap_wormhole_persona_state, + ensure_dm_mailbox_client_secret, + get_dm_identity, + read_dm_identity, + read_wormhole_persona_state, + sign_dm_wormhole_event, + sign_dm_wormhole_message, + write_dm_identity, +) + + +def _safe_int(val, default=0) -> int: + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _default_identity() -> dict[str, Any]: + return { + "bootstrapped": False, + "bootstrapped_at": 0, + "updated_at": 0, + "scope": "dm_alias", + "label": "dm-alias", + "node_id": "", + "public_key": "", + "public_key_algo": "Ed25519", + "private_key": "", + "sequence": 0, + "dh_pub_key": "", + "dh_algo": "X25519", + "dh_private_key": "", + "last_dh_timestamp": 0, + "bundle_fingerprint": "", + "bundle_sequence": 0, + "bundle_registered_at": 0, + "signed_prekey_id": 0, + "signed_prekey_pub": "", + "signed_prekey_priv": "", + "signed_prekey_signature": "", + "signed_prekey_generated_at": 0, + "signed_prekey_history": [], + "one_time_prekeys": [], + "prekey_bundle_registered_at": 0, + "prekey_republish_threshold": 0, + "prekey_republish_target": 0, + "prekey_next_republish_after": 0, + } + + +def _public_view(data: dict[str, Any]) -> dict[str, Any]: + return { + "bootstrapped": bool(data.get("bootstrapped")), + "bootstrapped_at": _safe_int(data.get("bootstrapped_at", 0) or 0), + "scope": str(data.get("scope", "dm_alias") or "dm_alias"), + "label": str(data.get("label", "dm-alias") or "dm-alias"), + "node_id": str(data.get("node_id", "") or ""), + "public_key": str(data.get("public_key", "") or ""), + "public_key_algo": str(data.get("public_key_algo", "Ed25519") or "Ed25519"), + "sequence": _safe_int(data.get("sequence", 0) or 0), + "dh_pub_key": str(data.get("dh_pub_key", "") or ""), + "dh_algo": str(data.get("dh_algo", "X25519") or "X25519"), + "last_dh_timestamp": _safe_int(data.get("last_dh_timestamp", 0) or 0), + "bundle_fingerprint": str(data.get("bundle_fingerprint", "") or ""), + "bundle_sequence": _safe_int(data.get("bundle_sequence", 0) or 0), + "bundle_registered_at": _safe_int(data.get("bundle_registered_at", 0) or 0), + "protocol_version": PROTOCOL_VERSION, + } + + +def read_wormhole_identity() -> dict[str, Any]: + bootstrap_wormhole_persona_state() + persona_state = read_wormhole_persona_state() + data = {**_default_identity(), **read_dm_identity()} + data["bootstrapped"] = True + data["bootstrapped_at"] = _safe_int(persona_state.get("bootstrapped_at", 0) or 0) + return data + + +def _write_identity(data: dict[str, Any]) -> dict[str, Any]: + current = read_wormhole_identity() + merged = {**current, **dict(data or {})} + merged["scope"] = "dm_alias" + merged["label"] = str(merged.get("label", "dm-alias") or "dm-alias") + merged["updated_at"] = int(time.time()) + saved = write_dm_identity(merged) + saved["bootstrapped"] = True + return {**_default_identity(), **saved} + + +def bootstrap_wormhole_identity(force: bool = False) -> dict[str, Any]: + bootstrap_wormhole_persona_state(force=force) + data = read_wormhole_identity() + if force: + data["bundle_fingerprint"] = "" + data["bundle_sequence"] = 0 + data["bundle_registered_at"] = 0 + data["signed_prekey_id"] = 0 + data["signed_prekey_pub"] = "" + data["signed_prekey_priv"] = "" + data["signed_prekey_signature"] = "" + data["signed_prekey_generated_at"] = 0 + data["signed_prekey_history"] = [] + data["one_time_prekeys"] = [] + data["prekey_bundle_registered_at"] = 0 + data["prekey_republish_threshold"] = 0 + data["prekey_republish_target"] = 0 + data["prekey_next_republish_after"] = 0 + data = _write_identity(data) + return _public_view(data) + + +def get_wormhole_identity() -> dict[str, Any]: + return get_dm_identity() + + +def sign_wormhole_event( + *, + event_type: str, + payload: dict[str, Any], + sequence: int | None = None, +) -> dict[str, Any]: + return sign_dm_wormhole_event(event_type=event_type, payload=payload, sequence=sequence) + + +def sign_wormhole_message(message: str) -> dict[str, Any]: + return sign_dm_wormhole_message(message) + + +def _bundle_fingerprint(data: dict[str, Any]) -> str: + raw = "|".join( + [ + str(data.get("dh_pub_key", "")), + str(data.get("dh_algo", "X25519")), + str(data.get("public_key", "")), + str(data.get("public_key_algo", "Ed25519")), + PROTOCOL_VERSION, + ] + ) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def register_wormhole_dm_key(force: bool = False) -> dict[str, Any]: + data = read_wormhole_identity() + + timestamp = int(time.time()) + fingerprint = _bundle_fingerprint(data) + if not force and fingerprint and fingerprint == data.get("bundle_fingerprint"): + return { + "ok": True, + **_public_view(data), + } + + payload = { + "dh_pub_key": str(data.get("dh_pub_key", "")), + "dh_algo": str(data.get("dh_algo", "X25519")), + "timestamp": timestamp, + } + signed = sign_wormhole_event(event_type="dm_key", payload=payload) + + from services.mesh.mesh_dm_relay import dm_relay + + accepted, detail, metadata = dm_relay.register_dh_key( + signed["node_id"], + payload["dh_pub_key"], + payload["dh_algo"], + payload["timestamp"], + signed["signature"], + signed["public_key"], + signed["public_key_algo"], + signed["protocol_version"], + signed["sequence"], + ) + if not accepted: + return {"ok": False, "detail": detail} + + data = read_wormhole_identity() + data["bundle_fingerprint"] = metadata.get("bundle_fingerprint", fingerprint) if metadata else fingerprint + data["bundle_sequence"] = _safe_int( + metadata.get("accepted_sequence", signed["sequence"]) if metadata else signed["sequence"], + _safe_int(signed.get("sequence", 0), 0), + ) + data["bundle_registered_at"] = timestamp + data["last_dh_timestamp"] = timestamp + saved = _write_identity(data) + return { + "ok": True, + **_public_view(saved), + **(metadata or {}), + } + + +def get_dm_mailbox_client_secret(*, generate: bool = True) -> str: + return ensure_dm_mailbox_client_secret(generate=generate) + + +def derive_dm_mailbox_token( + dm_alias_id: str | None = None, + *, + generate_secret: bool = True, +) -> str: + data = read_wormhole_identity() + alias_id = str(dm_alias_id or data.get("node_id", "") or "").strip() + if not alias_id: + return "" + secret_b64 = get_dm_mailbox_client_secret(generate=generate_secret) + if not secret_b64: + return "" + try: + secret = base64.b64decode(secret_b64.encode("ascii")) + except Exception: + return "" + return hmac.new(secret, alias_id.encode("utf-8"), hashlib.sha256).hexdigest() diff --git a/backend/services/mesh/mesh_wormhole_persona.py b/backend/services/mesh/mesh_wormhole_persona.py new file mode 100644 index 00000000..40c67948 --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_persona.py @@ -0,0 +1,1038 @@ +"""Wormhole-owned public identity compartments. + +This module separates Wormhole's internal root trust anchor from the public +identities used for transport and future gate-scoped personas. The current +phase keeps public posting on a dedicated transport identity while preparing +gate session/persona identities for later phases. +""" + +from __future__ import annotations + +import base64 +import logging +import random +import secrets +import time +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519, x25519 + +from services.mesh.mesh_crypto import build_signature_payload, derive_node_id +from services.mesh.mesh_privacy_logging import privacy_log_label +from services.mesh.mesh_protocol import PROTOCOL_VERSION, normalize_payload +from services.mesh.mesh_secure_storage import ( + read_domain_json, + read_secure_json, + write_domain_json, +) + +logger = logging.getLogger(__name__) + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +PERSONA_FILE = DATA_DIR / "wormhole_persona.json" +LEGACY_DM_IDENTITY_FILE = DATA_DIR / "wormhole_identity.json" +TRANSPORT_DOMAIN = "transport" +ROOT_DOMAIN = "root" +DM_ALIAS_DOMAIN = "dm_alias" +GATE_SESSION_DOMAIN = "gate_session" +GATE_PERSONA_DOMAIN = "gate_persona" +TRANSPORT_FILE = "wormhole_transport.json" +ROOT_FILE = "wormhole_root.json" +DM_ALIAS_FILE = "wormhole_dm_identity.json" +GATE_SESSION_FILE = "wormhole_gate_sessions.json" +GATE_PERSONA_FILE = "wormhole_gate_personas.json" + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _empty_identity(scope: str = "") -> dict[str, Any]: + return { + "scope": scope, + "gate_id": "", + "persona_id": "", + "label": "", + "node_id": "", + "public_key": "", + "public_key_algo": "Ed25519", + "private_key": "", + "sequence": 0, + "dh_pub_key": "", + "dh_algo": "X25519", + "dh_private_key": "", + "created_at": 0, + "last_used_at": 0, + "last_dh_timestamp": 0, + "bundle_fingerprint": "", + "bundle_sequence": 0, + "bundle_registered_at": 0, + "signed_prekey_id": 0, + "signed_prekey_pub": "", + "signed_prekey_priv": "", + "signed_prekey_signature": "", + "signed_prekey_generated_at": 0, + "signed_prekey_history": [], + "one_time_prekeys": [], + "prekey_bundle_registered_at": 0, + "prekey_republish_threshold": 0, + "prekey_republish_target": 0, + "prekey_next_republish_after": 0, + "mailbox_client_secret": "", + } + + +def _default_state() -> dict[str, Any]: + return { + "bootstrapped": False, + "bootstrapped_at": 0, + "updated_at": 0, + "root_identity": _empty_identity("root"), + "transport_identity": _empty_identity("transport"), + "dm_identity": _empty_identity("dm_alias"), + "gate_sessions": {}, + "gate_personas": {}, + "active_gate_personas": {}, + } + + +def _identity_record(*, scope: str, gate_id: str = "", persona_id: str = "", label: str = "") -> dict[str, Any]: + signing_priv = ed25519.Ed25519PrivateKey.generate() + signing_priv_raw = signing_priv.private_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PrivateFormat.Raw, + encryption_algorithm=serialization.NoEncryption(), + ) + signing_pub_raw = signing_priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + + dh_priv = x25519.X25519PrivateKey.generate() + dh_priv_raw = dh_priv.private_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PrivateFormat.Raw, + encryption_algorithm=serialization.NoEncryption(), + ) + dh_pub_raw = dh_priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + + now = int(time.time()) + return { + "scope": scope, + "gate_id": str(gate_id or "").lower(), + "persona_id": str(persona_id or ""), + "label": str(label or ""), + "node_id": derive_node_id(_b64(signing_pub_raw)), + "public_key": _b64(signing_pub_raw), + "public_key_algo": "Ed25519", + "private_key": _b64(signing_priv_raw), + "sequence": 0, + "dh_pub_key": _b64(dh_pub_raw), + "dh_algo": "X25519", + "dh_private_key": _b64(dh_priv_raw), + "created_at": now, + "last_used_at": now, + } + + +def _public_identity_view(identity: dict[str, Any]) -> dict[str, Any]: + return { + "scope": str(identity.get("scope", "") or ""), + "gate_id": str(identity.get("gate_id", "") or ""), + "persona_id": str(identity.get("persona_id", "") or ""), + "label": str(identity.get("label", "") or ""), + "node_id": str(identity.get("node_id", "") or ""), + "public_key": str(identity.get("public_key", "") or ""), + "public_key_algo": str(identity.get("public_key_algo", "Ed25519") or "Ed25519"), + "sequence": int(identity.get("sequence", 0) or 0), + "dh_pub_key": str(identity.get("dh_pub_key", "") or ""), + "dh_algo": str(identity.get("dh_algo", "X25519") or "X25519"), + "last_dh_timestamp": int(identity.get("last_dh_timestamp", 0) or 0), + "bundle_fingerprint": str(identity.get("bundle_fingerprint", "") or ""), + "bundle_sequence": int(identity.get("bundle_sequence", 0) or 0), + "bundle_registered_at": int(identity.get("bundle_registered_at", 0) or 0), + "created_at": int(identity.get("created_at", 0) or 0), + "last_used_at": int(identity.get("last_used_at", 0) or 0), + "protocol_version": PROTOCOL_VERSION, + } + + +def _read_legacy_dm_identity() -> dict[str, Any]: + try: + raw = read_secure_json(LEGACY_DM_IDENTITY_FILE, lambda: {}) + except Exception: + logger.warning("Legacy DM identity could not be decrypted — skipping migration") + LEGACY_DM_IDENTITY_FILE.unlink(missing_ok=True) + return {} + if not isinstance(raw, dict): + return {} + if not raw.get("private_key"): + return {} + return { + **_empty_identity("dm_alias"), + **raw, + "scope": "dm_alias", + "label": str(raw.get("label", "dm-alias") or "dm-alias"), + "last_used_at": int(raw.get("last_used_at", raw.get("updated_at", raw.get("bootstrapped_at", 0))) or 0), + "created_at": int(raw.get("created_at", raw.get("bootstrapped_at", 0)) or 0), + } + + +def _transport_domain_default() -> dict[str, Any]: + return { + "bootstrapped": False, + "bootstrapped_at": 0, + "updated_at": 0, + "transport_identity": _empty_identity("transport"), + } + + +def _root_domain_default() -> dict[str, Any]: + return {"root_identity": _empty_identity("root")} + + +def _dm_alias_domain_default() -> dict[str, Any]: + return {"dm_identity": _empty_identity("dm_alias")} + + +def _gate_session_domain_default() -> dict[str, Any]: + return {"gate_sessions": {}, "active_gate_personas": {}} + + +def _gate_persona_domain_default() -> dict[str, Any]: + return {"gate_personas": {}} + + +def _domain_transport_path() -> Path: + return DATA_DIR / TRANSPORT_DOMAIN / TRANSPORT_FILE + + +def _any_domain_persona_file_exists() -> bool: + return any( + path.exists() + for path in ( + _domain_transport_path(), + DATA_DIR / ROOT_DOMAIN / ROOT_FILE, + DATA_DIR / DM_ALIAS_DOMAIN / DM_ALIAS_FILE, + DATA_DIR / GATE_SESSION_DOMAIN / GATE_SESSION_FILE, + DATA_DIR / GATE_PERSONA_DOMAIN / GATE_PERSONA_FILE, + ) + ) + + +def _migrate_legacy_persona_state_if_needed() -> None: + if _any_domain_persona_file_exists() or not PERSONA_FILE.exists(): + return + try: + legacy_state = read_secure_json(PERSONA_FILE, _default_state) + except Exception: + logger.warning("Legacy persona state could not be decrypted — skipping migration") + PERSONA_FILE.unlink(missing_ok=True) + return + state = _default_state() + if isinstance(legacy_state, dict): + state.update(legacy_state) + write_domain_json( + TRANSPORT_DOMAIN, + TRANSPORT_FILE, + { + "bootstrapped": bool(state.get("bootstrapped")), + "bootstrapped_at": int(state.get("bootstrapped_at", 0) or 0), + "updated_at": int(state.get("updated_at", 0) or 0), + "transport_identity": dict(state.get("transport_identity") or {}), + }, + ) + write_domain_json( + ROOT_DOMAIN, + ROOT_FILE, + {"root_identity": dict(state.get("root_identity") or {})}, + ) + write_domain_json( + DM_ALIAS_DOMAIN, + DM_ALIAS_FILE, + {"dm_identity": dict(state.get("dm_identity") or {})}, + ) + write_domain_json( + GATE_SESSION_DOMAIN, + GATE_SESSION_FILE, + { + "gate_sessions": dict(state.get("gate_sessions") or {}), + "active_gate_personas": dict(state.get("active_gate_personas") or {}), + }, + ) + write_domain_json( + GATE_PERSONA_DOMAIN, + GATE_PERSONA_FILE, + {"gate_personas": dict(state.get("gate_personas") or {})}, + ) + PERSONA_FILE.unlink(missing_ok=True) + + +def read_wormhole_persona_state() -> dict[str, Any]: + _migrate_legacy_persona_state_if_needed() + transport_state = read_domain_json(TRANSPORT_DOMAIN, TRANSPORT_FILE, _transport_domain_default) + root_state = read_domain_json(ROOT_DOMAIN, ROOT_FILE, _root_domain_default) + dm_state = read_domain_json(DM_ALIAS_DOMAIN, DM_ALIAS_FILE, _dm_alias_domain_default) + gate_session_state = read_domain_json( + GATE_SESSION_DOMAIN, + GATE_SESSION_FILE, + _gate_session_domain_default, + ) + gate_persona_state = read_domain_json( + GATE_PERSONA_DOMAIN, + GATE_PERSONA_FILE, + _gate_persona_domain_default, + ) + state = _default_state() + if isinstance(transport_state, dict): + state["bootstrapped"] = bool(transport_state.get("bootstrapped")) + state["bootstrapped_at"] = int(transport_state.get("bootstrapped_at", 0) or 0) + state["updated_at"] = int(transport_state.get("updated_at", 0) or 0) + state["transport_identity"] = dict(transport_state.get("transport_identity") or {}) + if isinstance(root_state, dict): + state["root_identity"] = dict(root_state.get("root_identity") or {}) + if isinstance(dm_state, dict): + state["dm_identity"] = dict(dm_state.get("dm_identity") or {}) + if isinstance(gate_session_state, dict): + state["gate_sessions"] = dict(gate_session_state.get("gate_sessions") or {}) + state["active_gate_personas"] = dict(gate_session_state.get("active_gate_personas") or {}) + if isinstance(gate_persona_state, dict): + state["gate_personas"] = dict(gate_persona_state.get("gate_personas") or {}) + state["bootstrapped"] = bool(state.get("bootstrapped")) + state["bootstrapped_at"] = int(state.get("bootstrapped_at", 0) or 0) + state["updated_at"] = int(state.get("updated_at", 0) or 0) + state["root_identity"] = {**_empty_identity("root"), **dict(state.get("root_identity") or {})} + state["transport_identity"] = { + **_empty_identity("transport"), + **dict(state.get("transport_identity") or {}), + } + state["dm_identity"] = { + **_empty_identity("dm_alias"), + **dict(state.get("dm_identity") or {}), + } + state["gate_sessions"] = { + str(k).lower(): {**_empty_identity("gate_session"), **dict(v or {})} + for k, v in dict(state.get("gate_sessions") or {}).items() + } + state["gate_personas"] = { + str(k).lower(): [{**_empty_identity("gate_persona"), **dict(item or {})} for item in list(v or [])] + for k, v in dict(state.get("gate_personas") or {}).items() + } + state["active_gate_personas"] = { + str(k).lower(): str(v or "") + for k, v in dict(state.get("active_gate_personas") or {}).items() + } + return state + + +def _write_wormhole_persona_state(state: dict[str, Any]) -> dict[str, Any]: + payload = dict(state) + payload["updated_at"] = int(time.time()) + write_domain_json( + TRANSPORT_DOMAIN, + TRANSPORT_FILE, + { + "bootstrapped": bool(payload.get("bootstrapped")), + "bootstrapped_at": int(payload.get("bootstrapped_at", 0) or 0), + "updated_at": int(payload.get("updated_at", 0) or 0), + "transport_identity": dict(payload.get("transport_identity") or {}), + }, + ) + write_domain_json( + ROOT_DOMAIN, + ROOT_FILE, + {"root_identity": dict(payload.get("root_identity") or {})}, + ) + write_domain_json( + DM_ALIAS_DOMAIN, + DM_ALIAS_FILE, + {"dm_identity": dict(payload.get("dm_identity") or {})}, + ) + write_domain_json( + GATE_SESSION_DOMAIN, + GATE_SESSION_FILE, + { + "gate_sessions": dict(payload.get("gate_sessions") or {}), + "active_gate_personas": dict(payload.get("active_gate_personas") or {}), + }, + ) + write_domain_json( + GATE_PERSONA_DOMAIN, + GATE_PERSONA_FILE, + {"gate_personas": dict(payload.get("gate_personas") or {})}, + ) + PERSONA_FILE.unlink(missing_ok=True) + return payload + + +def bootstrap_wormhole_persona_state(force: bool = False) -> dict[str, Any]: + state = read_wormhole_persona_state() + now = int(time.time()) + changed = force or not bool(state.get("bootstrapped")) + if force or not state.get("root_identity", {}).get("private_key"): + state["root_identity"] = _identity_record(scope="root", label="root") + changed = True + if force or not state.get("transport_identity", {}).get("private_key"): + state["transport_identity"] = _identity_record(scope="transport", label="transport") + changed = True + if force or not state.get("dm_identity", {}).get("private_key"): + legacy_dm = _read_legacy_dm_identity() if not force else {} + state["dm_identity"] = legacy_dm or _identity_record(scope="dm_alias", label="dm-alias") + changed = True + if changed: + state["bootstrapped"] = True + if not state.get("bootstrapped_at") or force: + state["bootstrapped_at"] = now + state = _write_wormhole_persona_state(state) + return { + "bootstrapped": bool(state.get("bootstrapped")), + "bootstrapped_at": int(state.get("bootstrapped_at", 0) or 0), + "transport_identity": _public_identity_view(state.get("transport_identity") or {}), + } + + +def get_transport_identity() -> dict[str, Any]: + bootstrap_wormhole_persona_state() + full_state = read_wormhole_persona_state() + return { + "bootstrapped": bool(full_state.get("bootstrapped")), + "bootstrapped_at": int(full_state.get("bootstrapped_at", 0) or 0), + **_public_identity_view(full_state.get("transport_identity") or {}), + } + + +def read_dm_identity() -> dict[str, Any]: + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + return {**_empty_identity("dm_alias"), **dict(state.get("dm_identity") or {})} + + +def write_dm_identity(identity: dict[str, Any]) -> dict[str, Any]: + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + merged = {**_empty_identity("dm_alias"), **dict(identity or {})} + merged["scope"] = "dm_alias" + merged["label"] = str(merged.get("label", "dm-alias") or "dm-alias") + state["dm_identity"] = merged + updated = _write_wormhole_persona_state(state) + return {**_empty_identity("dm_alias"), **dict(updated.get("dm_identity") or {})} + + +def get_dm_identity() -> dict[str, Any]: + bootstrap_wormhole_persona_state() + full_state = read_wormhole_persona_state() + return { + "bootstrapped": bool(full_state.get("bootstrapped")), + "bootstrapped_at": int(full_state.get("bootstrapped_at", 0) or 0), + **_public_identity_view(full_state.get("dm_identity") or {}), + } + + +def _touch(identity: dict[str, Any]) -> None: + identity["last_used_at"] = int(time.time()) + + +def _next_sequence(identity: dict[str, Any], sequence: int | None = None) -> int: + if sequence is None: + next_value = int(identity.get("sequence", 0) or 0) + 1 + else: + next_value = max(int(identity.get("sequence", 0) or 0), int(sequence)) + identity["sequence"] = next_value + _touch(identity) + return next_value + + +def _sign_with_identity( + *, + identity: dict[str, Any], + event_type: str, + payload: dict[str, Any], + sequence: int | None = None, +) -> dict[str, Any]: + normalized = normalize_payload(event_type, dict(payload or {})) + signed_sequence = _next_sequence(identity, sequence) + payload_str = build_signature_payload( + event_type=event_type, + node_id=str(identity["node_id"]), + sequence=int(signed_sequence), + payload=normalized, + ) + signing_priv = ed25519.Ed25519PrivateKey.from_private_bytes( + _unb64(str(identity.get("private_key", ""))) + ) + signature = signing_priv.sign(payload_str.encode("utf-8")).hex() + return { + "node_id": str(identity["node_id"]), + "public_key": str(identity["public_key"]), + "public_key_algo": str(identity.get("public_key_algo", "Ed25519") or "Ed25519"), + "protocol_version": PROTOCOL_VERSION, + "sequence": int(signed_sequence), + "payload": normalized, + "signature": signature, + "signature_payload": payload_str, + } + + +def sign_public_wormhole_event( + *, + event_type: str, + payload: dict[str, Any], + sequence: int | None = None, +) -> dict[str, Any]: + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + identity = state.get("transport_identity") or _empty_identity("transport") + signed = _sign_with_identity(identity=identity, event_type=event_type, payload=payload, sequence=sequence) + _write_wormhole_persona_state(state) + return {**signed, "identity_scope": "transport"} + + +def sign_dm_wormhole_event( + *, + event_type: str, + payload: dict[str, Any], + sequence: int | None = None, +) -> dict[str, Any]: + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + identity = state.get("dm_identity") or _empty_identity("dm_alias") + signed = _sign_with_identity(identity=identity, event_type=event_type, payload=payload, sequence=sequence) + _write_wormhole_persona_state(state) + return {**signed, "identity_scope": "dm_alias"} + + +def sign_dm_wormhole_message(message: str) -> dict[str, Any]: + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + identity = state.get("dm_identity") or _empty_identity("dm_alias") + _touch(identity) + signing_priv = ed25519.Ed25519PrivateKey.from_private_bytes( + _unb64(str(identity.get("private_key", ""))) + ) + signature = signing_priv.sign(str(message or "").encode("utf-8")).hex() + _write_wormhole_persona_state(state) + return { + "node_id": str(identity.get("node_id", "") or ""), + "public_key": str(identity.get("public_key", "") or ""), + "public_key_algo": str(identity.get("public_key_algo", "Ed25519") or "Ed25519"), + "protocol_version": PROTOCOL_VERSION, + "signature": signature, + "message": str(message or ""), + "identity_scope": "dm_alias", + } + + +def _bound_dm_alias_blob(alias: str, payload: bytes) -> bytes: + alias_key = str(alias or "").strip().lower() + return f"dm-mls-binding|{alias_key}|".encode("utf-8") + bytes(payload or b"") + + +def sign_dm_alias_blob(alias: str, payload: bytes) -> dict[str, Any]: + alias_key = str(alias or "").strip().lower() + if not alias_key: + return {"ok": False, "detail": "alias required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + identity = state.get("dm_identity") or _empty_identity("dm_alias") + try: + signing_priv = ed25519.Ed25519PrivateKey.from_private_bytes( + _unb64(str(identity.get("private_key", "") or "")) + ) + signature = signing_priv.sign(_bound_dm_alias_blob(alias_key, payload)).hex() + except Exception: + logger.exception( + "dm alias blob sign failed for %s", + privacy_log_label(alias_key, label="alias"), + ) + return {"ok": False, "detail": "dm_alias_blob_sign_failed"} + _touch(identity) + state["dm_identity"] = identity + _write_wormhole_persona_state(state) + return { + "ok": True, + "alias": alias_key, + "signature": signature, + "public_key": str(identity.get("public_key", "") or ""), + "public_key_algo": str(identity.get("public_key_algo", "Ed25519") or "Ed25519"), + } + + +def verify_dm_alias_blob(alias: str, payload: bytes, signature: str) -> tuple[bool, str]: + alias_key = str(alias or "").strip().lower() + if not alias_key: + return False, "alias required" + if not str(signature or "").strip(): + return False, "signature required" + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + identity = state.get("dm_identity") or _empty_identity("dm_alias") + try: + signing_pub = ed25519.Ed25519PublicKey.from_public_bytes( + _unb64(str(identity.get("public_key", "") or "")) + ) + signing_pub.verify( + bytes.fromhex(str(signature or "")), + _bound_dm_alias_blob(alias_key, payload), + ) + except Exception: + return False, "dm alias blob signature invalid" + return True, "ok" + + +def ensure_dm_mailbox_client_secret(*, generate: bool = True) -> str: + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + identity = state.get("dm_identity") or _empty_identity("dm_alias") + secret = str(identity.get("mailbox_client_secret", "") or "").strip() + if secret or not generate: + return secret + secret = _b64(secrets.token_bytes(32)) + identity["mailbox_client_secret"] = secret + _touch(identity) + state["dm_identity"] = identity + _write_wormhole_persona_state(state) + return secret + + +def _ensure_gate_session(state: dict[str, Any], gate_key: str, *, rotate: bool = False) -> dict[str, Any]: + existing = dict(state.get("gate_sessions", {}).get(gate_key) or {}) + if not rotate and existing.get("private_key"): + from services.config import get_settings + + settings = get_settings() + msg_limit = int(settings.MESH_GATE_SESSION_ROTATE_MSGS or 0) + time_limit = int(settings.MESH_GATE_SESSION_ROTATE_S or 0) + jitter_limit = max(0.0, float(getattr(settings, "MESH_GATE_SESSION_ROTATE_JITTER_S", 0) or 0.0)) + msg_count = int(existing.get("_msg_count", 0) or 0) + created_at = float(existing.get("_created_at", 0) or 0) + now = time.time() + rotate_due_at = float(existing.get("_rotate_after", 0) or 0.0) + threshold_hit = False + if msg_limit > 0 and msg_count >= msg_limit: + threshold_hit = True + elif time_limit > 0 and created_at > 0 and (now - created_at) >= time_limit: + threshold_hit = True + if threshold_hit: + if jitter_limit > 0: + if rotate_due_at <= 0: + rotate_due_at = now + random.uniform(0.0, jitter_limit) + existing["_rotate_after"] = rotate_due_at + state.setdefault("gate_sessions", {})[gate_key] = existing + rotate = now >= rotate_due_at + else: + rotate = True + elif rotate_due_at > 0: + existing["_rotate_after"] = 0.0 + state.setdefault("gate_sessions", {})[gate_key] = existing + if rotate or not existing.get("private_key"): + new_identity = _identity_record( + scope="gate_session", + gate_id=gate_key, + label="anonymous", + ) + new_identity["_msg_count"] = 0 + new_identity["_created_at"] = time.time() + new_identity["_rotate_after"] = 0.0 + state.setdefault("gate_sessions", {})[gate_key] = new_identity + return state["gate_sessions"][gate_key] + + +def enter_gate_anonymously(gate_id: str, *, rotate: bool = False) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + # Entering anonymously must clear any previously active persona for the + # same gate so the caller cannot accidentally keep posting under a stale + # gate-local face after explicitly choosing anonymous mode. + state.setdefault("active_gate_personas", {}).pop(gate_key, None) + session = _ensure_gate_session(state, gate_key, rotate=rotate) + _touch(session) + _write_wormhole_persona_state(state) + return {"ok": True, "identity": _public_identity_view(session)} + + +def leave_gate(gate_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + removed = False + if gate_key in state.get("gate_sessions", {}): + state["gate_sessions"].pop(gate_key, None) + removed = True + if gate_key in state.get("active_gate_personas", {}): + state["active_gate_personas"].pop(gate_key, None) + removed = True + if removed: + _write_wormhole_persona_state(state) + return {"ok": True, "gate_id": gate_key, "cleared": removed} + + +def _unique_gate_persona_label(gate_key: str, requested_label: str, existing_personas: list[dict[str, Any]]) -> str: + base_label = str(requested_label or "").strip() + if not base_label: + base_label = f"{gate_key}-persona" + used_labels = { + str(persona.get("label", "") or "").strip().lower() + for persona in existing_personas + if str(persona.get("label", "") or "").strip() + } + if base_label.lower() not in used_labels: + return base_label + suffix = 2 + while f"{base_label}-{suffix}".lower() in used_labels: + suffix += 1 + return f"{base_label}-{suffix}" + + +def create_gate_persona(gate_id: str, *, label: str = "") -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + personas = list(state.get("gate_personas", {}).get(gate_key) or []) + persona_id = secrets.token_hex(8) + requested_label = str(label or f"anon_{persona_id[:4]}").strip() + persona = _identity_record( + scope="gate_persona", + gate_id=gate_key, + persona_id=persona_id, + label=_unique_gate_persona_label(gate_key, requested_label, personas), + ) + personas.append(persona) + state.setdefault("gate_personas", {})[gate_key] = personas + state.setdefault("active_gate_personas", {})[gate_key] = persona_id + _write_wormhole_persona_state(state) + return {"ok": True, "identity": _public_identity_view(persona)} + + +def list_gate_personas(gate_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + personas = [ + _public_identity_view(item) + for item in list(state.get("gate_personas", {}).get(gate_key) or []) + ] + return { + "ok": True, + "gate_id": gate_key, + "active_persona_id": str(state.get("active_gate_personas", {}).get(gate_key, "") or ""), + "personas": personas, + } + + +def activate_gate_persona(gate_id: str, persona_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + target_persona = str(persona_id or "").strip() + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + personas = list(state.get("gate_personas", {}).get(gate_key) or []) + for persona in personas: + if str(persona.get("persona_id", "") or "") == target_persona: + state.setdefault("active_gate_personas", {})[gate_key] = target_persona + _touch(persona) + _write_wormhole_persona_state(state) + return {"ok": True, "identity": _public_identity_view(persona)} + return {"ok": False, "detail": "persona not found"} + + +def retire_gate_persona(gate_id: str, persona_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + target_persona = str(persona_id or "").strip() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + if not target_persona: + return {"ok": False, "detail": "persona_id required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + personas = list(state.get("gate_personas", {}).get(gate_key) or []) + removed_persona: dict[str, Any] | None = None + remaining_personas: list[dict[str, Any]] = [] + for persona in personas: + if str(persona.get("persona_id", "") or "") == target_persona: + removed_persona = persona + continue + remaining_personas.append(persona) + if removed_persona is None: + return {"ok": False, "detail": "persona not found"} + + if remaining_personas: + state.setdefault("gate_personas", {})[gate_key] = remaining_personas + else: + state.setdefault("gate_personas", {}).pop(gate_key, None) + + active_persona_id = str(state.get("active_gate_personas", {}).get(gate_key, "") or "") + active_identity: dict[str, Any] | None = None + if active_persona_id == target_persona: + state.setdefault("active_gate_personas", {}).pop(gate_key, None) + session = _ensure_gate_session(state, gate_key, rotate=True) + _touch(session) + active_identity = _public_identity_view(session) + _write_wormhole_persona_state(state) + return { + "ok": True, + "gate_id": gate_key, + "retired_persona_id": target_persona, + "retired_identity": _public_identity_view(removed_persona), + "active_identity": active_identity, + } + + +def clear_active_gate_persona(gate_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + state.setdefault("active_gate_personas", {}).pop(gate_key, None) + # Returning to anonymous mode should yield a fresh, gate-scoped session + # identity instead of resurrecting an older anonymous session. + session = _ensure_gate_session(state, gate_key, rotate=True) + _touch(session) + _write_wormhole_persona_state(state) + return {"ok": True, "identity": _public_identity_view(session)} + + +def get_active_gate_identity(gate_id: str) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + active_persona_id = str(state.get("active_gate_personas", {}).get(gate_key, "") or "") + for persona in list(state.get("gate_personas", {}).get(gate_key) or []): + if str(persona.get("persona_id", "") or "") == active_persona_id: + return {"ok": True, "identity": _public_identity_view(persona), "source": "persona"} + session = dict(state.get("gate_sessions", {}).get(gate_key) or {}) + if session.get("private_key"): + return {"ok": True, "identity": _public_identity_view(session), "source": "anonymous"} + return {"ok": False, "detail": "no active gate identity"} + + +def _find_gate_persona_record(state: dict[str, Any], gate_id: str, persona_id: str) -> dict[str, Any] | None: + gate_key = str(gate_id or "").strip().lower() + target_persona = str(persona_id or "").strip() + for persona in list(state.get("gate_personas", {}).get(gate_key) or []): + if str(persona.get("persona_id", "") or "") == target_persona: + return persona + return None + + +def _find_gate_session_record(state: dict[str, Any], gate_id: str, node_id: str = "") -> dict[str, Any] | None: + gate_key = str(gate_id or "").strip().lower() + session = dict(state.get("gate_sessions", {}).get(gate_key) or {}) + if not session.get("private_key"): + return None + target_node_id = str(node_id or "").strip() + if target_node_id and str(session.get("node_id", "") or "").strip() != target_node_id: + return None + return session + + +def _bound_gate_persona_blob(gate_id: str, persona_id: str, payload: bytes) -> bytes: + gate_key = str(gate_id or "").strip().lower() + target_persona = str(persona_id or "").strip() + return ( + f"gate-mls-binding|{gate_key}|{target_persona}|".encode("utf-8") + + bytes(payload or b"") + ) + + +def _bound_gate_session_blob(gate_id: str, node_id: str, payload: bytes) -> bytes: + gate_key = str(gate_id or "").strip().lower() + target_node_id = str(node_id or "").strip() + return ( + f"gate-mls-binding|{gate_key}|session:{target_node_id}|".encode("utf-8") + + bytes(payload or b"") + ) + + +def sign_gate_persona_blob(gate_id: str, persona_id: str, payload: bytes) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + target_persona = str(persona_id or "").strip() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + if not target_persona: + return {"ok": False, "detail": "persona_id required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + persona = _find_gate_persona_record(state, gate_key, target_persona) + if persona is None: + return {"ok": False, "detail": "persona not found"} + try: + bound_payload = _bound_gate_persona_blob(gate_key, target_persona, bytes(payload or b"")) + signing_priv = ed25519.Ed25519PrivateKey.from_private_bytes( + _unb64(str(persona.get("private_key", "") or "")) + ) + signature = signing_priv.sign(bound_payload).hex() + except Exception: + logger.exception( + "gate persona blob sign failed for %s/%s", + privacy_log_label(gate_key, label="gate"), + privacy_log_label(target_persona, label="persona"), + ) + return {"ok": False, "detail": "persona_blob_sign_failed"} + _touch(persona) + _write_wormhole_persona_state(state) + return { + "ok": True, + "gate_id": gate_key, + "persona_id": target_persona, + "signature": signature, + "public_key": str(persona.get("public_key", "") or ""), + "public_key_algo": str(persona.get("public_key_algo", "Ed25519") or "Ed25519"), + } + + +def verify_gate_persona_blob( + gate_id: str, + persona_id: str, + payload: bytes, + signature: str, +) -> tuple[bool, str]: + gate_key = str(gate_id or "").strip().lower() + target_persona = str(persona_id or "").strip() + if not gate_key: + return False, "gate_id required" + if not target_persona: + return False, "persona_id required" + if not str(signature or "").strip(): + return False, "signature required" + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + persona = _find_gate_persona_record(state, gate_key, target_persona) + if persona is None: + return False, "persona not found" + try: + bound_payload = _bound_gate_persona_blob(gate_key, target_persona, bytes(payload or b"")) + signing_pub = ed25519.Ed25519PublicKey.from_public_bytes( + _unb64(str(persona.get("public_key", "") or "")) + ) + signing_pub.verify(bytes.fromhex(str(signature or "")), bound_payload) + except Exception: + return False, "persona blob signature invalid" + return True, "ok" + + +def sign_gate_session_blob(gate_id: str, node_id: str, payload: bytes) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + target_node_id = str(node_id or "").strip() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + if not target_node_id: + return {"ok": False, "detail": "node_id required"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + session = _find_gate_session_record(state, gate_key, target_node_id) + if session is None: + return {"ok": False, "detail": "anonymous gate session not found"} + try: + bound_payload = _bound_gate_session_blob(gate_key, target_node_id, bytes(payload or b"")) + signing_priv = ed25519.Ed25519PrivateKey.from_private_bytes( + _unb64(str(session.get("private_key", "") or "")) + ) + signature = signing_priv.sign(bound_payload).hex() + except Exception: + logger.exception( + "gate session blob sign failed for %s/%s", + privacy_log_label(gate_key, label="gate"), + privacy_log_label(target_node_id, label="session"), + ) + return {"ok": False, "detail": "gate_session_blob_sign_failed"} + _touch(session) + state.setdefault("gate_sessions", {})[gate_key] = session + _write_wormhole_persona_state(state) + return { + "ok": True, + "gate_id": gate_key, + "node_id": target_node_id, + "signature": signature, + "public_key": str(session.get("public_key", "") or ""), + "public_key_algo": str(session.get("public_key_algo", "Ed25519") or "Ed25519"), + } + + +def verify_gate_session_blob( + gate_id: str, + node_id: str, + payload: bytes, + signature: str, +) -> tuple[bool, str]: + gate_key = str(gate_id or "").strip().lower() + target_node_id = str(node_id or "").strip() + if not gate_key: + return False, "gate_id required" + if not target_node_id: + return False, "node_id required" + if not str(signature or "").strip(): + return False, "signature required" + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + session = _find_gate_session_record(state, gate_key, target_node_id) + if session is None: + return False, "anonymous gate session not found" + try: + bound_payload = _bound_gate_session_blob(gate_key, target_node_id, bytes(payload or b"")) + signing_pub = ed25519.Ed25519PublicKey.from_public_bytes( + _unb64(str(session.get("public_key", "") or "")) + ) + signing_pub.verify(bytes.fromhex(str(signature or "")), bound_payload) + except Exception: + return False, "gate session blob signature invalid" + return True, "ok" + + +def sign_gate_wormhole_event( + *, + gate_id: str, + event_type: str, + payload: dict[str, Any], + sequence: int | None = None, +) -> dict[str, Any]: + gate_key = str(gate_id or "").strip().lower() + if not gate_key: + return {"ok": False, "detail": "gate_id required"} + normalized_payload = normalize_payload(event_type, dict(payload or {})) + payload_gate = str( + normalized_payload.get("gate") + or normalized_payload.get("gate_id") + or "" + ).strip().lower() + if payload_gate and payload_gate != gate_key: + return {"ok": False, "detail": "gate payload mismatch"} + bootstrap_wormhole_persona_state() + state = read_wormhole_persona_state() + active_persona_id = str(state.get("active_gate_personas", {}).get(gate_key, "") or "") + identity: dict[str, Any] | None = None + identity_scope = "gate_session" + for persona in list(state.get("gate_personas", {}).get(gate_key) or []): + if str(persona.get("persona_id", "") or "") == active_persona_id: + identity = persona + identity_scope = "gate_persona" + break + if identity is None: + identity = _ensure_gate_session(state, gate_key, rotate=False) + signed = _sign_with_identity( + identity=identity, + event_type=event_type, + payload=normalized_payload, + sequence=sequence, + ) + if identity_scope == "gate_session": + identity["_msg_count"] = int(identity.get("_msg_count", 0) or 0) + 1 + _write_wormhole_persona_state(state) + return {**signed, "identity_scope": identity_scope, "gate_id": gate_key} diff --git a/backend/services/mesh/mesh_wormhole_prekey.py b/backend/services/mesh/mesh_wormhole_prekey.py new file mode 100644 index 00000000..e5ced213 --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_prekey.py @@ -0,0 +1,538 @@ +"""Wormhole-managed prekey bundles and X3DH-style bootstrap helpers.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import random +import time +from typing import Any + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ed25519, x25519 +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat + +from services.mesh.mesh_crypto import derive_node_id +from services.mesh.mesh_wormhole_identity import ( + _write_identity, + bootstrap_wormhole_identity, + read_wormhole_identity, + sign_wormhole_event, + sign_wormhole_message, +) + +PREKEY_TARGET = 8 +PREKEY_MIN_THRESHOLD = 3 +PREKEY_MAX_THRESHOLD = 5 +PREKEY_MIN_TARGET = 7 +PREKEY_MAX_TARGET = 9 +PREKEY_MIN_REPUBLISH_DELAY_S = 45 +PREKEY_MAX_REPUBLISH_DELAY_S = 120 +PREKEY_REPUBLISH_THRESHOLD_RANGE = (PREKEY_MIN_THRESHOLD, PREKEY_MAX_THRESHOLD) +PREKEY_REPUBLISH_TARGET_RANGE = (PREKEY_MIN_TARGET, PREKEY_MAX_TARGET) +PREKEY_REPUBLISH_DELAY_RANGE_S = (PREKEY_MIN_REPUBLISH_DELAY_S, PREKEY_MAX_REPUBLISH_DELAY_S) +SIGNED_PREKEY_ROTATE_AFTER_S = 24 * 60 * 60 +SIGNED_PREKEY_GRACE_S = 3 * 24 * 60 * 60 + + +def _safe_int(val, default=0) -> int: + try: + return int(val) + except (TypeError, ValueError): + return default + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _stable_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def _x25519_pair() -> dict[str, str]: + priv = x25519.X25519PrivateKey.generate() + priv_raw = priv.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()) + pub_raw = priv.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + return {"public_key": _b64(pub_raw), "private_key": _b64(priv_raw)} + + +def _derive(priv_b64: str, pub_b64: str) -> bytes: + priv = x25519.X25519PrivateKey.from_private_bytes(_unb64(priv_b64)) + pub = x25519.X25519PublicKey.from_public_bytes(_unb64(pub_b64)) + return priv.exchange(pub) + + +def _hkdf(ikm: bytes, info: str, length: int = 32) -> bytes: + return HKDF( + algorithm=hashes.SHA256(), + length=length, + salt=b"\xff" * 32, + info=info.encode("utf-8"), + ).derive(ikm) + + +def _bundle_payload(data: dict[str, Any]) -> dict[str, Any]: + one_time_prekeys = [ + { + "prekey_id": _safe_int(item.get("prekey_id", 0) or 0), + "public_key": str(item.get("public_key", "") or ""), + } + for item in list(data.get("one_time_prekeys") or []) + if item.get("public_key") + ] + return { + "identity_dh_pub_key": str(data.get("dh_pub_key", "") or ""), + "dh_algo": str(data.get("dh_algo", "X25519") or "X25519"), + "signed_prekey_id": _safe_int(data.get("signed_prekey_id", 0) or 0), + "signed_prekey_pub": str(data.get("signed_prekey_pub", "") or ""), + "signed_prekey_signature": str(data.get("signed_prekey_signature", "") or ""), + "signed_prekey_timestamp": _safe_int(data.get("signed_prekey_generated_at", 0) or 0), + "signed_at": _safe_int(data.get("prekey_bundle_signed_at", 0) or 0), + "bundle_signature": str(data.get("prekey_bundle_signature", "") or ""), + "mls_key_package": str(data.get("mls_key_package", "") or ""), + "one_time_prekeys": one_time_prekeys, + "one_time_prekey_count": len(one_time_prekeys), + } + + +def _bundle_signature_payload(data: dict[str, Any]) -> str: + # OTK binding: One-time key hashes are included in the bundle signature + # as of Sprint 12 (S12-3). Relay substitution of OTKs will now break + # the bundle signature and be rejected by verify_prekey_bundle(). + otk_hashes = sorted( + hashlib.sha256(str(item.get("public_key", "")).encode("utf-8")).hexdigest() + for item in (data.get("one_time_prekeys") or []) + ) + return _stable_json( + { + "identity_dh_pub_key": str(data.get("identity_dh_pub_key", "") or ""), + "dh_algo": str(data.get("dh_algo", "X25519") or "X25519"), + "signed_prekey_id": _safe_int(data.get("signed_prekey_id", 0) or 0), + "signed_prekey_pub": str(data.get("signed_prekey_pub", "") or ""), + "signed_prekey_signature": str(data.get("signed_prekey_signature", "") or ""), + "signed_at": _safe_int(data.get("signed_at", 0) or 0), + "mls_key_package": str(data.get("mls_key_package", "") or ""), + "one_time_prekey_hashes": otk_hashes, + } + ) + + +def _max_prekey_bundle_age_s() -> int: + return SIGNED_PREKEY_ROTATE_AFTER_S + SIGNED_PREKEY_GRACE_S + + +def trust_fingerprint_for_bundle_record(record: dict[str, Any]) -> str: + bundle = dict(record.get("bundle") or record or {}) + material = { + "agent_id": str(record.get("agent_id", "") or ""), + "identity_dh_pub_key": str(bundle.get("identity_dh_pub_key", "") or ""), + "dh_algo": str(bundle.get("dh_algo", "X25519") or "X25519"), + "public_key": str(record.get("public_key", "") or ""), + "public_key_algo": str(record.get("public_key_algo", "") or ""), + "protocol_version": str(record.get("protocol_version", "") or ""), + } + return hashlib.sha256(_stable_json(material).encode("utf-8")).hexdigest() + + +def _attach_bundle_signature(bundle: dict[str, Any], *, signed_at: int | None = None) -> dict[str, Any]: + # KNOWN LIMITATION: Bundle signature is self-signed by the identity key it contains. + # This proves possession of the private key and detects post-registration tampering, + # but cannot prevent initial impersonation (no external PKI). Mitigated by reputation + # system in Phase 9 (Oracle Rep). See threat-model.md for full analysis. + payload = dict(bundle or {}) + payload["signed_at"] = int(signed_at if signed_at is not None else time.time()) + signed = sign_wormhole_message(_bundle_signature_payload(payload)) + payload["bundle_signature"] = str(signed.get("signature", "") or "") + return payload + + +def _verify_bundle_signature(bundle: dict[str, Any], public_key: str) -> tuple[bool, str]: + try: + signing_pub = ed25519.Ed25519PublicKey.from_public_bytes(_unb64(public_key)) + signing_pub.verify( + bytes.fromhex(str(bundle.get("bundle_signature", "") or "")), + _bundle_signature_payload(bundle).encode("utf-8"), + ) + except Exception: + return False, "Prekey bundle signature invalid" + return True, "ok" + + +def _validate_bundle_record(record: dict[str, Any]) -> tuple[bool, str]: + bundle = dict(record.get("bundle") or {}) + now = time.time() + signed_at = _safe_int(bundle.get("signed_at", 0) or 0) + if signed_at <= 0: + return False, "Prekey bundle missing signed_at" + if signed_at > now + 299: + return False, "Prekey bundle signed_at is in the future" + if not str(bundle.get("bundle_signature", "") or "").strip(): + return False, "Prekey bundle missing bundle_signature" + public_key = str(record.get("public_key", "") or "") + if not public_key: + return False, "Prekey bundle missing signing key" + ok, reason = _verify_bundle_signature(bundle, public_key) + if not ok: + return False, reason + if (now - signed_at) > _max_prekey_bundle_age_s(): + return False, "Prekey bundle is stale" + if str(record.get("agent_id", "") or "").strip(): + derived = derive_node_id(public_key) + if derived != str(record.get("agent_id", "") or "").strip(): + return False, "Prekey bundle public key binding mismatch" + return True, "ok" + + +def _jittered_republish_policy(data: dict[str, Any], *, reset: bool = False) -> tuple[int, int]: + threshold = _safe_int(data.get("prekey_republish_threshold", 0) or 0) + target = _safe_int(data.get("prekey_republish_target", 0) or 0) + min_threshold, max_threshold = PREKEY_REPUBLISH_THRESHOLD_RANGE + min_target, max_target = PREKEY_REPUBLISH_TARGET_RANGE + if reset or threshold < min_threshold or threshold > max_threshold: + threshold = random.randint(min_threshold, max_threshold) + data["prekey_republish_threshold"] = threshold + if reset or target < min_target or target > max_target: + target = random.randint(min_target, max_target) + data["prekey_republish_target"] = target + return threshold, target + + +def _schedule_next_republish_window(data: dict[str, Any]) -> None: + min_delay_s, max_delay_s = PREKEY_REPUBLISH_DELAY_RANGE_S + data["prekey_next_republish_after"] = int( + time.time() + random.randint(min_delay_s, max_delay_s) + ) + + +def _archive_current_signed_prekey(data: dict[str, Any], retired_at: int) -> None: + current_id = _safe_int(data.get("signed_prekey_id", 0) or 0) + current_pub = str(data.get("signed_prekey_pub", "") or "") + current_priv = str(data.get("signed_prekey_priv", "") or "") + current_sig = str(data.get("signed_prekey_signature", "") or "") + current_generated_at = _safe_int(data.get("signed_prekey_generated_at", 0) or 0) + if not current_id or not current_pub or not current_priv: + return + history = list(data.get("signed_prekey_history") or []) + history.append( + { + "signed_prekey_id": current_id, + "signed_prekey_pub": current_pub, + "signed_prekey_priv": current_priv, + "signed_prekey_signature": current_sig, + "signed_prekey_generated_at": current_generated_at, + "retired_at": retired_at, + } + ) + cutoff = retired_at - SIGNED_PREKEY_GRACE_S + data["signed_prekey_history"] = [ + item + for item in history[-4:] + if _safe_int(item.get("retired_at", retired_at) or retired_at) >= cutoff + ] + + +def _find_signed_prekey_private(data: dict[str, Any], spk_id: int) -> str: + if _safe_int(data.get("signed_prekey_id", 0) or 0) == spk_id: + return str(data.get("signed_prekey_priv", "") or "") + for item in list(data.get("signed_prekey_history") or []): + if _safe_int(item.get("signed_prekey_id", 0) or 0) == spk_id: + return str(item.get("signed_prekey_priv", "") or "") + return "" + + +def ensure_wormhole_prekeys(force_signed_prekey: bool = False, replenish_target: int = PREKEY_TARGET) -> dict[str, Any]: + data = read_wormhole_identity() + if not data.get("bootstrapped"): + bootstrap_wormhole_identity() + data = read_wormhole_identity() + + changed = False + now = int(time.time()) + _, jitter_target = _jittered_republish_policy(data) + replenish_target = max(1, _safe_int(replenish_target or jitter_target, 1)) + + spk_generated_at = _safe_int(data.get("signed_prekey_generated_at", 0) or 0) + spk_too_old = bool(spk_generated_at and (now - spk_generated_at) >= SIGNED_PREKEY_ROTATE_AFTER_S) + if force_signed_prekey or spk_too_old or not data.get("signed_prekey_pub") or not data.get("signed_prekey_priv"): + _archive_current_signed_prekey(data, now) + pair = _x25519_pair() + spk_id = _safe_int(data.get("signed_prekey_id", 0) or 0) + 1 + signed_prekey_payload = { + "signed_prekey_id": spk_id, + "signed_prekey_pub": pair["public_key"], + "signed_prekey_timestamp": now, + } + signed = sign_wormhole_event( + event_type="dm_signed_prekey", + payload=signed_prekey_payload, + ) + data["signed_prekey_id"] = spk_id + data["signed_prekey_pub"] = pair["public_key"] + data["signed_prekey_priv"] = pair["private_key"] + data["signed_prekey_signature"] = signed["signature"] + data["signed_prekey_generated_at"] = now + changed = True + + existing_otks = list(data.get("one_time_prekeys") or []) + next_id = max([_safe_int(item.get("prekey_id", 0) or 0) for item in existing_otks] + [0]) + while len(existing_otks) < max(1, replenish_target): + next_id += 1 + pair = _x25519_pair() + existing_otks.append( + { + "prekey_id": next_id, + "public_key": pair["public_key"], + "private_key": pair["private_key"], + "created_at": now, + } + ) + changed = True + data["one_time_prekeys"] = existing_otks + _jittered_republish_policy(data) + + if changed: + _write_identity(data) + return _bundle_payload(data) + + +def register_wormhole_prekey_bundle(force_signed_prekey: bool = False) -> dict[str, Any]: + data = read_wormhole_identity() + if not data.get("bootstrapped"): + bootstrap_wormhole_identity() + data = read_wormhole_identity() + + _, jitter_target = _jittered_republish_policy(data, reset=force_signed_prekey) + if force_signed_prekey: + _schedule_next_republish_window(data) + _write_identity(data) + data = read_wormhole_identity() + + bundle = ensure_wormhole_prekeys(force_signed_prekey=force_signed_prekey, replenish_target=jitter_target) + from services.mesh.mesh_dm_mls import export_dm_key_package_for_alias + + mls_key_package = export_dm_key_package_for_alias(str(data.get("node_id", "") or "")) + if not mls_key_package.get("ok"): + return {"ok": False, "detail": str(mls_key_package.get("detail", "") or "mls key package unavailable")} + bundle["mls_key_package"] = str(mls_key_package.get("mls_key_package", "") or "") + bundle = _attach_bundle_signature(bundle) + signed = sign_wormhole_event( + event_type="dm_prekey_bundle", + payload=bundle, + ) + + from services.mesh.mesh_dm_relay import dm_relay + + accepted, detail, metadata = dm_relay.register_prekey_bundle( + signed["node_id"], + bundle, + signed["signature"], + signed["public_key"], + signed["public_key_algo"], + signed["protocol_version"], + signed["sequence"], + ) + if not accepted: + return {"ok": False, "detail": detail} + refreshed = read_wormhole_identity() + refreshed["prekey_bundle_registered_at"] = int(time.time()) + refreshed["prekey_bundle_signed_at"] = _safe_int(bundle.get("signed_at", 0) or 0) + refreshed["prekey_bundle_signature"] = str(bundle.get("bundle_signature", "") or "") + _schedule_next_republish_window(refreshed) + _jittered_republish_policy(refreshed, reset=True) + _write_identity(refreshed) + return { + "ok": True, + "agent_id": signed["node_id"], + "bundle": bundle, + "signature": signed["signature"], + "public_key": signed["public_key"], + "public_key_algo": signed["public_key_algo"], + "protocol_version": signed["protocol_version"], + "sequence": signed["sequence"], + **(metadata or {}), + } + + +def fetch_dm_prekey_bundle(agent_id: str) -> dict[str, Any]: + from services.mesh.mesh_dm_relay import dm_relay + + stored = dm_relay.get_prekey_bundle(agent_id) + if not stored: + return {"ok": False, "detail": "Prekey bundle not found"} + validated_record = {**dict(stored), "agent_id": str(agent_id or "").strip()} + ok, reason = _validate_bundle_record(validated_record) + if not ok: + return {"ok": False, "detail": reason} + bundle = dict(stored.get("bundle") or {}) + bundle["one_time_prekeys"] = [] + bundle["one_time_prekey_count"] = _safe_int(bundle.get("one_time_prekey_count", 0) or 0) + return { + "ok": True, + "agent_id": agent_id, + **bundle, + "signature": str(stored.get("signature", "") or ""), + "public_key": str(stored.get("public_key", "") or ""), + "public_key_algo": str(stored.get("public_key_algo", "") or ""), + "protocol_version": str(stored.get("protocol_version", "") or ""), + "sequence": _safe_int(stored.get("sequence", 0) or 0), + "trust_fingerprint": trust_fingerprint_for_bundle_record(validated_record), + } + + +def _consume_local_one_time_prekey(prekey_id: int) -> int: + if prekey_id <= 0: + data = read_wormhole_identity() + return len(list(data.get("one_time_prekeys") or [])) + data = read_wormhole_identity() + existing = list(data.get("one_time_prekeys") or []) + filtered = [ + item for item in existing if _safe_int(item.get("prekey_id", 0) or 0) != _safe_int(prekey_id) + ] + if len(filtered) == len(existing): + return len(existing) + data["one_time_prekeys"] = filtered + _write_identity(data) + return len(filtered) + + +def bootstrap_encrypt_for_peer(peer_id: str, plaintext: str) -> dict[str, Any]: + from services.mesh.mesh_dm_relay import dm_relay + + stored = dm_relay.get_prekey_bundle(peer_id) + if not stored: + return {"ok": False, "detail": "Peer prekey bundle not found"} + validated_record = {**dict(stored), "agent_id": str(peer_id or "").strip()} + ok, reason = _validate_bundle_record(validated_record) + if not ok: + return {"ok": False, "detail": reason} + peer_bundle_stored = dm_relay.consume_one_time_prekey(peer_id) + if not peer_bundle_stored: + return {"ok": False, "detail": "Peer prekey bundle not found"} + peer_bundle = dict(peer_bundle_stored.get("bundle") or {}) + peer_static = str(peer_bundle.get("identity_dh_pub_key", "") or "") + peer_spk = str(peer_bundle.get("signed_prekey_pub", "") or "") + peer_spk_id = _safe_int(peer_bundle.get("signed_prekey_id", 0) or 0) + peer_otk = dict(peer_bundle_stored.get("claimed_one_time_prekey") or {}) + + data = read_wormhole_identity() + if not data.get("bootstrapped"): + bootstrap_wormhole_identity() + data = read_wormhole_identity() + my_static_priv = str(data.get("dh_private_key", "") or "") + my_static_pub = str(data.get("dh_pub_key", "") or "") + if not my_static_priv or not my_static_pub or not peer_static or not peer_spk: + return {"ok": False, "detail": "Missing static or signed prekey material"} + + eph = _x25519_pair() + dh_parts = [ + _derive(my_static_priv, peer_spk), + _derive(eph["private_key"], peer_static), + _derive(eph["private_key"], peer_spk), + ] + otk_id = 0 + if peer_otk and peer_otk.get("public_key"): + dh_parts.append(_derive(eph["private_key"], str(peer_otk.get("public_key")))) + otk_id = _safe_int(peer_otk.get("prekey_id", 0) or 0) + secret = _hkdf(b"".join(dh_parts), "SB-X3DH", 32) + header = { + "v": 1, + "alg": "X25519", + "ik_pub": my_static_pub, + "ek_pub": eph["public_key"], + "spk_id": peer_spk_id, + "otk_id": otk_id, + } + aad = _stable_json(header).encode("utf-8") + iv = os.urandom(12) + ciphertext = AESGCM(secret).encrypt(iv, plaintext.encode("utf-8"), aad) + envelope = { + "h": header, + "ct": _b64(iv + ciphertext), + } + wrapped = _b64(_stable_json(envelope).encode("utf-8")) + return {"ok": True, "result": f"x3dh1:{wrapped}"} + + +def bootstrap_decrypt_from_sender(sender_id: str, ciphertext: str) -> dict[str, Any]: + if not ciphertext.startswith("x3dh1:"): + return {"ok": False, "detail": "legacy"} + try: + raw = ciphertext[len("x3dh1:") :] + envelope = json.loads(_unb64(raw).decode("utf-8")) + header = dict(envelope.get("h") or {}) + combined = _unb64(str(envelope.get("ct") or "")) + my_data = read_wormhole_identity() + if not my_data.get("bootstrapped"): + bootstrap_wormhole_identity() + my_data = read_wormhole_identity() + + sender_static_pub = str(header.get("ik_pub", "") or "") + sender_eph_pub = str(header.get("ek_pub", "") or "") + spk_id = _safe_int(header.get("spk_id", 0) or 0) + otk_id = _safe_int(header.get("otk_id", 0) or 0) + if not sender_static_pub or not sender_eph_pub: + return {"ok": False, "detail": "Missing sender bootstrap keys"} + + from services.mesh.mesh_dm_relay import dm_relay + + sender_dh = dm_relay.get_dh_key(sender_id) + if sender_dh and sender_dh.get("dh_pub_key") and str(sender_dh.get("dh_pub_key")) != sender_static_pub: + return {"ok": False, "detail": "Sender static DH key mismatch"} + + signed_prekey_priv = _find_signed_prekey_private(my_data, spk_id) + my_static_priv = str(my_data.get("dh_private_key", "") or "") + if not signed_prekey_priv or not my_static_priv: + return {"ok": False, "detail": "Missing local bootstrap private keys"} + + dh_parts = [ + _derive(signed_prekey_priv, sender_static_pub), + _derive(my_static_priv, sender_eph_pub), + _derive(signed_prekey_priv, sender_eph_pub), + ] + if otk_id: + otk_match = next( + ( + item + for item in list(my_data.get("one_time_prekeys") or []) + if _safe_int(item.get("prekey_id", 0) or 0) == otk_id and item.get("private_key") + ), + None, + ) + if not otk_match: + return {"ok": False, "detail": "One-time prekey mismatch"} + dh_parts.append(_derive(str(otk_match.get("private_key", "")), sender_eph_pub)) + + secret = _hkdf(b"".join(dh_parts), "SB-X3DH", 32) + aad = _stable_json(header).encode("utf-8") + iv = combined[:12] + ct = combined[12:] + plaintext = AESGCM(secret).decrypt(iv, ct, aad).decode("utf-8") + if otk_id: + remaining_otks = _consume_local_one_time_prekey(otk_id) + my_data = read_wormhole_identity() + threshold, target = _jittered_republish_policy(my_data) + next_republish_after = _safe_int(my_data.get("prekey_next_republish_after", 0) or 0) + now_ts = int(time.time()) + should_republish = remaining_otks <= 0 + if not should_republish and remaining_otks <= threshold and now_ts >= next_republish_after: + should_republish = True + if should_republish: + register_wormhole_prekey_bundle() + else: + _write_identity(my_data) + return {"ok": True, "result": plaintext} + except Exception as exc: + return {"ok": False, "detail": str(exc) or "bootstrap_decrypt_failed"} diff --git a/backend/services/mesh/mesh_wormhole_ratchet.py b/backend/services/mesh/mesh_wormhole_ratchet.py new file mode 100644 index 00000000..4849491b --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_ratchet.py @@ -0,0 +1,361 @@ +"""Wormhole-backed DM ratchet state and crypto. + +This is the first DM custody move out of the browser. When Wormhole is active, +the frontend no longer persists ratchet/session state in IndexedDB; instead the +agent owns the session records and performs ratchet encrypt/decrypt operations +locally on behalf of the UI. +""" + +from __future__ import annotations + +import base64 +import json +import os +import threading +import time +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives import hashes, hmac +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives.serialization import Encoding, PrivateFormat, PublicFormat, NoEncryption + +from services.mesh.mesh_wormhole_identity import bootstrap_wormhole_identity, read_wormhole_identity +from services.mesh.mesh_secure_storage import read_secure_json, write_secure_json + +DATA_DIR = Path(__file__).resolve().parents[2] / "data" +STATE_FILE = DATA_DIR / "wormhole_dm_ratchet.json" +STATE_LOCK = threading.RLock() + +MAX_SKIP = 32 +PAD_BUCKET = 1024 +PAD_STEP = 512 +PAD_MAX = 4096 +PAD_MAGIC = "SBP1" + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _zero_bytes(length: int) -> bytes: + return bytes([0] * length) + + +def _stable_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def _header_aad(header: dict[str, Any]) -> bytes: + return _stable_json(header).encode("utf-8") + + +def _build_padded_payload(plaintext: str) -> bytes: + data = plaintext.encode("utf-8") + length = len(data) + target = PAD_BUCKET + if length + 6 > target: + target = ((length + 6 + PAD_STEP - 1) // PAD_STEP) * PAD_STEP + if target > PAD_MAX: + target = ((length + 6 + PAD_STEP - 1) // PAD_STEP) * PAD_STEP + out = bytearray(target) + out[0:4] = PAD_MAGIC.encode("utf-8") + out[4] = (length >> 8) & 0xFF + out[5] = length & 0xFF + out[6 : 6 + length] = data + if target > length + 6: + out[6 + length :] = os.urandom(target - (6 + length)) + return bytes(out) + + +def _unpad_payload(data: bytes) -> str: + if len(data) < 6: + return data.decode("utf-8", errors="replace") + magic = data[:4].decode("utf-8", errors="ignore") + if magic != PAD_MAGIC: + return data.decode("utf-8", errors="replace") + length = (data[4] << 8) + data[5] + if length <= 0 or 6 + length > len(data): + return data.decode("utf-8", errors="replace") + return data[6 : 6 + length].decode("utf-8", errors="replace") + + +def _load_all_states() -> dict[str, dict[str, Any]]: + with STATE_LOCK: + try: + raw = read_secure_json(STATE_FILE, lambda: {}) + except Exception: + import logging + logging.getLogger(__name__).warning( + "Wormhole ratchet state could not be decrypted — starting fresh" + ) + STATE_FILE.unlink(missing_ok=True) + raw = {} + if not isinstance(raw, dict): + return {} + return {str(k): dict(v) for k, v in raw.items() if isinstance(v, dict)} + + +def _save_all_states(states: dict[str, dict[str, Any]]) -> None: + with STATE_LOCK: + DATA_DIR.mkdir(parents=True, exist_ok=True) + write_secure_json(STATE_FILE, states) + + +def _get_state(peer_id: str) -> dict[str, Any] | None: + return _load_all_states().get(peer_id) + + +def _set_state(peer_id: str, state: dict[str, Any]) -> None: + states = _load_all_states() + states[peer_id] = state + _save_all_states(states) + + +def reset_wormhole_dm_ratchet(peer_id: str | None = None) -> dict[str, Any]: + if peer_id: + states = _load_all_states() + states.pop(peer_id, None) + _save_all_states(states) + else: + _save_all_states({}) + return {"ok": True, "peer_id": peer_id or "", "cleared_all": not bool(peer_id)} + + +def _generate_ratchet_key_pair() -> dict[str, str]: + priv = x25519.X25519PrivateKey.generate() + priv_raw = priv.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()) + pub_raw = priv.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + return { + "pub": _b64(pub_raw), + "priv": _b64(priv_raw), + "algo": "X25519", + } + + +def _derive_dh_secret(priv_b64: str, their_pub_b64: str) -> bytes: + priv = x25519.X25519PrivateKey.from_private_bytes(_unb64(priv_b64)) + pub = x25519.X25519PublicKey.from_public_bytes(_unb64(their_pub_b64)) + return priv.exchange(pub) + + +def _wormhole_long_term_dh_priv_b64() -> str: + data = read_wormhole_identity() + if not data.get("bootstrapped") or not data.get("dh_private_key"): + bootstrap_wormhole_identity() + data = read_wormhole_identity() + return str(data.get("dh_private_key", "") or "") + + +def _hkdf(ikm: bytes, salt: bytes, info: str, length: int) -> bytes: + derived = HKDF( + algorithm=hashes.SHA256(), + length=length, + salt=salt, + info=info.encode("utf-8"), + ).derive(ikm) + return bytes(derived) + + +def _kdf_rk(rk: bytes, dh_out: bytes) -> tuple[bytes, bytes]: + salt = rk if rk else _zero_bytes(32) + out = _hkdf(dh_out, salt, "SB-DR-RK", 64) + return out[:32], out[32:64] + + +def _hmac_sha256(key_bytes: bytes, data: bytes) -> bytes: + mac = hmac.HMAC(key_bytes, hashes.SHA256()) + mac.update(data) + return bytes(mac.finalize()) + + +def _kdf_ck(ck: bytes) -> tuple[bytes, bytes]: + mk = _hmac_sha256(ck, b"\x01") + next_ck = _hmac_sha256(ck, b"\x02") + return next_ck, mk + + +def _aes_gcm_encrypt(mk: bytes, plaintext: str, aad: bytes) -> str: + iv = os.urandom(12) + aes = AESGCM(mk) + encoded = _build_padded_payload(plaintext) + ciphertext = aes.encrypt(iv, encoded, aad) + return _b64(iv + ciphertext) + + +def _aes_gcm_decrypt(mk: bytes, ciphertext_b64: str, aad: bytes) -> str: + combined = _unb64(ciphertext_b64) + iv = combined[:12] + ciphertext = combined[12:] + aes = AESGCM(mk) + plaintext = aes.decrypt(iv, ciphertext, aad) + return _unpad_payload(bytes(plaintext)) + + +def _skip_message_keys(state: dict[str, Any], until: int) -> None: + if not state.get("ckr"): + return + skipped = dict(state.get("skipped") or {}) + while int(state.get("nr", 0) or 0) < until: + next_ck, mk = _kdf_ck(_unb64(str(state["ckr"]))) + key_id = f"{state.get('dhRemote', '')}:{int(state.get('nr', 0) or 0)}" + if len(skipped) < MAX_SKIP: + skipped[key_id] = _b64(mk) + state["ckr"] = _b64(next_ck) + state["nr"] = int(state.get("nr", 0) or 0) + 1 + state["skipped"] = skipped + + +def _dh_ratchet(state: dict[str, Any], remote_dh: str, pn: int) -> dict[str, Any]: + _skip_message_keys(state, pn) + state["pn"] = int(state.get("ns", 0) or 0) + state["ns"] = 0 + state["nr"] = 0 + state["dhRemote"] = remote_dh + + rk_bytes = _unb64(str(state.get("rk", "") or "")) or _zero_bytes(32) + dh_out_1 = _derive_dh_secret(str(state.get("dhSelfPriv", "")), str(state.get("dhRemote", ""))) + rk_1, ck_r = _kdf_rk(rk_bytes, dh_out_1) + state["rk"] = _b64(rk_1) + state["ckr"] = _b64(ck_r) + + fresh = _generate_ratchet_key_pair() + state["dhSelfPub"] = fresh["pub"] + state["dhSelfPriv"] = fresh["priv"] + dh_out_2 = _derive_dh_secret(str(state.get("dhSelfPriv", "")), str(state.get("dhRemote", ""))) + rk_2, ck_s = _kdf_rk(_unb64(str(state.get("rk", ""))), dh_out_2) + state["rk"] = _b64(rk_2) + state["cks"] = _b64(ck_s) + state["algo"] = "X25519" + state["updated"] = int(time.time() * 1000) + return state + + +def _init_sender_state(peer_id: str, their_dh_pub: str) -> dict[str, Any]: + fresh = _generate_ratchet_key_pair() + dh_out = _derive_dh_secret(fresh["priv"], their_dh_pub) + rk, ck = _kdf_rk(_zero_bytes(32), dh_out) + return { + "algo": "X25519", + "rk": _b64(rk), + "cks": _b64(ck), + "ckr": "", + "dhSelfPub": fresh["pub"], + "dhSelfPriv": fresh["priv"], + "dhRemote": their_dh_pub, + "ns": 0, + "nr": 0, + "pn": 0, + "skipped": {}, + "updated": int(time.time() * 1000), + } + + +def _init_receiver_state(peer_id: str, sender_dh_pub: str) -> dict[str, Any]: + long_term_priv = _wormhole_long_term_dh_priv_b64() + if not long_term_priv: + raise ValueError("missing_long_term_key") + dh_out = _derive_dh_secret(long_term_priv, sender_dh_pub) + rk, ck = _kdf_rk(_zero_bytes(32), dh_out) + fresh = _generate_ratchet_key_pair() + return { + "algo": "X25519", + "rk": _b64(rk), + "cks": "", + "ckr": _b64(ck), + "dhSelfPub": fresh["pub"], + "dhSelfPriv": fresh["priv"], + "dhRemote": sender_dh_pub, + "ns": 0, + "nr": 0, + "pn": 0, + "skipped": {}, + "updated": int(time.time() * 1000), + } + + +def _ensure_send_chain(state: dict[str, Any]) -> dict[str, Any]: + if state.get("cks"): + return state + rk_bytes = _unb64(str(state.get("rk", "") or "")) or _zero_bytes(32) + dh_out = _derive_dh_secret(str(state.get("dhSelfPriv", "")), str(state.get("dhRemote", ""))) + rk, ck = _kdf_rk(rk_bytes, dh_out) + state["rk"] = _b64(rk) + state["cks"] = _b64(ck) + state["updated"] = int(time.time() * 1000) + return state + + +def encrypt_wormhole_dm(peer_id: str, peer_dh_pub: str, plaintext: str) -> dict[str, Any]: + if not peer_id or not peer_dh_pub: + return {"ok": False, "detail": "peer_id and peer_dh_pub are required"} + state = _get_state(peer_id) + if not state: + state = _init_sender_state(peer_id, peer_dh_pub) + state = _ensure_send_chain(state) + next_ck, mk = _kdf_ck(_unb64(str(state.get("cks", "")))) + n = int(state.get("ns", 0) or 0) + state["ns"] = n + 1 + state["cks"] = _b64(next_ck) + header = { + "v": 2, + "dh": str(state.get("dhSelfPub", "")), + "pn": int(state.get("pn", 0) or 0), + "n": n, + "alg": "X25519", + } + ct = _aes_gcm_encrypt(mk, plaintext, _header_aad(header)) + wrapped = _b64(_stable_json({"h": header, "ct": ct}).encode("utf-8")) + state["updated"] = int(time.time() * 1000) + _set_state(peer_id, state) + return {"ok": True, "result": f"dr2:{wrapped}"} + + +def decrypt_wormhole_dm(peer_id: str, ciphertext: str) -> dict[str, Any]: + if not ciphertext.startswith("dr2:"): + return {"ok": False, "detail": "legacy"} + try: + raw = ciphertext[4:] + payload = json.loads(_unb64(raw).decode("utf-8")) + header = dict(payload.get("h") or {}) + ct = str(payload.get("ct") or "") + remote_dh = str(header.get("dh") or "") + pn = int(header.get("pn", 0) or 0) + n = int(header.get("n", 0) or 0) + + state = _get_state(peer_id) + if not state: + state = _init_receiver_state(peer_id, remote_dh) + + if remote_dh and remote_dh != str(state.get("dhRemote", "")): + state = _dh_ratchet(state, remote_dh, pn) + + skipped = dict(state.get("skipped") or {}) + skip_key = f"{remote_dh}:{n}" + if skip_key in skipped: + mk = _unb64(skipped.pop(skip_key)) + state["skipped"] = skipped + _set_state(peer_id, state) + return {"ok": True, "result": _aes_gcm_decrypt(mk, ct, _header_aad(header))} + + _skip_message_keys(state, n) + if not state.get("ckr"): + return {"ok": False, "detail": "no_receive_chain"} + next_ck, mk = _kdf_ck(_unb64(str(state.get("ckr", "")))) + state["ckr"] = _b64(next_ck) + state["nr"] = int(state.get("nr", 0) or 0) + 1 + state["updated"] = int(time.time() * 1000) + _set_state(peer_id, state) + return {"ok": True, "result": _aes_gcm_decrypt(mk, ct, _header_aad(header))} + except Exception as exc: + return {"ok": False, "detail": str(exc) or "ratchet_decrypt_failed"} diff --git a/backend/services/mesh/mesh_wormhole_seal.py b/backend/services/mesh/mesh_wormhole_seal.py new file mode 100644 index 00000000..388826d5 --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_seal.py @@ -0,0 +1,267 @@ +"""Wormhole-owned sender seal helpers.""" + +from __future__ import annotations + +import base64 +import json +import os +from typing import Any + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat + +from services.mesh.mesh_crypto import ( + parse_public_key_algo, + verify_node_binding, + verify_signature, +) +from services.mesh.mesh_protocol import PROTOCOL_VERSION +from services.mesh.mesh_wormhole_identity import ( + bootstrap_wormhole_identity, + read_wormhole_identity, + sign_wormhole_message, +) +from services.wormhole_settings import read_wormhole_settings + + +def _unb64(data: str | bytes | None) -> bytes: + if not data: + return b"" + if isinstance(data, bytes): + return base64.b64decode(data) + return base64.b64decode(data.encode("ascii")) + + +def _derive_aes_key(my_private_b64: str, peer_public_b64: str) -> bytes: + priv = x25519.X25519PrivateKey.from_private_bytes(_unb64(my_private_b64)) + pub = x25519.X25519PublicKey.from_public_bytes(_unb64(peer_public_b64)) + secret = priv.exchange(pub) + # For compatibility with the browser path, use the raw 32-byte X25519 secret directly + # as the AES-256-GCM key material. + return secret + + +def _seal_salt(recipient_id: str, msg_id: str, extra: str = "") -> bytes: + material = f"SB-SEAL-SALT|{recipient_id}|{msg_id}|{PROTOCOL_VERSION}|{extra}".encode("utf-8") + digest = hashes.Hash(hashes.SHA256()) + digest.update(material) + return digest.finalize() + + +def _derive_seal_key_v2(my_private_b64: str, peer_public_b64: str, recipient_id: str, msg_id: str) -> bytes: + priv = x25519.X25519PrivateKey.from_private_bytes(_unb64(my_private_b64)) + pub = x25519.X25519PublicKey.from_public_bytes(_unb64(peer_public_b64)) + secret = priv.exchange(pub) + return HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=_seal_salt(recipient_id, msg_id), + info=b"SB-SENDER-SEAL-V2", + ).derive(secret) + + +def _x25519_pair() -> tuple[str, str]: + priv = x25519.X25519PrivateKey.generate() + priv_raw = priv.private_bytes( + encoding=Encoding.Raw, + format=PrivateFormat.Raw, + encryption_algorithm=NoEncryption(), + ) + pub_raw = priv.public_key().public_bytes( + encoding=Encoding.Raw, + format=PublicFormat.Raw, + ) + return _b64(priv_raw), _b64(pub_raw) + + +def _derive_seal_key_v3( + my_private_b64: str, + peer_public_b64: str, + recipient_id: str, + msg_id: str, + ephemeral_pub_b64: str, +) -> bytes: + priv = x25519.X25519PrivateKey.from_private_bytes(_unb64(my_private_b64)) + pub = x25519.X25519PublicKey.from_public_bytes(_unb64(peer_public_b64)) + secret = priv.exchange(pub) + return HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=_seal_salt(recipient_id, msg_id, ephemeral_pub_b64), + info=b"SB-SENDER-SEAL-V3", + ).derive(secret) + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +def _seal_payload_version(sender_seal: str) -> tuple[str, str, str]: + value = str(sender_seal or "").strip() + if value.startswith("v3:"): + _, ephemeral_pub, encoded = value.split(":", 2) + return "v3", ephemeral_pub, encoded + if value.startswith("v2:"): + return "v2", "", value[3:] + return "legacy", "", value + + +def _legacy_seal_allowed() -> bool: + try: + settings = read_wormhole_settings() + if bool(settings.get("enabled")) or bool(settings.get("anonymous_mode")): + return False + except Exception: + pass + return True + + +def build_sender_seal( + *, + recipient_id: str, + recipient_dh_pub: str, + msg_id: str, + timestamp: int, +) -> dict[str, Any]: + recipient_id = str(recipient_id or "").strip() + recipient_dh_pub = str(recipient_dh_pub or "").strip() + msg_id = str(msg_id or "").strip() + timestamp = int(timestamp or 0) + if not recipient_id or not recipient_dh_pub or not msg_id or timestamp <= 0: + return {"ok": False, "detail": "recipient_id, recipient_dh_pub, msg_id, and timestamp required"} + + identity = read_wormhole_identity() + if not identity.get("bootstrapped"): + bootstrap_wormhole_identity() + identity = read_wormhole_identity() + my_private = str(identity.get("dh_private_key", "") or "") + if not my_private: + return {"ok": False, "detail": "Missing Wormhole DH private key"} + + try: + ephemeral_private, ephemeral_public = _x25519_pair() + signed = sign_wormhole_message( + f"seal|v3|{msg_id}|{timestamp}|{recipient_id}|{ephemeral_public}" + ) + if not verify_node_binding( + str(signed.get("node_id", "") or ""), + str(signed.get("public_key", "") or ""), + ): + return {"ok": False, "detail": "Sender seal node binding failed"} + key = _derive_seal_key_v3( + ephemeral_private, + recipient_dh_pub, + recipient_id, + msg_id, + ephemeral_public, + ) + plaintext = json.dumps( + { + "seal_version": "v3", + "ephemeral_pub_key": ephemeral_public, + "sender_id": str(signed.get("node_id", "") or ""), + "public_key": str(signed.get("public_key", "") or ""), + "public_key_algo": str(signed.get("public_key_algo", "") or ""), + "msg_id": msg_id, + "timestamp": timestamp, + "signature": str(signed.get("signature", "") or ""), + "protocol_version": str(signed.get("protocol_version", "") or ""), + }, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + iv = _b64(os.urandom(12)) + except Exception as exc: + return {"ok": False, "detail": str(exc) or "sender_seal_build_failed"} + + iv_bytes = _unb64(iv) + ciphertext = AESGCM(key).encrypt(iv_bytes, plaintext, None) + combined = iv_bytes + ciphertext + return { + "ok": True, + "sender_seal": f"v3:{ephemeral_public}:{_b64(combined)}", + "sender_id": str(signed.get("node_id", "") or ""), + "public_key": str(signed.get("public_key", "") or ""), + "public_key_algo": str(signed.get("public_key_algo", "") or ""), + "protocol_version": str(signed.get("protocol_version", "") or ""), + } + + +def open_sender_seal( + *, + sender_seal: str, + candidate_dh_pub: str, + recipient_id: str, + expected_msg_id: str, +) -> dict[str, Any]: + if not sender_seal or not candidate_dh_pub or not recipient_id or not expected_msg_id: + return {"ok": False, "detail": "Missing sender_seal, candidate_dh_pub, recipient_id, or expected_msg_id"} + + identity = read_wormhole_identity() + if not identity.get("bootstrapped"): + bootstrap_wormhole_identity() + identity = read_wormhole_identity() + my_private = str(identity.get("dh_private_key", "") or "") + if not my_private: + return {"ok": False, "detail": "Missing Wormhole DH private key"} + + try: + seal_version, ephemeral_pub, encoded = _seal_payload_version(sender_seal) + if seal_version == "v3": + key = _derive_seal_key_v3(my_private, ephemeral_pub, recipient_id, expected_msg_id, ephemeral_pub) + elif seal_version == "v2": + key = _derive_seal_key_v2(my_private, candidate_dh_pub, recipient_id, expected_msg_id) + else: + if not _legacy_seal_allowed(): + return {"ok": False, "detail": "Legacy sender seals are disabled in hardened modes"} + key = _derive_aes_key(my_private, candidate_dh_pub) + combined = _unb64(encoded) + iv = combined[:12] + ciphertext = combined[12:] + plaintext = AESGCM(key).decrypt(iv, ciphertext, None).decode("utf-8") + seal = json.loads(plaintext) + except Exception as exc: + return {"ok": False, "detail": str(exc) or "sender_seal_decrypt_failed"} + + sender_id = str(seal.get("sender_id", "") or "") + public_key = str(seal.get("public_key", "") or "") + public_key_algo = str(seal.get("public_key_algo", "") or "") + msg_id = str(seal.get("msg_id", "") or "") + timestamp = int(seal.get("timestamp", 0) or 0) + signature = str(seal.get("signature", "") or "") + if not sender_id or not public_key or not public_key_algo or not msg_id or not signature: + return {"ok": False, "detail": "Malformed sender seal"} + if msg_id != expected_msg_id: + return {"ok": False, "detail": "Sender seal message mismatch"} + if seal_version == "v3" and str(seal.get("ephemeral_pub_key", "") or "") != ephemeral_pub: + return {"ok": False, "detail": "Sender seal ephemeral key mismatch"} + + if not verify_node_binding(sender_id, public_key): + return {"ok": True, "sender_id": sender_id, "seal_verified": False} + + algo = parse_public_key_algo(public_key_algo) + if not algo: + return {"ok": True, "sender_id": sender_id, "seal_verified": False} + + if seal_version == "v3": + message = f"seal|v3|{msg_id}|{timestamp}|{recipient_id}|{ephemeral_pub}" + else: + message = f"seal|{msg_id}|{timestamp}|{recipient_id}" + verified = verify_signature( + public_key_b64=public_key, + public_key_algo=algo, + signature_hex=signature, + payload=message, + ) + return { + "ok": True, + "sender_id": sender_id, + "seal_verified": bool(verified), + "public_key": public_key, + "public_key_algo": public_key_algo, + "timestamp": timestamp, + "msg_id": msg_id, + } diff --git a/backend/services/mesh/mesh_wormhole_sender_token.py b/backend/services/mesh/mesh_wormhole_sender_token.py new file mode 100644 index 00000000..bdba4f59 --- /dev/null +++ b/backend/services/mesh/mesh_wormhole_sender_token.py @@ -0,0 +1,134 @@ +"""Short-lived Wormhole sender tokens for DM metadata reduction. + +These tokens let the client send a sealed DM without placing the long-term +sender id and public key directly into the DM send request body. The token is +single-use, recipient-bound, and kept in memory only. +""" + +from __future__ import annotations + +import hashlib +import secrets +import time +from typing import Any + +from cachetools import TTLCache + +from services.mesh.mesh_wormhole_identity import bootstrap_wormhole_identity, read_wormhole_identity +from services.mesh.mesh_protocol import PROTOCOL_VERSION + +_SENDER_TOKEN_TTL_S = 5 * 60 +_sender_tokens: TTLCache[str, dict[str, Any]] = TTLCache(maxsize=2048, ttl=_SENDER_TOKEN_TTL_S) + + +def _sender_token_hash(token: str) -> str: + return hashlib.sha256((token or "").encode("utf-8")).hexdigest() + + +def _token_binding_hash(recipient_token: str) -> str: + return hashlib.sha256((recipient_token or "").encode("utf-8")).hexdigest() + + +def issue_wormhole_dm_sender_token( + *, + recipient_id: str, + delivery_class: str, + recipient_token: str = "", + ttl_seconds: int = _SENDER_TOKEN_TTL_S, +) -> dict[str, Any]: + recipient_id = str(recipient_id or "").strip() + delivery_class = str(delivery_class or "").strip().lower() + if delivery_class not in ("request", "shared"): + return {"ok": False, "detail": "Invalid delivery_class"} + if not recipient_id: + return {"ok": False, "detail": "recipient_id required"} + if delivery_class == "shared" and not recipient_token: + return {"ok": False, "detail": "recipient_token required for shared delivery"} + + data = read_wormhole_identity() + if not data.get("bootstrapped"): + bootstrap_wormhole_identity() + data = read_wormhole_identity() + if not data.get("node_id") or not data.get("public_key"): + return {"ok": False, "detail": "Wormhole identity unavailable"} + + token = secrets.token_urlsafe(32) + now = int(time.time()) + expires_at = now + max(30, min(int(ttl_seconds or _SENDER_TOKEN_TTL_S), _SENDER_TOKEN_TTL_S)) + _sender_tokens[token] = { + "sender_id": str(data.get("node_id", "")), + "public_key": str(data.get("public_key", "")), + "public_key_algo": str(data.get("public_key_algo", "Ed25519") or "Ed25519"), + "protocol_version": PROTOCOL_VERSION, + "recipient_id": recipient_id, + "delivery_class": delivery_class, + "recipient_token_hash": _token_binding_hash(recipient_token), + "issued_at": now, + "expires_at": expires_at, + } + return { + "ok": True, + "sender_token": token, + "expires_at": expires_at, + "delivery_class": delivery_class, + } + + +def issue_wormhole_dm_sender_tokens( + *, + recipient_id: str, + delivery_class: str, + recipient_token: str = "", + count: int = 3, + ttl_seconds: int = _SENDER_TOKEN_TTL_S, +) -> dict[str, Any]: + token_count = max(1, min(int(count or 1), 4)) + tokens: list[dict[str, Any]] = [] + for _ in range(token_count): + issued = issue_wormhole_dm_sender_token( + recipient_id=recipient_id, + delivery_class=delivery_class, + recipient_token=recipient_token, + ttl_seconds=ttl_seconds, + ) + if not issued.get("ok"): + return issued + tokens.append( + { + "sender_token": str(issued.get("sender_token", "")), + "expires_at": int(issued.get("expires_at", 0) or 0), + } + ) + return { + "ok": True, + "delivery_class": delivery_class, + "tokens": tokens, + } + + +def consume_wormhole_dm_sender_token( + *, + sender_token: str, + recipient_id: str, + delivery_class: str, + recipient_token: str = "", +) -> dict[str, Any]: + token = str(sender_token or "").strip() + if not token: + return {"ok": False, "detail": "sender_token required"} + token_hash = _sender_token_hash(token) + record = _sender_tokens.pop(token, None) + if not record: + return {"ok": False, "detail": "sender_token invalid or expired"} + bound_recipient_id = str(record.get("recipient_id", "") or "") + normalized_recipient_id = str(recipient_id or "").strip() + if normalized_recipient_id and bound_recipient_id != normalized_recipient_id: + return {"ok": False, "detail": "sender_token recipient mismatch"} + if str(record.get("delivery_class", "")) != str(delivery_class or "").strip().lower(): + return {"ok": False, "detail": "sender_token delivery_class mismatch"} + if str(record.get("recipient_token_hash", "")) != _token_binding_hash(recipient_token): + return {"ok": False, "detail": "sender_token mailbox binding mismatch"} + expires_at = int(record.get("expires_at", 0) or 0) + if expires_at and expires_at < int(time.time()): + return {"ok": False, "detail": "sender_token expired"} + return {"ok": True, "sender_token_hash": token_hash, **record} diff --git a/backend/services/mesh/meshtastic_topics.py b/backend/services/mesh/meshtastic_topics.py new file mode 100644 index 00000000..ea08eacb --- /dev/null +++ b/backend/services/mesh/meshtastic_topics.py @@ -0,0 +1,176 @@ +"""Helpers for Meshtastic MQTT roots, topic parsing, and subscriptions.""" + +from __future__ import annotations + +import re +from typing import Iterable + +# Official/default region roots we actively watch on the public broker. +DEFAULT_ROOTS: tuple[str, ...] = ( + "US", + "EU_868", + "EU_433", + "CN", + "JP", + "KR", + "TW", + "RU", + "IN", + "ANZ", + "ANZ_433", + "NZ_865", + "TH", + "UA_868", + "UA_433", + "MY_433", + "MY_919", + "SG_923", + "LORA_24", +) + +# Legacy/community roots still seen in the wild on public/community brokers. +COMMUNITY_ROOTS: tuple[str, ...] = ( + "EU", + "AU", + "UA", + "BR", + "AF", + "ME", + "SEA", + "SA", + "PL", +) + +_ROOT_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_+\-]+$") +_TOPIC_SEGMENT_RE = re.compile(r"^[A-Za-z0-9_+\-#]+$") + + +def _dedupe(values: Iterable[str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for value in values: + if value not in seen: + out.append(value) + seen.add(value) + return out + + +def _split_config_values(raw: str) -> list[str]: + if not raw: + return [] + normalized = raw.replace("\n", ",").replace(";", ",") + return [item.strip() for item in normalized.split(",") if item.strip()] + + +def normalize_root(value: str) -> str | None: + """Normalize a Meshtastic root like `PL` or `US/rob/snd`.""" + + raw = str(value or "").strip() + if not raw: + return None + if raw.startswith("msh/"): + raw = raw[4:] + raw = raw.strip("/") + if raw.endswith("/#"): + raw = raw[:-2].rstrip("/") + if not raw: + return None + parts = [part for part in raw.split("/") if part] + if not parts: + return None + if any(part in {"+", "#"} for part in parts): + return None + if any(not _ROOT_SEGMENT_RE.match(part) for part in parts): + return None + return "/".join(parts) + + +def normalize_topic_filter(value: str) -> str | None: + """Normalize a full MQTT subscription filter.""" + + raw = str(value or "").strip() + if not raw: + return None + if not raw.startswith("msh/"): + root = normalize_root(raw) + return f"msh/{root}/#" if root else None + raw = raw.strip("/") + parts = [part for part in raw.split("/") if part] + if not parts or parts[0] != "msh": + return None + if any(part != "+" and not _TOPIC_SEGMENT_RE.match(part) for part in parts[1:]): + return None + return "/".join(parts) + + +def build_subscription_topics( + extra_roots: str = "", + extra_topics: str = "", + include_defaults: bool = True, +) -> list[str]: + roots: list[str] = [] + if include_defaults: + roots.extend(DEFAULT_ROOTS) + roots.extend(COMMUNITY_ROOTS) + roots.extend(root for root in (normalize_root(item) for item in _split_config_values(extra_roots)) if root) + + topics = [f"msh/{root}/#" for root in _dedupe(roots)] + topics.extend( + topic + for topic in ( + normalize_topic_filter(item) for item in _split_config_values(extra_topics) + ) + if topic + ) + return _dedupe(topics) + + +def known_roots(extra_roots: str = "", include_defaults: bool = True) -> list[str]: + topics = build_subscription_topics(extra_roots=extra_roots, include_defaults=include_defaults) + roots: list[str] = [] + for topic in topics: + if not topic.startswith("msh/") or not topic.endswith("/#"): + continue + root = normalize_root(topic[4:-2]) + if root: + roots.append(root) + return _dedupe(roots) + + +def parse_topic_metadata(topic: str) -> dict[str, str]: + """Extract region/root/channel metadata from a Meshtastic MQTT topic.""" + + parts = [part for part in str(topic or "").strip("/").split("/") if part] + if not parts or parts[0] != "msh": + return {"region": "?", "root": "?", "channel": "LongFast", "mode": "", "version": ""} + + mode_idx = -1 + for idx in range(1, len(parts)): + if parts[idx] in {"e", "c", "json"}: + mode_idx = idx + break + + version = "" + root_parts = parts[1:] + channel = "LongFast" + mode = "" + if mode_idx != -1: + mode = parts[mode_idx] + maybe_version_idx = mode_idx - 1 + if maybe_version_idx >= 1 and parts[maybe_version_idx].isdigit(): + version = parts[maybe_version_idx] + root_parts = parts[1:maybe_version_idx] + else: + root_parts = parts[1:mode_idx] + if len(parts) > mode_idx + 1: + channel = parts[mode_idx + 1] + + root = "/".join(root_parts) if root_parts else "?" + region = root_parts[0] if root_parts else "?" + return { + "region": region, + "root": root, + "channel": channel or "LongFast", + "mode": mode, + "version": version, + } diff --git a/backend/services/network_utils.py b/backend/services/network_utils.py index e6966ae1..664dd590 100644 --- a/backend/services/network_utils.py +++ b/backend/services/network_utils.py @@ -34,6 +34,11 @@ # Lock protecting _domain_fail_cache and _circuit_breaker mutations _cb_lock = threading.Lock() + +class UpstreamCircuitBreakerError(OSError): + """Raised when a domain recently failed hard and is temporarily skipped.""" + + class _DummyResponse: """Minimal response object matching requests.Response interface.""" def __init__(self, status_code, text): @@ -67,7 +72,9 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None) # Circuit breaker: if domain failed completely <2min ago, fail fast with _cb_lock: if domain in _circuit_breaker and (time.time() - _circuit_breaker[domain]) < _CIRCUIT_BREAKER_TTL: - raise Exception(f"Circuit breaker open for {domain} (failed <{_CIRCUIT_BREAKER_TTL}s ago)") + raise UpstreamCircuitBreakerError( + f"Circuit breaker open for {domain} (failed <{_CIRCUIT_BREAKER_TTL}s ago)" + ) # Check if this domain recently failed with requests — skip straight to curl with _cb_lock: @@ -81,6 +88,9 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None) res = _session.post(url, json=json_data, timeout=req_timeout, headers=default_headers) else: res = _session.get(url, timeout=req_timeout, headers=default_headers) + if res.status_code == 429: + logger.warning(f"Upstream rate limit hit for {url}; not bypassing with curl.") + return res res.raise_for_status() # Clear failure caches on success with _cb_lock: @@ -106,9 +116,9 @@ def fetch_with_curl(url, method="GET", json_data=None, timeout=15, headers=None) stdin_data = json.dumps(json_data) if (method == "POST" and json_data) else None res = subprocess.run( cmd, capture_output=True, text=True, timeout=timeout + 5, - input=stdin_data + input=stdin_data, encoding="utf-8", errors="replace" ) - if res.returncode == 0 and res.stdout.strip(): + if res.returncode == 0 and (res.stdout or "").strip(): # Parse HTTP status code from -w output (last line) lines = res.stdout.rstrip().rsplit("\n", 1) body = lines[0] if len(lines) > 1 else res.stdout diff --git a/backend/services/news_feed_config.py b/backend/services/news_feed_config.py index 6522eb4e..3e21d3ec 100644 --- a/backend/services/news_feed_config.py +++ b/backend/services/news_feed_config.py @@ -2,6 +2,7 @@ News feed configuration — manages the user-customisable RSS feed list. Feeds are stored in backend/config/news_feeds.json and persist across restarts. """ + import json import logging from pathlib import Path @@ -9,7 +10,18 @@ logger = logging.getLogger(__name__) CONFIG_PATH = Path(__file__).parent.parent / "config" / "news_feeds.json" -MAX_FEEDS = 25 +MAX_FEEDS = 50 +_FEED_URL_REPLACEMENTS = { + "https://www.channelnewsasia.com/rssfeed/8395986": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml", +} +_DEAD_FEED_URLS = { + "https://www3.nhk.or.jp/nhkworld/rss/world.xml", + "https://focustaiwan.tw/rss", + "https://english.kyodonews.net/rss/news.xml", + "https://www.stripes.com/feeds/pacific.rss", + "https://asia.nikkei.com/rss", + "https://www.taipeitimes.com/xml/pda.rss", +} DEFAULT_FEEDS = [ {"name": "NPR", "url": "https://feeds.npr.org/1004/rss.xml", "weight": 4}, @@ -17,23 +29,40 @@ {"name": "AlJazeera", "url": "https://www.aljazeera.com/xml/rss/all.xml", "weight": 2}, {"name": "NYT", "url": "https://rss.nytimes.com/services/xml/rss/nyt/World.xml", "weight": 1}, {"name": "GDACS", "url": "https://www.gdacs.org/xml/rss.xml", "weight": 5}, - {"name": "NHK", "url": "https://www3.nhk.or.jp/nhkworld/rss/world.xml", "weight": 3}, - {"name": "CNA", "url": "https://www.channelnewsasia.com/rssfeed/8395986", "weight": 3}, + {"name": "CNA", "url": "https://www.channelnewsasia.com/api/v1/rss-outbound-feed?_format=xml", "weight": 3}, {"name": "Mercopress", "url": "https://en.mercopress.com/rss/", "weight": 3}, - {"name": "FocusTaiwan", "url": "https://focustaiwan.tw/rss", "weight": 5}, - {"name": "Kyodo", "url": "https://english.kyodonews.net/rss/news.xml", "weight": 4}, {"name": "SCMP", "url": "https://www.scmp.com/rss/91/feed", "weight": 4}, {"name": "The Diplomat", "url": "https://thediplomat.com/feed/", "weight": 4}, - {"name": "Stars and Stripes", "url": "https://www.stripes.com/feeds/pacific.rss", "weight": 4}, {"name": "Yonhap", "url": "https://en.yna.co.kr/RSS/news.xml", "weight": 4}, - {"name": "Nikkei Asia", "url": "https://asia.nikkei.com/rss", "weight": 3}, - {"name": "Taipei Times", "url": "https://www.taipeitimes.com/xml/pda.rss", "weight": 4}, {"name": "Asia Times", "url": "https://asiatimes.com/feed/", "weight": 3}, {"name": "Defense News", "url": "https://www.defensenews.com/arc/outboundfeeds/rss/", "weight": 3}, {"name": "Japan Times", "url": "https://www.japantimes.co.jp/feed/", "weight": 3}, + {"name": "CSM", "url": "https://www.csmonitor.com/rss/world", "weight": 4}, + {"name": "PBS NewsHour", "url": "https://www.pbs.org/newshour/feeds/rss/world", "weight": 4}, + {"name": "France 24", "url": "https://www.france24.com/en/rss", "weight": 4}, + {"name": "DW", "url": "https://rss.dw.com/xml/rss-en-world", "weight": 4}, ] +def _normalise_feeds(feeds: list[dict]) -> list[dict]: + cleaned: list[dict] = [] + for feed in feeds: + if not isinstance(feed, dict): + continue + item = dict(feed) + url = str(item.get("url", "")).strip() + if not url: + continue + if url in _FEED_URL_REPLACEMENTS: + item["url"] = _FEED_URL_REPLACEMENTS[url] + url = item["url"] + if url in _DEAD_FEED_URLS: + logger.warning("Dropping dead RSS feed URL from configuration: %s", url) + continue + cleaned.append(item) + return cleaned + + def get_feeds() -> list[dict]: """Load feeds from config file, falling back to defaults.""" try: @@ -41,7 +70,10 @@ def get_feeds() -> list[dict]: data = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) feeds = data.get("feeds", []) if isinstance(data, dict) else data if isinstance(feeds, list) and len(feeds) > 0: - return feeds + normalised = _normalise_feeds(feeds) + if normalised != feeds: + save_feeds(normalised) + return normalised except (IOError, OSError, json.JSONDecodeError, ValueError) as e: logger.warning(f"Failed to read news feed config: {e}") return list(DEFAULT_FEEDS) @@ -51,6 +83,7 @@ def save_feeds(feeds: list[dict]) -> bool: """Validate and save feeds to config file. Returns True on success.""" if not isinstance(feeds, list): return False + feeds = _normalise_feeds(feeds) if len(feeds) > MAX_FEEDS: return False # Validate each feed entry diff --git a/backend/services/node_settings.py b/backend/services/node_settings.py new file mode 100644 index 00000000..f9fe1a9b --- /dev/null +++ b/backend/services/node_settings.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +DATA_DIR = Path(__file__).parent.parent / "data" +NODE_FILE = DATA_DIR / "node.json" +_cache: dict | None = None +_cache_ts: float = 0.0 +_CACHE_TTL = 5.0 +_DEFAULTS = { + "enabled": False, +} + + +def _safe_int(value: object, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def read_node_settings() -> dict: + global _cache, _cache_ts + now = time.monotonic() + if _cache is not None and (now - _cache_ts) < _CACHE_TTL: + return _cache + if not NODE_FILE.exists(): + result = {**_DEFAULTS, "updated_at": 0} + else: + try: + data = json.loads(NODE_FILE.read_text(encoding="utf-8")) + except Exception: + result = {**_DEFAULTS, "updated_at": 0} + else: + result = { + "enabled": bool(data.get("enabled", _DEFAULTS["enabled"])), + "updated_at": _safe_int(data.get("updated_at", 0) or 0), + } + _cache = result + _cache_ts = now + return result + + +def write_node_settings(*, enabled: bool | None = None) -> dict: + DATA_DIR.mkdir(parents=True, exist_ok=True) + existing = read_node_settings() + payload = { + "enabled": bool(existing.get("enabled", _DEFAULTS["enabled"])) if enabled is None else bool(enabled), + "updated_at": int(time.time()), + } + NODE_FILE.write_text(json.dumps(payload, indent=2), encoding="utf-8") + global _cache, _cache_ts + _cache = payload + _cache_ts = time.monotonic() + return payload diff --git a/backend/services/oracle_service.py b/backend/services/oracle_service.py new file mode 100644 index 00000000..56679204 --- /dev/null +++ b/backend/services/oracle_service.py @@ -0,0 +1,395 @@ +"""Oracle Service — deterministic intelligence ranking for news items. + +Enriches news items with: +- oracle_score: risk_score weighted by source confidence (0–10) +- sentiment: VADER compound score (-1.0 to +1.0) +- prediction_odds: matched prediction market probabilities (or None) +- machine_assessment: structured human-readable analysis string +""" + +import logging + +logger = logging.getLogger(__name__) + +_analyzer = None + + +def _get_analyzer(): + global _analyzer + if _analyzer is None: + from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + _analyzer = SentimentIntensityAnalyzer() + return _analyzer + + +def compute_sentiment(headline: str) -> float: + """VADER compound sentiment score for a headline. Range: -1.0 to +1.0.""" + if not headline: + return 0.0 + return _get_analyzer().polarity_scores(headline)["compound"] + + +def compute_oracle_score(risk_score: int, source_weight: float) -> float: + """Weighted oracle score: risk_score scaled by source confidence. + + source_weight is 1–5 (from feed config). Normalised to 0.2–1.0 multiplier. + Result range: 0.0–10.0. + """ + multiplier = source_weight / 5.0 # 1→0.2, 5→1.0 + return round(risk_score * multiplier, 1) + + +_STOP_WORDS = frozenset({ + "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", + "being", "have", "has", "had", "do", "does", "did", "will", "would", + "could", "should", "may", "might", "shall", "can", "this", "that", + "these", "those", "it", "its", "if", "not", "no", "so", "as", "up", + "out", "about", "into", "over", "after", "before", "between", "under", + "than", "then", "more", "most", "other", "some", "such", "only", "own", + "same", "also", "just", "how", "what", "which", "who", "whom", "when", + "where", "why", "all", "each", "every", "both", "few", "many", "much", + "any", "very", "too", "here", "there", "now", "new", "says", "said", + "-", "--", "—", "vs", "vs.", "&", "he", "she", "they", "we", "you", + "his", "her", "my", "our", "your", "their", "him", "us", "them", +}) + + +def _tokenize(text: str) -> set[str]: + """Lowercase, strip punctuation, remove stop words.""" + import re + words = re.findall(r"[a-z0-9]+(?:'[a-z]+)?", text.lower()) + return {w for w in words if w not in _STOP_WORDS and len(w) > 1} + + +def _match_prediction_markets(title: str, markets: list[dict]) -> dict | None: + """Find best-matching prediction market for a news headline. + + Uses Jaccard similarity on meaningful tokens (stop words removed). + Requires at least 2 meaningful keyword overlaps AND Jaccard >= 0.15. + """ + if not markets or not title: + return None + + title_words = _tokenize(title) + if len(title_words) < 2: + return None + + best_match = None + best_score = 0.0 + + for market in markets: + market_title = market.get("title", "") + market_words = _tokenize(market_title) + if len(market_words) < 2: + continue + + intersection = title_words & market_words + if len(intersection) < 2: + continue + + union = title_words | market_words + jaccard = len(intersection) / len(union) if union else 0.0 + + if jaccard > best_score and jaccard >= 0.15: + best_score = jaccard + best_match = market + + if not best_match: + return None + + return { + "title": best_match.get("title", ""), + "polymarket_pct": best_match.get("polymarket_pct"), + "kalshi_pct": best_match.get("kalshi_pct"), + "consensus_pct": best_match.get("consensus_pct"), + "match_score": round(best_score, 2), + } + + +def _build_assessment(oracle_score: float, sentiment: float, prediction: dict | None) -> str: + """Build structured machine_assessment string.""" + parts = [] + + # Oracle tier + if oracle_score >= 7: + tier = "CRITICAL" + elif oracle_score >= 4: + tier = "ELEVATED" + else: + tier = "ROUTINE" + parts.append(f"ORACLE: {oracle_score}/10 [{tier}]") + + # Sentiment + if sentiment >= 0.05: + sdir = "POSITIVE" + elif sentiment <= -0.05: + sdir = "NEGATIVE" + else: + sdir = "NEUTRAL" + parts.append(f"SENTIMENT: {sentiment:+.2f} [{sdir}]") + + # Prediction market + if prediction: + consensus = prediction.get("consensus_pct") + if consensus is not None: + parts.append(f"MKT CONSENSUS: {consensus}%") + poly = prediction.get("polymarket_pct") + kalshi = prediction.get("kalshi_pct") + sources = [] + if poly is not None: + sources.append(f"Polymarket {poly}%") + if kalshi is not None: + sources.append(f"Kalshi {kalshi}%") + if sources: + parts.append(f" Sources: {' | '.join(sources)}") + + return " // ".join(parts[:3]) + ("\n" + parts[3] if len(parts) > 3 else "") + + +def enrich_news_items( + news_items: list[dict], source_weights: dict[str, float], markets: list[dict] | None = None +) -> list[dict]: + """Enrich news items with oracle scores, sentiment, and prediction market odds. + + Args: + news_items: list of news item dicts (modified in-place) + source_weights: {source_name: weight} from feed config (1–5 scale) + markets: merged prediction market events list (or None) + + Returns: + The same list, enriched with oracle_score, sentiment, prediction_odds, machine_assessment. + """ + if markets is None: + markets = [] + + for item in news_items: + title = item.get("title", "") + source = item.get("source", "") + risk_score = item.get("risk_score", 1) + weight = source_weights.get(source, 3) # default weight 3 (mid-range) + + sentiment = compute_sentiment(title) + oracle_score = compute_oracle_score(risk_score, weight) + prediction = _match_prediction_markets(title, markets) + + item["sentiment"] = sentiment + item["oracle_score"] = oracle_score + item["prediction_odds"] = prediction + item["machine_assessment"] = _build_assessment(oracle_score, sentiment, prediction) + + return news_items + + +# --------------------------------------------------------------------------- +# Global threat level +# --------------------------------------------------------------------------- + +_THREAT_TIERS = [ + (80, "SEVERE", "#ef4444"), # red + (60, "HIGH", "#f97316"), # orange + (40, "ELEVATED", "#eab308"), # yellow + (20, "GUARDED", "#3b82f6"), # blue + (0, "GREEN", "#22c55e"), # green +] + + +def compute_global_threat_level( + news_items: list[dict], + markets: list[dict] | None = None, + military_flights: list[dict] | None = None, + gps_jamming: list[dict] | None = None, + ships: list[dict] | None = None, + correlations: list[dict] | None = None, +) -> dict: + """Fuse news sentiment, prediction-market conflict odds, event frequency, + military activity, GPS jamming, and cross-layer correlations into a single + 0-100 threat score. + + Formula (weights sum to 1.0): + 0.25 × negative_sentiment_intensity + 0.25 × conflict_market_avg_probability + 0.10 × high_risk_event_ratio + 0.10 × max_oracle_score (normalised to 0-100) + 0.10 × military_activity_anomaly + 0.10 × gps_jamming_indicator + 0.10 × correlation_alerts + """ + if not news_items: + return {"score": 0, "level": "GREEN", "color": "#22c55e", "drivers": []} + + # --- Component 1: negative sentiment intensity (0-100) --- + neg_scores = [abs(it.get("sentiment", 0)) for it in news_items if (it.get("sentiment") or 0) <= -0.05] + neg_intensity = (sum(neg_scores) / len(news_items)) * 100 if news_items else 0 + neg_intensity = min(100, neg_intensity * 2.5) # scale up — avg abs sentiment rarely > 0.4 + + # --- Component 2: conflict market avg probability (0-100) --- + conflict_probs: list[float] = [] + for m in (markets or []): + if m.get("category") == "CONFLICT": + pct = m.get("consensus_pct") or m.get("polymarket_pct") or m.get("kalshi_pct") + if pct is not None: + conflict_probs.append(float(pct)) + conflict_avg = sum(conflict_probs) / len(conflict_probs) if conflict_probs else 0 + + # --- Component 3: high-risk event ratio (0-100) --- + high_risk = sum(1 for it in news_items if (it.get("risk_score") or 0) >= 7) + event_ratio = (high_risk / len(news_items)) * 100 if news_items else 0 + + # --- Component 4: max oracle score (0-100) --- + max_oracle = max((it.get("oracle_score") or 0) for it in news_items) + max_oracle_pct = max_oracle * 10 # 0-10 → 0-100 + + # --- Component 5: military activity anomaly (0-100) --- + mil_count = len(military_flights or []) + # Baseline: ~20-50 military flights is normal. Spike above 80 is anomalous. + mil_anomaly = min(100, max(0, (mil_count - 30) * 2)) if mil_count > 30 else 0 + + # --- Component 6: GPS jamming indicator (0-100) --- + jam_zones = gps_jamming or [] + high_jam = sum(1 for z in jam_zones if z.get("severity") == "high") + med_jam = sum(1 for z in jam_zones if z.get("severity") == "medium") + jam_score = min(100, high_jam * 25 + med_jam * 10) + + # --- Component 7: cross-layer correlation alerts (0-100) --- + corr_list: list[dict] = correlations if correlations else [] + corr_points = sum( + 15 if a.get("severity") == "high" else 8 if a.get("severity") == "medium" else 3 + for a in corr_list + ) + corr_score = min(100, corr_points) + + # --- Weighted fusion --- + score = ( + 0.25 * neg_intensity + + 0.25 * conflict_avg + + 0.10 * event_ratio + + 0.10 * max_oracle_pct + + 0.10 * mil_anomaly + + 0.10 * jam_score + + 0.10 * corr_score + ) + score = max(0, min(100, round(score))) + + # --- Tier --- + level, color = "GREEN", "#22c55e" + for threshold, name, c in _THREAT_TIERS: + if score >= threshold: + level, color = name, c + break + + # --- Drivers (top reasons for current level) --- + drivers: list[str] = [] + if high_risk: + drivers.append(f"{high_risk} CRITICAL-tier news item{'s' if high_risk != 1 else ''}") + if conflict_avg >= 30: + drivers.append(f"CONFLICT markets avg {conflict_avg:.0f}%") + if neg_intensity >= 40: + drivers.append(f"Negative sentiment intensity {neg_intensity:.0f}/100") + if max_oracle >= 7: + drivers.append(f"Max oracle score {max_oracle}/10") + if mil_anomaly >= 30: + drivers.append(f"Military flight spike: {mil_count} tracked") + if jam_score >= 25: + drivers.append(f"GPS jamming: {high_jam} HIGH + {med_jam} MED zones") + if corr_score >= 15: + corr_high = sum(1 for a in corr_list if a.get("severity") == "high") + corr_med = sum(1 for a in corr_list if a.get("severity") == "medium") + drivers.append(f"Cross-layer correlations: {corr_high} HIGH + {corr_med} MED") + if not drivers: + drivers.append("Baseline — no significant threat indicators") + + return { + "score": score, + "level": level, + "color": color, + "drivers": drivers[:4], + } + + +def detect_breaking_events(news_items: list[dict]) -> None: + """Mark news items as 'breaking' when multiple credible sources converge. + + Criteria: cluster_count >= 3 AND risk_score >= 7. + Modifies items in-place by setting ``breaking = True``. + """ + for item in news_items: + cluster = item.get("cluster_count", 1) + risk = item.get("risk_score", 0) + if cluster >= 3 and risk >= 7: + item["breaking"] = True + + +# --------------------------------------------------------------------------- +# Region oracle intel (for map entity tooltips) +# --------------------------------------------------------------------------- + +_region_cache: dict[str, tuple[float, dict]] = {} # "lat,lng" -> (timestamp, result) +_REGION_CACHE_TTL = 60 # seconds +_REGION_RADIUS_DEG = 5.0 # ~500km at equator + + +def get_region_oracle_intel(lat: float, lng: float, news_items: list[dict]) -> dict: + """Get oracle intelligence summary for a geographic region. + + Finds news items within ~5 degrees, returns top oracle_score item, + average sentiment, and best market match. Cached on 0.5-degree grid. + """ + import time + + # Grid-snap for cache key (0.5 degree grid) + grid_lat = round(lat * 2) / 2 + grid_lng = round(lng * 2) / 2 + cache_key = f"{grid_lat},{grid_lng}" + + now = time.time() + if cache_key in _region_cache: + ts, cached_result = _region_cache[cache_key] + if now - ts < _REGION_CACHE_TTL: + return cached_result + + # Find nearby news items + nearby = [] + for item in news_items: + coords = item.get("coords") + if not coords or len(coords) < 2: + continue + ilat, ilng = coords[0], coords[1] + if abs(ilat - lat) <= _REGION_RADIUS_DEG and abs(ilng - lng) <= _REGION_RADIUS_DEG: + nearby.append(item) + + if not nearby: + result = {"found": False} + _region_cache[cache_key] = (now, result) + return result + + # Top oracle score item + top = max(nearby, key=lambda x: x.get("oracle_score", 0)) + avg_sentiment = sum(it.get("sentiment", 0) for it in nearby) / len(nearby) + + # Best market match from nearby items + best_market = None + for it in nearby: + po = it.get("prediction_odds") + if po and po.get("consensus_pct") is not None: + if best_market is None or (po.get("consensus_pct") or 0) > (best_market.get("consensus_pct") or 0): + best_market = po + + # Oracle tier + oracle_score = top.get("oracle_score", 0) + tier = "CRITICAL" if oracle_score >= 7 else "ELEVATED" if oracle_score >= 4 else "ROUTINE" + + result = { + "found": True, + "top_headline": top.get("title", ""), + "oracle_score": oracle_score, + "tier": tier, + "avg_sentiment": round(avg_sentiment, 2), + "nearby_count": len(nearby), + "market": { + "title": best_market.get("title", ""), + "consensus_pct": best_market.get("consensus_pct"), + } if best_market else None, + } + _region_cache[cache_key] = (now, result) + return result diff --git a/backend/services/privacy_core_client.py b/backend/services/privacy_core_client.py new file mode 100644 index 00000000..13850da9 --- /dev/null +++ b/backend/services/privacy_core_client.py @@ -0,0 +1,440 @@ +"""ctypes bridge for the Rust privacy-core crate. + +This module follows the architecture docs in extra/docs-internal: +- Python orchestrates +- Rust owns private protocol state +- Python sees opaque integer handles and serialized ciphertext only +""" + +from __future__ import annotations + +import ctypes +import json +import os +from pathlib import Path +from typing import Iterable + + +class PrivacyCoreError(RuntimeError): + """Raised when the Rust privacy-core returns an error.""" + + +class PrivacyCoreUnavailable(PrivacyCoreError): + """Raised when the shared library cannot be found or loaded.""" + + +class _ByteBuffer(ctypes.Structure): + _fields_ = [ + ("data", ctypes.POINTER(ctypes.c_uint8)), + ("len", ctypes.c_size_t), + ] + + +class PrivacyCoreClient: + """Handle-based interface to the local Rust privacy-core.""" + + def __init__(self, library: ctypes.CDLL, library_path: Path) -> None: + self._library = library + self.library_path = library_path + self._configure_functions() + + @classmethod + def load(cls, library_path: str | os.PathLike[str] | None = None) -> "PrivacyCoreClient": + resolved = cls._resolve_library_path(library_path) + try: + library = ctypes.CDLL(str(resolved)) + except OSError as exc: + raise PrivacyCoreUnavailable(f"failed to load privacy-core library: {resolved}") from exc + return cls(library, resolved) + + @staticmethod + def _resolve_library_path(library_path: str | os.PathLike[str] | None) -> Path: + if library_path: + resolved = Path(library_path).expanduser().resolve() + if not resolved.exists(): + raise PrivacyCoreUnavailable(f"privacy-core library not found: {resolved}") + return resolved + + env_override = os.environ.get("PRIVACY_CORE_LIB") + if env_override: + resolved = Path(env_override).expanduser().resolve() + if not resolved.exists(): + raise PrivacyCoreUnavailable(f"privacy-core library not found: {resolved}") + return resolved + + repo_root = Path(__file__).resolve().parents[2] + candidates = [] + for profile in ("debug", "release"): + target_dir = repo_root / "privacy-core" / "target" / profile + candidates.extend( + [ + target_dir / "privacy_core.dll", + target_dir / "libprivacy_core.so", + target_dir / "libprivacy_core.dylib", + ] + ) + + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + + searched = "\n".join(str(candidate) for candidate in candidates) + raise PrivacyCoreUnavailable( + "privacy-core shared library not found. Looked in:\n" f"{searched}" + ) + + def _configure_functions(self) -> None: + self._library.privacy_core_version.argtypes = [] + self._library.privacy_core_version.restype = _ByteBuffer + + self._library.privacy_core_last_error_message.argtypes = [] + self._library.privacy_core_last_error_message.restype = _ByteBuffer + + self._library.privacy_core_free_buffer.argtypes = [_ByteBuffer] + self._library.privacy_core_free_buffer.restype = None + + self._library.privacy_core_create_identity.argtypes = [] + self._library.privacy_core_create_identity.restype = ctypes.c_uint64 + + self._library.privacy_core_export_key_package.argtypes = [ctypes.c_uint64] + self._library.privacy_core_export_key_package.restype = _ByteBuffer + + self._library.privacy_core_import_key_package.argtypes = [ + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_import_key_package.restype = ctypes.c_uint64 + + self._library.privacy_core_create_group.argtypes = [ctypes.c_uint64] + self._library.privacy_core_create_group.restype = ctypes.c_uint64 + + self._library.privacy_core_add_member.argtypes = [ctypes.c_uint64, ctypes.c_uint64] + self._library.privacy_core_add_member.restype = ctypes.c_uint64 + + self._library.privacy_core_remove_member.argtypes = [ctypes.c_uint64, ctypes.c_uint32] + self._library.privacy_core_remove_member.restype = ctypes.c_uint64 + + self._library.privacy_core_encrypt_group_message.argtypes = [ + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_encrypt_group_message.restype = _ByteBuffer + + self._library.privacy_core_decrypt_group_message.argtypes = [ + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_decrypt_group_message.restype = _ByteBuffer + + self._library.privacy_core_export_public_bundle.argtypes = [ctypes.c_uint64] + self._library.privacy_core_export_public_bundle.restype = _ByteBuffer + + self._library.privacy_core_handle_stats.argtypes = [ + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_handle_stats.restype = ctypes.c_int64 + + self._library.privacy_core_commit_message_bytes.argtypes = [ctypes.c_uint64] + self._library.privacy_core_commit_message_bytes.restype = _ByteBuffer + + self._library.privacy_core_commit_welcome_message_bytes.argtypes = [ + ctypes.c_uint64, + ctypes.c_size_t, + ] + self._library.privacy_core_commit_welcome_message_bytes.restype = _ByteBuffer + + self._library.privacy_core_commit_joined_group_handle.argtypes = [ + ctypes.c_uint64, + ctypes.c_size_t, + ] + self._library.privacy_core_commit_joined_group_handle.restype = ctypes.c_uint64 + + self._library.privacy_core_create_dm_session.argtypes = [ + ctypes.c_uint64, + ctypes.c_uint64, + ] + self._library.privacy_core_create_dm_session.restype = ctypes.c_int64 + + self._library.privacy_core_dm_encrypt.argtypes = [ + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_dm_encrypt.restype = ctypes.c_int64 + + self._library.privacy_core_dm_decrypt.argtypes = [ + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_dm_decrypt.restype = ctypes.c_int64 + + self._library.privacy_core_dm_session_welcome.argtypes = [ + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_dm_session_welcome.restype = ctypes.c_int64 + + self._library.privacy_core_join_dm_session.argtypes = [ + ctypes.c_uint64, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + ] + self._library.privacy_core_join_dm_session.restype = ctypes.c_int64 + + self._library.privacy_core_release_dm_session.argtypes = [ctypes.c_uint64] + self._library.privacy_core_release_dm_session.restype = ctypes.c_int32 + + self._library.privacy_core_release_identity.argtypes = [ctypes.c_uint64] + self._library.privacy_core_release_identity.restype = ctypes.c_bool + + self._library.privacy_core_release_key_package.argtypes = [ctypes.c_uint64] + self._library.privacy_core_release_key_package.restype = ctypes.c_bool + + self._library.privacy_core_release_group.argtypes = [ctypes.c_uint64] + self._library.privacy_core_release_group.restype = ctypes.c_bool + + self._library.privacy_core_release_commit.argtypes = [ctypes.c_uint64] + self._library.privacy_core_release_commit.restype = ctypes.c_bool + + self._library.privacy_core_reset_all_state.argtypes = [] + self._library.privacy_core_reset_all_state.restype = ctypes.c_bool + + def version(self) -> str: + return self._consume_string(self._library.privacy_core_version()) + + def create_identity(self) -> int: + return self._ensure_handle(self._library.privacy_core_create_identity(), "create_identity") + + def export_key_package(self, identity_handle: int) -> bytes: + return self._consume_bytes( + self._library.privacy_core_export_key_package(ctypes.c_uint64(identity_handle)), + "export_key_package", + ) + + def import_key_package(self, data: bytes) -> int: + buffer = self._as_ubyte_buffer(data) + handle = self._library.privacy_core_import_key_package(buffer, len(data)) + return self._ensure_handle(handle, "import_key_package") + + def create_group(self, identity_handle: int) -> int: + handle = self._library.privacy_core_create_group(ctypes.c_uint64(identity_handle)) + return self._ensure_handle(handle, "create_group") + + def add_member(self, group_handle: int, key_package_handle: int) -> int: + handle = self._library.privacy_core_add_member( + ctypes.c_uint64(group_handle), + ctypes.c_uint64(key_package_handle), + ) + return self._ensure_handle(handle, "add_member") + + def remove_member(self, group_handle: int, member_ref: int) -> int: + handle = self._library.privacy_core_remove_member( + ctypes.c_uint64(group_handle), + ctypes.c_uint32(member_ref), + ) + return self._ensure_handle(handle, "remove_member") + + def encrypt_group_message(self, group_handle: int, plaintext: bytes) -> bytes: + buffer = self._as_ubyte_buffer(plaintext) + return self._consume_bytes( + self._library.privacy_core_encrypt_group_message( + ctypes.c_uint64(group_handle), + buffer, + len(plaintext), + ), + "encrypt_group_message", + ) + + def decrypt_group_message(self, group_handle: int, ciphertext: bytes) -> bytes: + buffer = self._as_ubyte_buffer(ciphertext) + return self._consume_bytes( + self._library.privacy_core_decrypt_group_message( + ctypes.c_uint64(group_handle), + buffer, + len(ciphertext), + ), + "decrypt_group_message", + ) + + def export_public_bundle(self, identity_handle: int) -> bytes: + return self._consume_bytes( + self._library.privacy_core_export_public_bundle(ctypes.c_uint64(identity_handle)), + "export_public_bundle", + ) + + def handle_stats(self) -> dict: + payload = self._call_i64_bytes_op( + "handle_stats", + lambda out_buf, out_cap: self._library.privacy_core_handle_stats(out_buf, out_cap), + ) + try: + return json.loads(payload.decode("utf-8")) + except Exception as exc: + raise PrivacyCoreError(f"handle_stats failed: invalid JSON: {exc}") from exc + + def commit_message_bytes(self, commit_handle: int) -> bytes: + return self._consume_bytes( + self._library.privacy_core_commit_message_bytes(ctypes.c_uint64(commit_handle)), + "commit_message_bytes", + ) + + def commit_welcome_message_bytes(self, commit_handle: int, index: int = 0) -> bytes: + return self._consume_bytes( + self._library.privacy_core_commit_welcome_message_bytes( + ctypes.c_uint64(commit_handle), + ctypes.c_size_t(index), + ), + "commit_welcome_message_bytes", + ) + + def commit_joined_group_handle(self, commit_handle: int, index: int = 0) -> int: + handle = self._library.privacy_core_commit_joined_group_handle( + ctypes.c_uint64(commit_handle), + ctypes.c_size_t(index), + ) + return self._ensure_handle(handle, "commit_joined_group_handle") + + def create_dm_session(self, initiator_identity: int, responder_key_package: int) -> int: + handle = self._library.privacy_core_create_dm_session( + ctypes.c_uint64(initiator_identity), + ctypes.c_uint64(responder_key_package), + ) + if handle > 0: + return int(handle) + raise self._error_for("create_dm_session") + + def dm_encrypt(self, session_handle: int, plaintext: bytes) -> bytes: + buffer = self._as_ubyte_buffer(plaintext) + return self._call_i64_bytes_op( + "dm_encrypt", + lambda out_buf, out_cap: self._library.privacy_core_dm_encrypt( + ctypes.c_uint64(session_handle), + buffer, + len(plaintext), + out_buf, + out_cap, + ), + ) + + def dm_decrypt(self, session_handle: int, ciphertext: bytes) -> bytes: + buffer = self._as_ubyte_buffer(ciphertext) + return self._call_i64_bytes_op( + "dm_decrypt", + lambda out_buf, out_cap: self._library.privacy_core_dm_decrypt( + ctypes.c_uint64(session_handle), + buffer, + len(ciphertext), + out_buf, + out_cap, + ), + ) + + def dm_session_welcome(self, session_handle: int) -> bytes: + return self._call_i64_bytes_op( + "dm_session_welcome", + lambda out_buf, out_cap: self._library.privacy_core_dm_session_welcome( + ctypes.c_uint64(session_handle), + out_buf, + out_cap, + ), + ) + + def join_dm_session(self, responder_identity: int, welcome: bytes) -> int: + buffer = self._as_ubyte_buffer(welcome) + handle = self._library.privacy_core_join_dm_session( + ctypes.c_uint64(responder_identity), + buffer, + len(welcome), + ) + if handle > 0: + return int(handle) + raise self._error_for("join_dm_session") + + def release_dm_session(self, handle: int) -> bool: + return bool(self._library.privacy_core_release_dm_session(ctypes.c_uint64(handle))) + + def release_identity(self, handle: int) -> bool: + return bool(self._library.privacy_core_release_identity(ctypes.c_uint64(handle))) + + def release_key_package(self, handle: int) -> bool: + return bool(self._library.privacy_core_release_key_package(ctypes.c_uint64(handle))) + + def release_group(self, handle: int) -> bool: + return bool(self._library.privacy_core_release_group(ctypes.c_uint64(handle))) + + def release_commit(self, handle: int) -> bool: + return bool(self._library.privacy_core_release_commit(ctypes.c_uint64(handle))) + + def reset_all_state(self) -> bool: + return bool(self._library.privacy_core_reset_all_state()) + + def _consume_string(self, buffer: _ByteBuffer) -> str: + payload = self._consume_buffer(buffer) + return payload.decode("utf-8") + + def _consume_bytes(self, buffer: _ByteBuffer, operation: str) -> bytes: + payload = self._consume_buffer(buffer) + if payload: + return payload + raise self._error_for(operation) + + def _consume_buffer(self, buffer: _ByteBuffer) -> bytes: + try: + if not buffer.data or buffer.len == 0: + return b"" + return bytes(ctypes.string_at(buffer.data, buffer.len)) + finally: + self._library.privacy_core_free_buffer(buffer) + + def _ensure_handle(self, handle: int, operation: str) -> int: + if handle: + return int(handle) + raise self._error_for(operation) + + def _call_i64_bytes_op(self, operation: str, invoker) -> bytes: + required = int(invoker(None, 0)) + if required < 0: + raise self._error_for(operation) + if required == 0: + return b"" + output = (ctypes.c_uint8 * required)() + written = int(invoker(output, required)) + if written < 0: + raise self._error_for(operation) + return bytes(output[:written]) + + def _error_for(self, operation: str) -> PrivacyCoreError: + message = self._last_error() + if message: + return PrivacyCoreError(f"{operation} failed: {message}") + return PrivacyCoreError(f"{operation} failed without an error message") + + def _last_error(self) -> str: + return self._consume_string(self._library.privacy_core_last_error_message()) + + @staticmethod + def _as_ubyte_buffer(data: bytes | bytearray | memoryview) -> ctypes.Array[ctypes.c_uint8]: + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("privacy-core byte arguments must be bytes-like") + raw = bytes(data) + return (ctypes.c_uint8 * len(raw)).from_buffer_copy(raw) + + +def candidate_library_paths() -> Iterable[Path]: + """Expose the default search order for diagnostics/tests.""" + + repo_root = Path(__file__).resolve().parents[2] + for profile in ("debug", "release"): + target_dir = repo_root / "privacy-core" / "target" / profile + yield target_dir / "privacy_core.dll" + yield target_dir / "libprivacy_core.so" + yield target_dir / "libprivacy_core.dylib" diff --git a/backend/services/psk_reporter_fetcher.py b/backend/services/psk_reporter_fetcher.py new file mode 100644 index 00000000..41f5124f --- /dev/null +++ b/backend/services/psk_reporter_fetcher.py @@ -0,0 +1,113 @@ +""" +PSK Reporter fetcher — pulls recent digital mode signal reports (FT8, WSPR, etc.) +from the global PSK Reporter network. No API key required. + +Docs: https://pskreporter.info/pskdev.html +""" + +import logging +import xml.etree.ElementTree as ET + +import requests +from cachetools import TTLCache, cached + +logger = logging.getLogger(__name__) + +_cache = TTLCache(maxsize=1, ttl=600) # 10-minute cache + +_ENDPOINT = "https://retrieve.pskreporter.info/query" + + +def maidenhead_to_latlon(locator: str) -> tuple[float, float] | None: + """Convert a 4-or-6 character Maidenhead grid locator to (lat, lon).""" + loc = locator.strip().upper() + if len(loc) < 4: + return None + try: + lon = (ord(loc[0]) - ord("A")) * 20 - 180 + lat = (ord(loc[1]) - ord("A")) * 10 - 90 + lon += int(loc[2]) * 2 + lat += int(loc[3]) + if len(loc) >= 6: + lon += (ord(loc[4]) - ord("A")) * (2 / 24) + lat += (ord(loc[5]) - ord("A")) * (1 / 24) + # center of sub-square + lon += 1 / 24 + lat += 1 / 48 + else: + # center of grid square + lon += 1 + lat += 0.5 + if abs(lat) > 90 or abs(lon) > 180: + return None + return round(lat, 4), round(lon, 4) + except (IndexError, ValueError): + return None + + +@cached(_cache) +def fetch_psk_reporter_spots() -> list[dict]: + """Fetch recent FT8 reception reports from PSK Reporter.""" + try: + resp = requests.get( + _ENDPOINT, + params={ + "mode": "FT8", + "flowStartSeconds": -900, # last 15 minutes + "rronly": 1, # reception reports only + "noactive": 1, # exclude active monitor noise + "rptlimit": 5000, # cap payload size + }, + timeout=30, + headers={"Accept": "application/xml"}, + ) + resp.raise_for_status() + + root = ET.fromstring(resp.content) + + # PSK Reporter XML uses namespaces + ns = "" + if root.tag.startswith("{"): + ns = root.tag.split("}")[0] + "}" + + spots: list[dict] = [] + for rec in root.iter(f"{ns}receptionReport"): + receiver_loc = rec.get("receiverLocator", "") + sender_loc = rec.get("senderLocator", "") + + # Prefer receiver location (where the signal was heard) + loc_str = receiver_loc or sender_loc + if not loc_str: + continue + coords = maidenhead_to_latlon(loc_str) + if coords is None: + continue + lat, lon = coords + + try: + freq = int(rec.get("frequency", "0")) + except (ValueError, TypeError): + freq = 0 + + try: + snr = int(rec.get("sNR", "0")) + except (ValueError, TypeError): + snr = 0 + + spots.append({ + "lat": lat, + "lon": lon, + "sender": (rec.get("senderCallsign") or "")[:20], + "receiver": (rec.get("receiverCallsign") or "")[:20], + "frequency": freq, + "mode": (rec.get("mode") or "FT8")[:10], + "snr": snr, + "time": rec.get("flowStartSeconds", ""), + }) + + logger.info("PSK Reporter: fetched %d spots", len(spots)) + return spots + + except (requests.RequestException, ET.ParseError, Exception) as e: + logger.error("PSK Reporter fetch error: %s", e) + return [] diff --git a/backend/services/radio_intercept.py b/backend/services/radio_intercept.py index 66bbe1b0..78534144 100644 --- a/backend/services/radio_intercept.py +++ b/backend/services/radio_intercept.py @@ -10,6 +10,7 @@ # Cache the top feeds for 5 minutes so we don't hammer Broadcastify radio_cache = TTLCache(maxsize=1, ttl=300) + @cached(radio_cache) def get_top_broadcastify_feeds(): """ @@ -18,130 +19,184 @@ def get_top_broadcastify_feeds(): """ logger.info("Scraping Broadcastify Top Feeds (Cache Miss)") headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.9', + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", } - + try: res = requests.get("https://www.broadcastify.com/listen/top", headers=headers, timeout=10) if res.status_code != 200: logger.error(f"Broadcastify Scrape Failed: HTTP {res.status_code}") return [] - - soup = BeautifulSoup(res.text, 'html.parser') - - table = soup.find('table', {'class': 'btable'}) + + soup = BeautifulSoup(res.text, "html.parser") + + table = soup.find("table", {"class": "btable"}) if not table: logger.error("Could not find feeds table on Broadcastify.") return [] - + feeds = [] - rows = table.find_all('tr')[1:] # Skip header row - + rows = table.find_all("tr")[1:] # Skip header row + for row in rows: - cols = row.find_all('td') + cols = row.find_all("td") if len(cols) >= 5: # Top layout: [Listeners, Feed ID (hidden), Location, Feed Name, Category, Genre] - listeners_str = cols[0].text.strip().replace(',', '') + listeners_str = cols[0].text.strip().replace(",", "") listeners = int(listeners_str) if listeners_str.isdigit() else 0 - - link_tag = cols[2].find('a') + + link_tag = cols[2].find("a") if not link_tag: continue - - href = link_tag.get('href', '') - feed_id = href.split('/')[-1] if '/listen/feed/' in href else None - + + href = link_tag.get("href", "") + feed_id = href.split("/")[-1] if "/listen/feed/" in href else None + if not feed_id: continue - + location = cols[1].text.strip() name = cols[2].text.strip() category = cols[3].text.strip() - - feeds.append({ - "id": feed_id, - "listeners": listeners, - "location": location, - "name": name, - "category": category, - "stream_url": f"https://broadcastify.cdnstream1.com/{feed_id}" - }) - + + feeds.append( + { + "id": feed_id, + "listeners": listeners, + "location": location, + "name": name, + "category": category, + "stream_url": f"https://broadcastify.cdnstream1.com/{feed_id}", + } + ) + logger.info(f"Successfully scraped {len(feeds)} top feeds from Broadcastify.") return feeds - + except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: logger.error(f"Broadcastify Scrape Exception: {e}") return [] + # Cache OpenMHZ systems mapping so we don't have to fetch all 450+ every time openmhz_systems_cache = TTLCache(maxsize=1, ttl=3600) + @cached(openmhz_systems_cache) def get_openmhz_systems(): """Fetches the full directory of OpenMHZ systems.""" logger.info("Scraping OpenMHZ Systems (Cache Miss)") - scraper = cloudscraper.create_scraper(browser={'browser': 'chrome', 'platform': 'windows', 'desktop': True}) - + scraper = cloudscraper.create_scraper( + browser={"browser": "chrome", "platform": "windows", "desktop": True} + ) + try: res = scraper.get("https://api.openmhz.com/systems", timeout=15) if res.status_code == 200: data = res.json() # Return list of systems - return data.get('systems', []) if isinstance(data, dict) else [] + return data.get("systems", []) if isinstance(data, dict) else [] return [] except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: logger.error(f"OpenMHZ Systems Scrape Exception: {e}") return [] + # Cache specific city calls briefly (15-30s) to limit our polling rate openmhz_calls_cache = TTLCache(maxsize=100, ttl=20) + @cached(openmhz_calls_cache) def get_recent_openmhz_calls(sys_name: str): """Fetches the actual audio burst .m4a URLs for a specific system (e.g., 'wmata').""" logger.info(f"Fetching OpenMHZ calls for {sys_name} (Cache Miss)") - scraper = cloudscraper.create_scraper(browser={'browser': 'chrome', 'platform': 'windows', 'desktop': True}) - + scraper = cloudscraper.create_scraper( + browser={"browser": "chrome", "platform": "windows", "desktop": True} + ) + try: url = f"https://api.openmhz.com/{sys_name}/calls" res = scraper.get(url, timeout=15) if res.status_code == 200: data = res.json() - return data.get('calls', []) if isinstance(data, dict) else [] + return data.get("calls", []) if isinstance(data, dict) else [] return [] except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: logger.error(f"OpenMHZ Calls Scrape Exception ({sys_name}): {e}") return [] + US_STATES = { - 'Alabama': 'AL', 'Alaska': 'AK', 'Arizona': 'AZ', 'Arkansas': 'AR', 'California': 'CA', - 'Colorado': 'CO', 'Connecticut': 'CT', 'Delaware': 'DE', 'Florida': 'FL', 'Georgia': 'GA', - 'Hawaii': 'HI', 'Idaho': 'ID', 'Illinois': 'IL', 'Indiana': 'IN', 'Iowa': 'IA', - 'Kansas': 'KS', 'Kentucky': 'KY', 'Louisiana': 'LA', 'Maine': 'ME', 'Maryland': 'MD', - 'Massachusetts': 'MA', 'Michigan': 'MI', 'Minnesota': 'MN', 'Mississippi': 'MS', - 'Missouri': 'MO', 'Montana': 'MT', 'Nebraska': 'NE', 'Nevada': 'NV', 'New Hampshire': 'NH', - 'New Jersey': 'NJ', 'New Mexico': 'NM', 'New York': 'NY', 'North Carolina': 'NC', - 'North Dakota': 'ND', 'Ohio': 'OH', 'Oklahoma': 'OK', 'Oregon': 'OR', 'Pennsylvania': 'PA', - 'Rhode Island': 'RI', 'South Carolina': 'SC', 'South Dakota': 'SD', 'Tennessee': 'TN', - 'Texas': 'TX', 'Utah': 'UT', 'Vermont': 'VT', 'Virginia': 'VA', 'Washington': 'WA', - 'West Virginia': 'WV', 'Wisconsin': 'WI', 'Wyoming': 'WY', 'Washington, D.C.': 'DC', 'District of Columbia': 'DC' + "Alabama": "AL", + "Alaska": "AK", + "Arizona": "AZ", + "Arkansas": "AR", + "California": "CA", + "Colorado": "CO", + "Connecticut": "CT", + "Delaware": "DE", + "Florida": "FL", + "Georgia": "GA", + "Hawaii": "HI", + "Idaho": "ID", + "Illinois": "IL", + "Indiana": "IN", + "Iowa": "IA", + "Kansas": "KS", + "Kentucky": "KY", + "Louisiana": "LA", + "Maine": "ME", + "Maryland": "MD", + "Massachusetts": "MA", + "Michigan": "MI", + "Minnesota": "MN", + "Mississippi": "MS", + "Missouri": "MO", + "Montana": "MT", + "Nebraska": "NE", + "Nevada": "NV", + "New Hampshire": "NH", + "New Jersey": "NJ", + "New Mexico": "NM", + "New York": "NY", + "North Carolina": "NC", + "North Dakota": "ND", + "Ohio": "OH", + "Oklahoma": "OK", + "Oregon": "OR", + "Pennsylvania": "PA", + "Rhode Island": "RI", + "South Carolina": "SC", + "South Dakota": "SD", + "Tennessee": "TN", + "Texas": "TX", + "Utah": "UT", + "Vermont": "VT", + "Virginia": "VA", + "Washington": "WA", + "West Virginia": "WV", + "Wisconsin": "WI", + "Wyoming": "WY", + "Washington, D.C.": "DC", + "District of Columbia": "DC", } import math + def haversine_distance(lat1, lon1, lat2, lon2): - R = 3958.8 # Earth radius in miles + R = 3958.8 # Earth radius in miles dLat = math.radians(lat2 - lat1) dLon = math.radians(lon2 - lon1) - a = math.sin(dLat/2) * math.sin(dLat/2) + \ - math.cos(math.radians(lat1)) * math.cos(math.radians(lat2)) * \ - math.sin(dLon/2) * math.sin(dLon/2) - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a)) + a = math.sin(dLat / 2) * math.sin(dLat / 2) + math.cos(math.radians(lat1)) * math.cos( + math.radians(lat2) + ) * math.sin(dLon / 2) * math.sin(dLon / 2) + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) return R * c + def find_nearest_openmhz_systems_list(lat: float, lng: float, limit: int = 5): """ Finds the strictly nearest OpenMHZ systems by distance. @@ -153,20 +208,21 @@ def find_nearest_openmhz_systems_list(lat: float, lng: float, limit: int = 5): # Calculate distance for all systems that provide coordinates valid_systems = [] for s in systems: - s_lat = s.get('lat') - s_lng = s.get('lng') + s_lat = s.get("lat") + s_lng = s.get("lng") if s_lat is not None and s_lng is not None: dist = haversine_distance(lat, lng, float(s_lat), float(s_lng)) - s['distance_miles'] = dist + s["distance_miles"] = dist valid_systems.append(s) if not valid_systems: return [] # Sort strictly by distance - valid_systems.sort(key=lambda x: x['distance_miles']) + valid_systems.sort(key=lambda x: x["distance_miles"]) return valid_systems[:limit] + def find_nearest_openmhz_system(lat: float, lng: float): """ Returns the single closest OpenMHZ system by distance. diff --git a/backend/services/region_dossier.py b/backend/services/region_dossier.py index b3f70175..48a4d3dc 100644 --- a/backend/services/region_dossier.py +++ b/backend/services/region_dossier.py @@ -16,13 +16,38 @@ _nominatim_last_call = 0.0 +def _reverse_geocode_offline(lat: float, lng: float) -> dict: + """Offline fallback via reverse_geocoder when external reverse geocoding is blocked.""" + try: + import reverse_geocoder as rg + + hit = rg.search((lat, lng), mode=1)[0] + country_code = (hit.get("cc") or "").upper() + city = hit.get("name") or "" + state = hit.get("admin1") or "" + display = ", ".join(part for part in [city, state, country_code] if part) + return { + "city": city, + "state": state, + "country": country_code or "Unknown", + "country_code": country_code, + "display_name": display, + "offline_fallback": True, + } + except Exception as e: + logger.warning(f"Offline reverse geocode failed: {e}") + return {} + + def _reverse_geocode(lat: float, lng: float) -> dict: global _nominatim_last_call url = ( f"https://nominatim.openstreetmap.org/reverse?" f"lat={lat}&lon={lng}&format=json&zoom=10&addressdetails=1&accept-language=en" ) - headers = {"User-Agent": "ShadowBroker-OSINT/1.0 (live-risk-dashboard; contact@shadowbroker.app)"} + headers = { + "User-Agent": "ShadowBroker-OSINT/1.0 (live-risk-dashboard; contact@shadowbroker.app)" + } for attempt in range(2): # Enforce Nominatim's 1 req/sec policy @@ -33,26 +58,32 @@ def _reverse_geocode(lat: float, lng: float) -> dict: try: # Use requests directly — fetch_with_curl raises on non-200 which breaks 429 handling - res = _requests.get(url, timeout=10, headers=headers) + res = _requests.get(url, timeout=4, headers=headers) if res.status_code == 200: data = res.json() addr = data.get("address", {}) return { - "city": addr.get("city") or addr.get("town") or addr.get("village") or addr.get("county") or "", + "city": addr.get("city") + or addr.get("town") + or addr.get("village") + or addr.get("county") + or "", "state": addr.get("state") or addr.get("region") or "", "country": addr.get("country") or "", "country_code": (addr.get("country_code") or "").upper(), "display_name": data.get("display_name", ""), } elif res.status_code == 429: - logger.warning(f"Nominatim 429 rate-limited, retrying after 2s (attempt {attempt+1})") - time.sleep(2) + logger.warning( + f"Nominatim 429 rate-limited, retrying after 1s (attempt {attempt+1})" + ) + time.sleep(1) continue else: logger.warning(f"Nominatim returned {res.status_code}") except (_requests.RequestException, ConnectionError, TimeoutError, OSError) as e: logger.warning(f"Reverse geocode failed: {e}") - return {} + return _reverse_geocode_offline(lat, lng) def _fetch_country_data(country_code: str) -> dict: @@ -63,9 +94,12 @@ def _fetch_country_data(country_code: str) -> dict: f"?fields=name,population,capital,languages,region,subregion,area,currencies,borders,flag" ) try: - res = fetch_with_curl(url, timeout=10) + res = fetch_with_curl(url, timeout=5) if res.status_code == 200: - return res.json() + data = res.json() + if isinstance(data, list): + return data[0] if data and isinstance(data[0], dict) else {} + return data if isinstance(data, dict) else {} except (ConnectionError, TimeoutError, ValueError, KeyError, OSError) as e: logger.warning(f"RestCountries failed for {country_code}: {e}") return {} @@ -87,7 +121,7 @@ def _fetch_wikidata_leader(country_name: str) -> dict: """ url = f"https://query.wikidata.org/sparql?query={quote(sparql)}&format=json" try: - res = fetch_with_curl(url, timeout=15) + res = fetch_with_curl(url, timeout=6) if res.status_code == 200: results = res.json().get("results", {}).get("bindings", []) if results: @@ -113,7 +147,7 @@ def _fetch_local_wiki_summary(place_name: str, country_name: str = "") -> dict: slug = quote(name.replace(" ", "_")) url = f"https://en.wikipedia.org/api/rest_v1/page/summary/{slug}" try: - res = fetch_with_curl(url, timeout=10) + res = fetch_with_curl(url, timeout=5) if res.status_code == 200: data = res.json() if data.get("type") != "disambiguation": @@ -122,7 +156,13 @@ def _fetch_local_wiki_summary(place_name: str, country_name: str = "") -> dict: "extract": data.get("extract", ""), "thumbnail": data.get("thumbnail", {}).get("source", ""), } - except (ConnectionError, TimeoutError, ValueError, KeyError, OSError): # Intentional: optional enrichment + except ( + ConnectionError, + TimeoutError, + ValueError, + KeyError, + OSError, + ): # Intentional: optional enrichment continue return {} @@ -148,33 +188,37 @@ def get_region_dossier(lat: float, lng: float) -> dict: city_name = geo.get("city", "") state_name = geo.get("state", "") - # Step 2: Parallel fetch with timeouts to prevent hanging - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + # Step 2: Parallel fetch with real timeouts that do not block on executor shutdown + pool = concurrent.futures.ThreadPoolExecutor(max_workers=4) + try: country_fut = pool.submit(_fetch_country_data, country_code) leader_fut = pool.submit(_fetch_wikidata_leader, country_name) - local_fut = pool.submit(_fetch_local_wiki_summary, city_name or state_name, country_name) - # Also fetch country-level Wikipedia summary as fallback for local + local_fut = pool.submit( + _fetch_local_wiki_summary, city_name or state_name, country_name + ) country_wiki_fut = pool.submit(_fetch_local_wiki_summary, country_name, "") - try: - country_data = country_fut.result(timeout=12) - except Exception: # Intentional: optional enrichment - logger.warning("Country data fetch timed out or failed") - country_data = {} - try: - leader_data = leader_fut.result(timeout=12) - except Exception: # Intentional: optional enrichment - logger.warning("Leader data fetch timed out or failed") - leader_data = {"leader": "Unknown", "government_type": "Unknown"} - try: - local_data = local_fut.result(timeout=12) - except Exception: # Intentional: optional enrichment - logger.warning("Local wiki fetch timed out or failed") - local_data = {} - try: - country_wiki_data = country_wiki_fut.result(timeout=12) - except Exception: # Intentional: optional enrichment - country_wiki_data = {} + try: + country_data = country_fut.result(timeout=6) + except Exception: # Intentional: optional enrichment + logger.warning("Country data fetch timed out or failed") + country_data = {} + try: + leader_data = leader_fut.result(timeout=6) + except Exception: # Intentional: optional enrichment + logger.warning("Leader data fetch timed out or failed") + leader_data = {"leader": "Unknown", "government_type": "Unknown"} + try: + local_data = local_fut.result(timeout=5) + except Exception: # Intentional: optional enrichment + logger.warning("Local wiki fetch timed out or failed") + local_data = {} + try: + country_wiki_data = country_wiki_fut.result(timeout=5) + except Exception: # Intentional: optional enrichment + country_wiki_data = {} + finally: + pool.shutdown(wait=False, cancel_futures=True) # If no local data but we have country wiki summary, use that if not local_data.get("extract") and country_wiki_data.get("extract"): @@ -203,7 +247,11 @@ def get_region_dossier(lat: float, lng: float) -> dict: "leader": leader_data.get("leader", "Unknown"), "government_type": leader_data.get("government_type", "Unknown"), "population": country_data.get("population", 0), - "capital": (country_data.get("capital") or ["Unknown"])[0] if isinstance(country_data.get("capital"), list) else "Unknown", + "capital": ( + (country_data.get("capital") or ["Unknown"])[0] + if isinstance(country_data.get("capital"), list) + else "Unknown" + ), "languages": lang_list, "currencies": currency_list, "region": country_data.get("region", ""), diff --git a/backend/services/satnogs_fetcher.py b/backend/services/satnogs_fetcher.py new file mode 100644 index 00000000..ccc9ad46 --- /dev/null +++ b/backend/services/satnogs_fetcher.py @@ -0,0 +1,112 @@ +""" +SatNOGS ground station + observation fetcher. +Queries the SatNOGS Network API for online ground stations and recent +satellite observations. No API key required for read-only access. +""" + +import logging +import requests +from cachetools import TTLCache, cached + +logger = logging.getLogger(__name__) + +_station_cache = TTLCache(maxsize=1, ttl=600) # 10-minute cache +_obs_cache = TTLCache(maxsize=1, ttl=300) # 5-minute cache + + +@cached(_station_cache) +def fetch_satnogs_stations() -> list[dict]: + """Fetch online SatNOGS ground stations (status=2 = online).""" + try: + resp = requests.get( + "https://network.satnogs.org/api/stations/", + params={"format": "json", "status": 2}, + timeout=20, + headers={"Accept": "application/json"}, + ) + resp.raise_for_status() + stations = [] + for s in resp.json(): + lat, lng = s.get("lat"), s.get("lng") + if lat is None or lng is None: + continue + try: + lat, lng = float(lat), float(lng) + except (ValueError, TypeError): + continue + if abs(lat) > 90 or abs(lng) > 180: + continue + + antennas = s.get("antenna") or [] + antenna_str = ", ".join( + a.get("antenna_type", "") for a in antennas if a.get("antenna_type") + ) + + stations.append( + { + "id": s.get("id"), + "name": (s.get("name") or "Unknown")[:120], + "lat": round(lat, 5), + "lng": round(lng, 5), + "altitude": s.get("altitude"), + "antenna": antenna_str[:200], + "observations": s.get("observations", 0), + "status": s.get("status"), + "last_seen": s.get("last_seen"), + } + ) + logger.info(f"SatNOGS: fetched {len(stations)} online stations") + return stations + except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: + logger.error(f"SatNOGS stations error: {e}") + return [] + + +@cached(_obs_cache) +def fetch_satnogs_observations() -> list[dict]: + """Fetch recent good observations (first page, ~25 results).""" + try: + resp = requests.get( + "https://network.satnogs.org/api/observations/", + params={"format": "json", "status": "good"}, + timeout=20, + headers={"Accept": "application/json"}, + ) + resp.raise_for_status() + obs = [] + for o in resp.json(): + lat = o.get("station_lat") + lng = o.get("station_lng") + if lat is None or lng is None: + continue + try: + lat, lng = float(lat), float(lng) + except (ValueError, TypeError): + continue + + # Satellite name from TLE line 0, or fall back to NORAD ID + tle0 = (o.get("tle0") or "").strip() + sat_name = tle0 if tle0 else f"NORAD {o.get('norad_cat_id', '?')}" + + obs.append( + { + "id": o.get("id"), + "satellite_name": sat_name[:80], + "norad_id": o.get("norad_cat_id"), + "station_name": (o.get("station_name") or "Unknown")[:80], + "lat": round(lat, 5), + "lng": round(lng, 5), + "start": o.get("start"), + "end": o.get("end"), + "frequency": o.get("transmitter_downlink_low"), + "mode": o.get("transmitter_mode"), + "waterfall": o.get("waterfall"), + "audio": o.get("archive_url") or o.get("payload"), + "status": o.get("vetted_status"), + } + ) + logger.info(f"SatNOGS: fetched {len(obs)} recent observations") + return obs + except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: + logger.error(f"SatNOGS observations error: {e}") + return [] diff --git a/backend/services/sentinel_search.py b/backend/services/sentinel_search.py index 741fcec2..ec66896e 100644 --- a/backend/services/sentinel_search.py +++ b/backend/services/sentinel_search.py @@ -1,6 +1,9 @@ """ Sentinel-2 satellite imagery search via Microsoft Planetary Computer STAC API. Free, keyless search for metadata + thumbnails. Used in the right-click dossier. + +We use the raw STAC HTTP API with explicit timeouts so the right-click dossier +cannot hang behind a slow client library call. """ import logging @@ -14,6 +17,32 @@ _sentinel_cache = TTLCache(maxsize=200, ttl=3600) +def _esri_imagery_fallback(lat: float, lng: float) -> dict: + lat_span = 0.18 + lng_span = 0.24 + bbox = f"{lng - lng_span},{lat - lat_span},{lng + lng_span},{lat + lat_span}" + fullres = ( + "https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/" + f"export?bbox={bbox}&bboxSR=4326&imageSR=4326&size=1600,900&format=png32&f=image" + ) + thumbnail = ( + "https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/" + f"export?bbox={bbox}&bboxSR=4326&imageSR=4326&size=640,360&format=png32&f=image" + ) + return { + "found": True, + "scene_id": None, + "datetime": None, + "cloud_cover": None, + "thumbnail_url": thumbnail, + "fullres_url": fullres, + "bbox": [lng - lng_span, lat - lat_span, lng + lng_span, lat + lat_span], + "platform": "Esri World Imagery", + "fallback": True, + "message": "Planetary Computer unavailable; using Esri World Imagery fallback", + } + + def search_sentinel2_scene(lat: float, lng: float) -> dict: """Search for the latest Sentinel-2 L2A scene covering a point.""" cache_key = f"{round(lat, 2)}_{round(lng, 2)}" @@ -21,62 +50,56 @@ def search_sentinel2_scene(lat: float, lng: float) -> dict: return _sentinel_cache[cache_key] try: - from pystac_client import Client - - catalog = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1") end = datetime.utcnow() start = end - timedelta(days=30) - - search = catalog.search( - collections=["sentinel-2-l2a"], - intersects={"type": "Point", "coordinates": [lng, lat]}, - datetime=f"{start.isoformat()}Z/{end.isoformat()}Z", - sortby=[{"field": "datetime", "direction": "desc"}], - max_items=3, - query={"eo:cloud_cover": {"lt": 30}}, + search_payload = { + "collections": ["sentinel-2-l2a"], + "intersects": {"type": "Point", "coordinates": [lng, lat]}, + "datetime": f"{start.isoformat()}Z/{end.isoformat()}Z", + "sortby": [{"field": "datetime", "direction": "desc"}], + "limit": 3, + "query": {"eo:cloud_cover": {"lt": 30}}, + } + search_res = requests.post( + "https://planetarycomputer.microsoft.com/api/stac/v1/search", + json=search_payload, + timeout=8, + headers={"User-Agent": "ShadowBroker-OSINT/1.0 (live-risk-dashboard)"}, ) - - items = list(search.items()) - if not items: - result = {"found": False, "message": "No clear scenes in last 30 days"} + search_res.raise_for_status() + data = search_res.json() + features = data.get("features", []) + if not features: + result = _esri_imagery_fallback(lat, lng) _sentinel_cache[cache_key] = result return result - item = items[0] - # Try to sign item first for Azure blob URLs - try: - import planetary_computer - item = planetary_computer.sign_item(item) - except ImportError: - pass # planetary_computer not installed, try unsigned URLs - except (ConnectionError, TimeoutError, ValueError) as e: - logger.warning(f"Sentinel-2 signing failed: {e}") - - # Get the rendered_preview (full-res PNG) and thumbnail separately - rendered = item.assets.get("rendered_preview") - thumbnail = item.assets.get("thumbnail") + item = features[0] + assets = item.get("assets", {}) or {} + rendered = assets.get("rendered_preview") or {} + thumbnail = assets.get("thumbnail") or {} # Full-res image URL — what opens when user clicks - fullres_url = rendered.href if rendered else (thumbnail.href if thumbnail else None) + fullres_url = rendered.get("href") or thumbnail.get("href") # Thumbnail URL — what shows in the popup card - thumb_url = thumbnail.href if thumbnail else (rendered.href if rendered else None) + thumb_url = thumbnail.get("href") or rendered.get("href") result = { "found": True, - "scene_id": item.id, - "datetime": item.datetime.isoformat() if item.datetime else None, - "cloud_cover": item.properties.get("eo:cloud_cover"), + "scene_id": item.get("id"), + "datetime": item.get("properties", {}).get("datetime"), + "cloud_cover": item.get("properties", {}).get("eo:cloud_cover"), "thumbnail_url": thumb_url, "fullres_url": fullres_url, - "bbox": list(item.bbox) if item.bbox else None, - "platform": item.properties.get("platform", "Sentinel-2"), + "bbox": list(item.get("bbox", [])) if item.get("bbox") else None, + "platform": item.get("properties", {}).get("platform", "Sentinel-2"), } _sentinel_cache[cache_key] = result return result - except ImportError: - logger.warning("pystac-client not installed — Sentinel-2 search unavailable") - return {"found": False, "error": "pystac-client not installed"} except (requests.RequestException, ConnectionError, TimeoutError, ValueError) as e: logger.error(f"Sentinel-2 search failed for ({lat}, {lng}): {e}") - return {"found": False, "error": str(e)} + result = _esri_imagery_fallback(lat, lng) + result["error"] = str(e) + _sentinel_cache[cache_key] = result + return result diff --git a/backend/services/shodan_connector.py b/backend/services/shodan_connector.py new file mode 100644 index 00000000..f69470c6 --- /dev/null +++ b/backend/services/shodan_connector.py @@ -0,0 +1,316 @@ +""" +Local-only Shodan connector. + +This module intentionally does NOT merge Shodan results into the dashboard's +canonical live-data store. It exposes manual, operator-triggered lookups that +can be rendered locally in the UI as a temporary investigative overlay. +""" + +from __future__ import annotations + +import logging +import os +import threading +import time +from typing import Any + +import requests +from cachetools import TTLCache + +logger = logging.getLogger(__name__) + +_SHODAN_BASE = "https://api.shodan.io" +_USER_AGENT = "ShadowBroker/0.9.6 local Shodan connector" +_REQUEST_TIMEOUT = 15 +_MIN_INTERVAL_SECONDS = 1.05 # Shodan docs say API plans are rate limited to ~1 req/sec. +_DEFAULT_SEARCH_PAGES = 1 +_MAX_SEARCH_PAGES = 2 + +_search_cache: TTLCache[str, dict[str, Any]] = TTLCache(maxsize=24, ttl=90) +_count_cache: TTLCache[str, dict[str, Any]] = TTLCache(maxsize=24, ttl=120) +_host_cache: TTLCache[str, dict[str, Any]] = TTLCache(maxsize=32, ttl=300) + +_request_lock = threading.Lock() +_last_request_at = 0.0 + + +class ShodanConnectorError(Exception): + def __init__(self, detail: str, status_code: int = 400): + super().__init__(detail) + self.detail = detail + self.status_code = status_code + + +def _get_api_key() -> str: + api_key = os.environ.get("SHODAN_API_KEY", "").strip() + if not api_key: + raise ShodanConnectorError( + "Shodan API key not configured. Add SHODAN_API_KEY in Settings > API Keys.", + status_code=428, + ) + return api_key + + +def _clean_query(value: str | None) -> str: + query = (value or "").strip() + if not query: + raise ShodanConnectorError("Shodan query cannot be empty.", status_code=400) + if "\n" in query or "\r" in query: + raise ShodanConnectorError("Shodan query must be a single line.", status_code=400) + return query + + +def _cache_key(prefix: str, payload: dict[str, Any]) -> str: + normalized = tuple(sorted((str(k), str(v)) for k, v in payload.items())) + return f"{prefix}:{normalized!r}" + + +def _normalize_string_list(values: Any, limit: int = 10) -> list[str]: + if not isinstance(values, list): + return [] + cleaned: list[str] = [] + for item in values: + text = str(item).strip() + if text: + cleaned.append(text) + if len(cleaned) >= limit: + break + return cleaned + + +def _location_label(location: dict[str, Any]) -> str | None: + parts = [ + str(location.get("city") or "").strip(), + str(location.get("region_code") or "").strip(), + str(location.get("country_code") or "").strip(), + ] + label = ", ".join([p for p in parts if p]) + return label or None + + +def _normalize_match(match: dict[str, Any]) -> dict[str, Any]: + location = match.get("location") or {} + lat = location.get("latitude") + lng = location.get("longitude") + port = match.get("port") + ip_str = str(match.get("ip_str") or match.get("ip") or "").strip() + host_id = f"shodan-{ip_str or 'unknown'}-{port or 'na'}" + vulns = match.get("vulns") or [] + if isinstance(vulns, dict): + vuln_list = _normalize_string_list(list(vulns.keys()), limit=12) + else: + vuln_list = _normalize_string_list(vulns, limit=12) + return { + "id": host_id, + "ip": ip_str or "UNKNOWN", + "port": port, + "transport": match.get("transport"), + "timestamp": match.get("timestamp"), + "lat": lat if isinstance(lat, (int, float)) else None, + "lng": lng if isinstance(lng, (int, float)) else None, + "city": location.get("city"), + "region_code": location.get("region_code"), + "country_code": location.get("country_code"), + "country_name": location.get("country_name"), + "location_label": _location_label(location), + "asn": match.get("asn"), + "org": match.get("org"), + "isp": match.get("isp"), + "product": match.get("product"), + "os": match.get("os"), + "hostnames": _normalize_string_list(match.get("hostnames")), + "domains": _normalize_string_list(match.get("domains")), + "tags": _normalize_string_list(match.get("tags")), + "vulns": vuln_list, + "data_snippet": str(match.get("data") or "").strip()[:280] or None, + "attribution": "Data from Shodan", + } + + +def _normalize_services(items: Any) -> list[dict[str, Any]]: + if not isinstance(items, list): + return [] + services: list[dict[str, Any]] = [] + for item in items[:30]: + if not isinstance(item, dict): + continue + services.append( + { + "port": item.get("port"), + "transport": item.get("transport"), + "product": item.get("product"), + "timestamp": item.get("timestamp"), + "tags": _normalize_string_list(item.get("tags"), limit=8), + "banner_excerpt": str(item.get("data") or "").strip()[:320] or None, + } + ) + return services + + +def _normalize_facets(raw_facets: Any) -> dict[str, list[dict[str, Any]]]: + if not isinstance(raw_facets, dict): + return {} + normalized: dict[str, list[dict[str, Any]]] = {} + for key, bucket_list in raw_facets.items(): + if not isinstance(bucket_list, list): + continue + normalized[str(key)] = [ + {"value": str(bucket.get("value") or ""), "count": int(bucket.get("count") or 0)} + for bucket in bucket_list[:12] + if isinstance(bucket, dict) + ] + return normalized + + +def _request(path: str, *, params: dict[str, Any], cache: TTLCache[str, dict[str, Any]] | None = None) -> dict[str, Any]: + api_key = _get_api_key() + payload = {**params, "key": api_key} + cache_key = _cache_key(path, {k: v for k, v in payload.items() if k != "key"}) + if cache is not None and cache_key in cache: + return cache[cache_key] + + global _last_request_at + with _request_lock: + elapsed = time.monotonic() - _last_request_at + if elapsed < _MIN_INTERVAL_SECONDS: + time.sleep(_MIN_INTERVAL_SECONDS - elapsed) + try: + response = requests.get( + f"{_SHODAN_BASE}{path}", + params=payload, + timeout=_REQUEST_TIMEOUT, + headers={"User-Agent": _USER_AGENT, "Accept": "application/json"}, + ) + finally: + _last_request_at = time.monotonic() + + if response.status_code == 401: + raise ShodanConnectorError("Shodan rejected the API key. Check SHODAN_API_KEY.", 401) + if response.status_code == 402: + raise ShodanConnectorError( + "Shodan returned payment/plan required. This feature needs a paid Shodan API plan.", + 402, + ) + if response.status_code == 429: + raise ShodanConnectorError( + "Shodan rate limit reached. Slow down queries and try again shortly.", + 429, + ) + if response.status_code >= 400: + detail = response.text.strip()[:240] or "Unexpected Shodan API error." + raise ShodanConnectorError(f"Shodan request failed: {detail}", response.status_code) + + try: + parsed = response.json() + except ValueError as exc: + raise ShodanConnectorError(f"Shodan returned invalid JSON: {exc}", 502) from exc + + if cache is not None: + cache[cache_key] = parsed + return parsed + + +def get_shodan_connector_status() -> dict[str, Any]: + has_key = bool(os.environ.get("SHODAN_API_KEY", "").strip()) + return { + "ok": True, + "configured": has_key, + "source": "Shodan", + "mode": "operator-supplied local overlay", + "paid_api": True, + "manual_only": True, + "background_polling": False, + "local_only": True, + "attribution": "Data from Shodan", + "warning": ( + "Shodan is a paid API. Searches use your local SHODAN_API_KEY, results stay local to " + "your ShadowBroker session by default, and any downstream use is your responsibility." + ), + "limits": { + "default_pages_per_search": _DEFAULT_SEARCH_PAGES, + "max_pages_per_search": _MAX_SEARCH_PAGES, + "cooldown_seconds": _MIN_INTERVAL_SECONDS, + }, + } + + +def search_shodan(query: str, page: int = 1, facets: list[str] | None = None) -> dict[str, Any]: + cleaned_query = _clean_query(query) + safe_page = max(1, min(int(page or 1), _MAX_SEARCH_PAGES)) + facet_list = [str(f).strip() for f in (facets or []) if str(f).strip()][:6] + params: dict[str, Any] = {"query": cleaned_query, "page": safe_page} + if facet_list: + params["facets"] = ",".join(facet_list) + raw = _request("/shodan/host/search", params=params, cache=_search_cache) + matches = [_normalize_match(match) for match in raw.get("matches") or [] if isinstance(match, dict)] + return { + "ok": True, + "source": "Shodan", + "attribution": "Data from Shodan", + "query": cleaned_query, + "page": safe_page, + "total": int(raw.get("total") or 0), + "matches": matches, + "facets": _normalize_facets(raw.get("facets")), + "note": "Operator-triggered Shodan results. Not part of ShadowBroker core feeds.", + } + + +def count_shodan(query: str, facets: list[str] | None = None) -> dict[str, Any]: + cleaned_query = _clean_query(query) + facet_list = [str(f).strip() for f in (facets or []) if str(f).strip()][:8] + params: dict[str, Any] = {"query": cleaned_query} + if facet_list: + params["facets"] = ",".join(facet_list) + raw = _request("/shodan/host/count", params=params, cache=_count_cache) + return { + "ok": True, + "source": "Shodan", + "attribution": "Data from Shodan", + "query": cleaned_query, + "total": int(raw.get("total") or 0), + "facets": _normalize_facets(raw.get("facets")), + "note": "Count/facets query only. No persistent ShadowBroker storage.", + } + + +def lookup_shodan_host(ip: str, history: bool = False) -> dict[str, Any]: + clean_ip = str(ip or "").strip() + if not clean_ip: + raise ShodanConnectorError("Host lookup requires an IP address.", 400) + raw = _request( + f"/shodan/host/{clean_ip}", + params={"history": "true" if history else "false"}, + cache=_host_cache, + ) + location = raw.get("location") or {} + host = { + "id": f"shodan-{clean_ip}-host", + "ip": str(raw.get("ip_str") or clean_ip), + "lat": location.get("latitude") if isinstance(location.get("latitude"), (int, float)) else None, + "lng": location.get("longitude") if isinstance(location.get("longitude"), (int, float)) else None, + "city": location.get("city"), + "region_code": location.get("region_code"), + "country_code": location.get("country_code"), + "country_name": location.get("country_name"), + "location_label": _location_label(location), + "asn": raw.get("asn"), + "org": raw.get("org"), + "isp": raw.get("isp"), + "os": raw.get("os"), + "hostnames": _normalize_string_list(raw.get("hostnames")), + "domains": _normalize_string_list(raw.get("domains")), + "tags": _normalize_string_list(raw.get("tags")), + "ports": [int(p) for p in (raw.get("ports") or []) if isinstance(p, int)], + "services": _normalize_services(raw.get("data")), + "vulns": _normalize_string_list(list((raw.get("vulns") or {}).keys()) if isinstance(raw.get("vulns"), dict) else raw.get("vulns"), limit=20), + "attribution": "Data from Shodan", + } + return { + "ok": True, + "source": "Shodan", + "attribution": "Data from Shodan", + "host": host, + "history": bool(history), + "note": "Operator-triggered Shodan host lookup. Not merged into ShadowBroker datasets.", + } diff --git a/backend/services/sigint_bridge.py b/backend/services/sigint_bridge.py new file mode 100644 index 00000000..ca3ea03e --- /dev/null +++ b/backend/services/sigint_bridge.py @@ -0,0 +1,1133 @@ +"""SIGINT Grid — unified radio intelligence bridge. + +Three protocol bridges feeding a shared signal buffer: + - APRS-IS: TCP to rotate.aprs2.net:14580 (amateur radio positions/weather) + - Meshtastic: MQTT to mqtt.meshtastic.org:1883 (mesh network messages) + - JS8Call: TCP to 127.0.0.1:2442 (HF digital mode, local radio only) + +Each bridge runs in a daemon thread and pushes parsed signals into a shared +collections.deque (thread-safe, bounded). The SIGINTGrid orchestrator merges +and deduplicates all signals on demand. +""" + +import json +import socket +import struct +import threading +import time +import logging +from collections import deque +from datetime import datetime, timezone + +from services.config import get_settings +from services.mesh.meshtastic_topics import build_subscription_topics, known_roots, parse_topic_metadata + +logger = logging.getLogger("services.sigint") + +# Maximum signals retained per bridge (prevents unbounded memory) +_MAX_SIGNALS = 500 +# Maximum age of signals before discard (seconds) +_MAX_AGE_S = 600 # 10 minutes + + +def _is_plausible_land(lat: float, lng: float) -> bool: + """Reject coordinates that are obviously in the middle of the ocean. + + Uses coarse bounding boxes for major landmasses. Not perfect, but filters + out the bulk of garbage coordinates from bad GPS / protobuf parsing. + Radio operators are on land (or near coasts), not mid-ocean. + """ + # Major landmass bounding boxes (generous margins for coastal/island coverage) + _LAND_BOXES = [ + # North America (incl. Caribbean, Central America) + (15, 72, -170, -50), + # South America + (-60, 15, -82, -34), + # Europe + (35, 72, -12, 45), + # Africa + (-36, 38, -18, 52), + # Asia (incl. Middle East, India, SE Asia) + (0, 75, 25, 180), + # Australia / Oceania + (-50, -8, 110, 180), + # New Zealand / Pacific islands + (-48, -10, 165, 180), + # Japan / Korea / Taiwan + (20, 46, 124, 146), + # Indonesia / Philippines + (-12, 20, 95, 130), + # UK / Ireland / Iceland + (50, 67, -25, 2), + # Alaska + (51, 72, -180, -130), + # Hawaii + (18, 23, -161, -154), + # Caribbean islands + (10, 28, -86, -59), + # Madagascar + (-26, -12, 43, 51), + ] + for min_lat, max_lat, min_lng, max_lng in _LAND_BOXES: + if min_lat <= lat <= max_lat and min_lng <= lng <= max_lng: + return True + return False + + +# ─── Emergency Lexicon (multilingual SOS/crisis keyword scanner) ────────────── +# Extracted from Pete's universal_translator.py — real Unicode keywords + +_EMERGENCY_LEXICON: dict[str, list[str]] = { + # English + "en": ["SOS", "MAYDAY", "EMERGENCY", "HELP", "MEDIC", "EVACUAT"], + # Mandarin (Chinese) + "zh": ["救命", "求助", "停电", "医生", "地震", "火灾", "爆炸"], + # Russian + "ru": ["помощь", "удар", "врач", "эвакуация", "пожар"], + # Ukrainian + "uk": ["допомога", "вогонь", "обстріл", "евакуація", "лікар"], + # Farsi (Persian) + "fa": ["کمک", "انفجار", "پزشک", "برق", "زلزله"], + # Arabic + "ar": ["مساعدة", "طبيب", "قنبلة", "ماء", "إغاثة"], + # Burmese (Myanmar) + "my": ["ကူညီပါ", "ဆေးဆရာ", "မီးပျက်"], + # Hebrew + "he": ["עזרה", "חובש", "פיצוץ", "אש"], + # Korean + "ko": ["도와주세요", "응급", "화재", "지진"], + # Japanese + "ja": ["助けて", "緊急", "地震", "火事", "避難"], +} + +# Flatten all keywords into a single set for fast scanning +_ALL_EMERGENCY_KEYWORDS: set[str] = set() +for _kws in _EMERGENCY_LEXICON.values(): + for _kw in _kws: + _ALL_EMERGENCY_KEYWORDS.add(_kw.upper()) + _ALL_EMERGENCY_KEYWORDS.add(_kw) # keep original case for CJK + + +def _scan_emergency(text: str) -> str | None: + """Check if text contains any emergency keyword. Returns matched keyword or None.""" + if not text: + return None + text_upper = text.upper() + for kw in _ALL_EMERGENCY_KEYWORDS: + if kw in text_upper or kw in text: + return kw + return None + + +# ─── APRS Symbol Decoding ──────────────────────────────────────────────────── + +# Primary table (/) symbol codes → human-readable labels +_APRS_SYMBOLS: dict[str, str] = { + "/-": "House/QTH", + "/!": "Police", + "/#": "Digipeater", + "/$": "Phone", + "/%": "DX Cluster", + "/&": "HF Gateway", + "/'": "Aircraft (small)", + "/(": "Mobile Sat", + "/)": "Wheelchair", + "/*": "Snowmobile", + "/+": "Red Cross", + "/,": "Boy Scout", + "/.": "Unknown/X", + "//": "Red Dot", + "/:": "Fire", + "/;": "Campground", + "/<": "Motorcycle", + "/=": "Railroad", + "/>": "Car", + "/?": "Server/Info", + "/@": "Hurricane/Tropical", + "/A": "Aid Station", + "/E": "Eyeball", + "/F": "Farm/Tractor", + "/H": "Hotel", + "/I": "TCP/IP", + "/K": "School", + "/N": "NTS Station", + "/O": "Balloon", + "/P": "Police", + "/R": "RV", + "/S": "Shuttle", + "/T": "SSTV", + "/U": "Bus", + "/W": "NWS Site", + "/Y": "Yacht/Sailboat", + "/[": "Jogger/Human", + "/\\": "Triangle", + "/^": "Aircraft (large)", + "/_": "Weather Station", + "/a": "Ambulance", + "/b": "Bicycle", + "/c": "Incident", + "/d": "Fire Dept", + "/e": "Horse", + "/f": "Fire Truck", + "/g": "Glider", + "/h": "Hospital", + "/i": "IOTA", + "/j": "Jeep", + "/k": "Truck", + "/l": "Laptop", + "/n": "Node/Relay", + "/o": "EOC", + "/p": "Rover/Dog", + "/r": "Antenna", + "/s": "Powerboat", + "/u": "Truck (18-wheel)", + "/v": "Van", + "/w": "Water Station", + "/y": "House+Yagi", +} + +# Alternate table (\) — common overrides +_APRS_SYMBOLS_ALT: dict[str, str] = { + "\\-": "House (HF)", + "\\>": "Car", + "\\#": "Digipeater (alt)", + "\\/": "Red Dot", + "\\&": "Gateway/Digi", + "\\^": "Aircraft", + "\\_": "WX Station", + "\\k": "SUV", + "\\n": "Node", +} + +# D-Star / DMR gateways use 'D' table prefix +_APRS_DSTAR: dict[str, str] = { + "D&": "D-Star/DMR Gateway", + "D#": "D-Star Digipeater", +} + + +def _decode_aprs_symbol(symbol: str) -> str: + """Decode APRS symbol table+code into a human-readable station type.""" + if not symbol or len(symbol) < 2: + return "Station" + return ( + _APRS_SYMBOLS.get(symbol) + or _APRS_SYMBOLS_ALT.get(symbol) + or _APRS_DSTAR.get(symbol) + or "Station" + ) + + +def _parse_aprs_comment(comment: str) -> dict: + """Extract structured metadata from APRS comment field. + + Returns dict with optional keys: frequency, altitude_ft, course, speed_knots, power + """ + import re + + meta: dict = {} + + # Frequency: e.g., "146.520MHz" or "439.01250MHz" + freq_match = re.search(r"(\d{2,3}\.\d{2,6})\s*MHz", comment, re.IGNORECASE) + if freq_match: + meta["frequency"] = f"{freq_match.group(1)} MHz" + + # Altitude: /A=NNNNNN (in feet) + alt_match = re.search(r"/A=(\d{6})", comment) + if alt_match: + alt = int(alt_match.group(1)) + if alt > 0: + meta["altitude_ft"] = alt + + # Course/Speed: CCC/SSS at start of comment (course deg / speed knots) + cs_match = re.match(r"^(\d{3})/(\d{3})", comment) + if cs_match: + course = int(cs_match.group(1)) + speed = int(cs_match.group(2)) + if speed > 0: + meta["course"] = course + meta["speed_knots"] = speed + + # Battery voltage: "Bat:X.XV" or "XX.XV" at end + batt_match = re.search(r"Bat[:\s]*(\d+\.\d+)\s*V", comment, re.IGNORECASE) + if batt_match: + meta["battery_v"] = float(batt_match.group(1)) + + # PHG (Power-Height-Gain-Directivity) + phg_match = re.search(r"PHG(\d)(\d)(\d)(\d)", comment) + if phg_match: + power_code = int(phg_match.group(1)) + power_watts = power_code**2 # APRS PHG power encoding + meta["power_watts"] = power_watts + + # Clean comment: strip leading course/speed, PHG, /A= cruft + clean = comment + clean = re.sub(r"^\d{3}/\d{3}/", "", clean) + clean = re.sub(r"/A=\d{6}", "", clean) + clean = re.sub(r"PHG\d{4,}", "", clean) + clean = clean.strip(" /") + if clean: + meta["status"] = clean[:80] + + return meta + + +# ─── APRS-IS Bridge ───────────────────────────────────────────────────────── + + +class APRSBridge: + """Connects to APRS-IS and parses position reports.""" + + HOST = "rotate.aprs2.net" + PORT = 14580 + # Read-only login (no callsign needed for receive-only) + LOGIN = "user N0CALL pass -1 vers ShadowBroker 1.0 filter r/0/0/25000\r\n" + CONFIDENCE = 0.7 + + def __init__(self): + self.signals: deque[dict] = deque(maxlen=_MAX_SIGNALS) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + + def start(self): + if self._thread and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True, name="aprs-bridge") + self._thread.start() + logger.info("APRS-IS bridge started") + + def stop(self): + self._stop.set() + + def _run(self): + while not self._stop.is_set(): + try: + self._connect_and_read() + except Exception as e: + logger.warning(f"APRS-IS connection error: {e}") + if not self._stop.is_set(): + time.sleep(15) # reconnect delay + + @staticmethod + def _decode_line(raw_bytes: bytes) -> str: + """Decode APRS packet bytes trying UTF-8 first, then GBK (Chinese), then latin-1.""" + try: + return raw_bytes.decode("utf-8") + except UnicodeDecodeError: + pass + try: + return raw_bytes.decode("gbk") + except UnicodeDecodeError: + pass + return raw_bytes.decode("latin-1") # latin-1 never fails (1:1 byte mapping) + + def _connect_and_read(self): + with socket.create_connection((self.HOST, self.PORT), timeout=30) as sock: + sock.settimeout(90) + # Read server banner + banner = sock.recv(512).decode("utf-8", errors="replace") + logger.info(f"APRS-IS: {banner.strip()}") + # Send login + sock.sendall(self.LOGIN.encode("ascii")) + buf = b"" + while not self._stop.is_set(): + try: + chunk = sock.recv(4096) + except socket.timeout: + # Send keepalive + sock.sendall(b"#keepalive\r\n") + continue + if not chunk: + break + buf += chunk + while b"\n" in buf: + line_bytes, buf = buf.split(b"\n", 1) + line_bytes = line_bytes.strip() + if not line_bytes or line_bytes.startswith(b"#"): + continue + line = self._decode_line(line_bytes) + self._parse_packet(line) + + def _parse_packet(self, raw: str): + """Parse an APRS packet and extract position if present.""" + try: + # Format: CALLSIGN>PATH:PAYLOAD + if ":" not in raw: + return + header, payload = raw.split(":", 1) + callsign = header.split(">")[0].strip() + if not callsign or callsign == "N0CALL": + return + + # Position reports start with ! @ / or = + if not payload or payload[0] not in "!@/=": + return + + # Try to extract lat/lng from uncompressed position + # Format: !DDMM.MMN/DDDMM.MMW... or similar + pos = payload[1:] + lat = self._parse_lat(pos[:8]) + lng = self._parse_lng(pos[9:18]) + if lat is None or lng is None: + return + + symbol = pos[8] + pos[18] if len(pos) > 18 else "" + comment = pos[19:].strip() if len(pos) > 19 else "" + + station_type = _decode_aprs_symbol(symbol) + meta = _parse_aprs_comment(comment) + + sig = { + "callsign": callsign, + "lat": lat, + "lng": lng, + "source": "aprs", + "confidence": self.CONFIDENCE, + "timestamp": datetime.now(timezone.utc).isoformat(), + "raw_message": raw[:200], + "symbol": symbol, + "station_type": station_type, + "comment": comment[:100], + } + # Merge parsed metadata into signal + if meta.get("frequency"): + sig["frequency"] = meta["frequency"] + if meta.get("altitude_ft"): + sig["altitude_ft"] = meta["altitude_ft"] + if meta.get("speed_knots"): + sig["speed_knots"] = meta["speed_knots"] + sig["course"] = meta.get("course", 0) + if meta.get("battery_v"): + sig["battery_v"] = meta["battery_v"] + if meta.get("power_watts"): + sig["power_watts"] = meta["power_watts"] + if meta.get("status"): + sig["status"] = meta["status"] + + # Emergency keyword scan across all text fields + emergency_kw = _scan_emergency(comment) or _scan_emergency(sig.get("status", "")) + if emergency_kw: + sig["emergency"] = True + sig["emergency_keyword"] = emergency_kw + + self.signals.append(sig) + except (ValueError, IndexError): + pass + + @staticmethod + def _parse_lat(s: str) -> float | None: + """Parse APRS latitude: DDMM.MMN""" + try: + if len(s) < 8: + return None + deg = int(s[:2]) + minutes = float(s[2:7]) + direction = s[7].upper() + lat = deg + minutes / 60.0 + if direction == "S": + lat = -lat + if -90 <= lat <= 90: + return round(lat, 5) + except (ValueError, IndexError): + pass + return None + + @staticmethod + def _parse_lng(s: str) -> float | None: + """Parse APRS longitude: DDDMM.MMW""" + try: + if len(s) < 9: + return None + deg = int(s[:3]) + minutes = float(s[3:8]) + direction = s[8].upper() + lng = deg + minutes / 60.0 + if direction == "W": + lng = -lng + if -180 <= lng <= 180: + return round(lng, 5) + except (ValueError, IndexError): + pass + return None + + +# ─── Meshtastic MQTT Bridge ───────────────────────────────────────────────── + + +class MeshtasticBridge: + """Connects to Meshtastic public MQTT broker for mesh network messages.""" + + HOST = "mqtt.meshtastic.org" + PORT = 1883 + USER = "meshdev" + PASS = "large4cats" + CONFIDENCE = 0.5 + + def __init__(self): + self.signals: deque[dict] = deque(maxlen=_MAX_SIGNALS) + self.messages: deque[dict] = deque(maxlen=500) + self._message_dedupe: dict[str, float] = {} + self._thread: threading.Thread | None = None + self._stop = threading.Event() + + def _dedupe_message( + self, + sender: str, + channel: str, + text: str, + recipient: str = "broadcast", + root: str = "", + ) -> bool: + now = time.time() + cutoff = now - 120 + for key, ts in list(self._message_dedupe.items()): + if ts < cutoff: + del self._message_dedupe[key] + key = f"{sender}:{recipient}:{root}:{channel}:{text}" + if key in self._message_dedupe: + return True + self._message_dedupe[key] = now + return False + + def start(self): + if self._thread and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True, name="mesh-bridge") + self._thread.start() + logger.info("Meshtastic MQTT bridge started") + + def stop(self): + self._stop.set() + + def _subscription_topics(self) -> list[str]: + settings = get_settings() + return build_subscription_topics( + extra_roots=str(getattr(settings, "MESH_MQTT_EXTRA_ROOTS", "") or ""), + extra_topics=str(getattr(settings, "MESH_MQTT_EXTRA_TOPICS", "") or ""), + include_defaults=bool(getattr(settings, "MESH_MQTT_INCLUDE_DEFAULT_ROOTS", True)), + ) + + def _run(self): + while not self._stop.is_set(): + try: + self._connect() + except Exception as e: + logger.warning(f"Meshtastic MQTT error: {e}") + if not self._stop.is_set(): + time.sleep(15) + + def _connect(self): + try: + import paho.mqtt.client as mqtt + except ImportError: + logger.error("paho-mqtt not installed — Meshtastic bridge disabled") + self._stop.set() + return + + topics = self._subscription_topics() + + def _on_connect(client, userdata, flags, rc): + if rc == 0: + logger.info(f"Meshtastic MQTT connected, subscribing to {topics}") + for topic in topics: + client.subscribe(topic, qos=0) + else: + logger.error(f"Meshtastic MQTT connection refused: rc={rc}") + + client = mqtt.Client(client_id="shadowbroker-mesh", protocol=mqtt.MQTTv311) + client.username_pw_set(self.USER, self.PASS) + client.on_connect = _on_connect + client.on_message = self._on_message + + client.connect(self.HOST, self.PORT, keepalive=60) + + while not self._stop.is_set(): + client.loop(timeout=1.0) + client.disconnect() + + def _on_message(self, client, userdata, msg): + """Parse Meshtastic MQTT messages — protobuf + AES decryption.""" + try: + payload = msg.payload + topic = msg.topic + + # Try JSON first (some nodes publish JSON on /json/ topics) + if "/json/" in topic: + try: + data = json.loads(payload) + self._ingest_data(data, topic) + return + except (json.JSONDecodeError, UnicodeDecodeError): + pass + + # Protobuf ServiceEnvelope (the standard format) + data = self._decode_protobuf(payload, topic) + if data: + # Text messages don't have positions — store in message log + if data.get("portnum") == "TEXT_MESSAGE_APP" and data.get("text"): + topic_meta = parse_topic_metadata(topic) + recipient = data.get("to", "broadcast") + if self._dedupe_message( + data.get("from", "???"), + topic_meta["channel"], + data["text"], + recipient, + topic_meta["root"], + ): + return + self.messages.appendleft( + { + "from": data.get("from", "???"), + "to": recipient, + "text": data["text"], + "region": topic_meta["region"], + "root": topic_meta["root"], + "channel": topic_meta["channel"], + "timestamp": datetime.utcnow().isoformat() + "Z", + } + ) + else: + self._ingest_data(data, topic) + + except Exception as e: + logger.debug(f"Meshtastic parse error: {e}") + + def _decode_protobuf(self, payload: bytes, topic: str) -> dict | None: + """Decode a Meshtastic ServiceEnvelope protobuf with AES decryption.""" + try: + from meshtastic import mesh_pb2, mqtt_pb2, portnums_pb2 + except ImportError: + return None + + try: + envelope = mqtt_pb2.ServiceEnvelope() + envelope.ParseFromString(payload) + except Exception: + return None + + packet = envelope.packet + if not packet or not packet.HasField("encrypted"): + # Already decoded or empty + if packet and packet.HasField("decoded"): + return self._extract_from_decoded(packet, topic) + return None + + # Decrypt with default LongFast PSK (hardcoded 16-byte AES-128 key) + try: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + # Meshtastic default channel key (firmware hardcoded for PSK=0x01) + default_key = bytes( + [ + 0xD4, + 0xF1, + 0xBB, + 0x3A, + 0x20, + 0x29, + 0x07, + 0x59, + 0xF0, + 0xBC, + 0xFF, + 0xAB, + 0xCF, + 0x4E, + 0x69, + 0x01, + ] + ) + + # Nonce: packetId (little-endian u64) + fromNode (little-endian u64) = 16 bytes + nonce = struct.pack(" dict | None: + """Extract data from an already-decoded MeshPacket.""" + decoded = packet.decoded + return self._extract_from_data(decoded, packet, topic) + + def _extract_from_data(self, data_msg, packet, topic: str) -> dict | None: + """Extract position/text from a decoded Data message.""" + try: + from meshtastic import mesh_pb2, portnums_pb2 + except ImportError: + return None + + portnum = data_msg.portnum + from_id = getattr(packet, "from", 0) + to_id = getattr(packet, "to", 0) + callsign = f"!{from_id:08x}" if from_id else topic.split("/")[-1] + + result = {"from": callsign} + if to_id == 0xFFFFFFFF: + result["to"] = "broadcast" + elif to_id: + result["to"] = f"!{to_id:08x}" + + if portnum == portnums_pb2.PortNum.POSITION_APP: + try: + pos = mesh_pb2.Position() + pos.ParseFromString(data_msg.payload) + if pos.latitude_i and pos.longitude_i: + result["latitude_i"] = pos.latitude_i + result["longitude_i"] = pos.longitude_i + if pos.altitude: + result["altitude"] = pos.altitude + return result + except Exception: + pass + + elif portnum == portnums_pb2.PortNum.TEXT_MESSAGE_APP: + try: + text = data_msg.payload.decode("utf-8", errors="replace") + result["text"] = text + result["portnum"] = "TEXT_MESSAGE_APP" + return result + except Exception: + pass + + elif portnum == portnums_pb2.PortNum.NODEINFO_APP: + try: + user = mesh_pb2.User() + user.ParseFromString(data_msg.payload) + if user.long_name: + result["long_name"] = user.long_name + if user.short_name: + result["short_name"] = user.short_name + # No position in nodeinfo + return None + except Exception: + pass + + return None + + def _ingest_data(self, data: dict, topic: str): + """Process a decoded data dict into a signal entry.""" + lat = data.get("latitude_i") or data.get("lat") + lng = data.get("longitude_i") or data.get("lng") or data.get("lon") + if lat is None or lng is None: + return + + # Meshtastic stores lat/lng as int32 × 1e-7 + if isinstance(lat, int) and abs(lat) > 1000: + lat = lat / 1e7 + if isinstance(lng, int) and abs(lng) > 1000: + lng = lng / 1e7 + + lat = float(lat) + lng = float(lng) + if not (-90 <= lat <= 90 and -180 <= lng <= 180): + return + if lat == 0.0 and lng == 0.0: + return + if abs(lat) < 0.1 and abs(lng) < 0.1: + return + if not _is_plausible_land(lat, lng): + return + + callsign = data.get("from", data.get("sender", topic.split("/")[-1])) + if isinstance(callsign, int): + callsign = f"!{callsign:08x}" + + topic_meta = parse_topic_metadata(topic) + + text_content = data.get("text", data.get("message", "")) + sig = { + "callsign": str(callsign)[:20], + "lat": round(lat, 5), + "lng": round(lng, 5), + "source": "meshtastic", + "region": topic_meta["region"], + "root": topic_meta["root"], + "channel": topic_meta["channel"], + "confidence": self.CONFIDENCE, + "timestamp": datetime.now(timezone.utc).isoformat(), + "raw_message": str(data)[:200], + "snr": data.get("snr"), + "altitude": data.get("altitude"), + } + if text_content: + sig["status"] = str(text_content)[:200] + emergency_kw = _scan_emergency(str(text_content)) + if emergency_kw: + sig["emergency"] = True + sig["emergency_keyword"] = emergency_kw + self.signals.append(sig) + + +# ─── JS8Call Bridge ────────────────────────────────────────────────────────── + + +class JS8CallBridge: + """Connects to local JS8Call API for HF digital mode intelligence. + + Requires JS8Call running locally with API enabled on port 2442. + Gracefully disables itself if not available. + """ + + HOST = "127.0.0.1" + PORT = 2442 + CONFIDENCE = 0.9 + + def __init__(self): + self.signals: deque[dict] = deque(maxlen=_MAX_SIGNALS) + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._available = True + + def start(self): + if self._thread and self._thread.is_alive(): + return + self._stop.clear() + self._available = True + self._thread = threading.Thread(target=self._run, daemon=True, name="js8-bridge") + self._thread.start() + logger.info("JS8Call bridge started (will check for local instance)") + + def stop(self): + self._stop.set() + + def _run(self): + failures = 0 + while not self._stop.is_set(): + try: + self._connect_and_read() + failures = 0 + except ConnectionRefusedError: + if self._available: + logger.info("JS8Call not running locally — bridge inactive (will retry)") + self._available = False + failures += 1 + except Exception as e: + logger.warning(f"JS8Call error: {e}") + failures += 1 + + # Exponential backoff: 30s, 60s, 120s, max 300s + delay = min(30 * (2 ** min(failures, 4)), 300) + self._stop.wait(delay) + + def _connect_and_read(self): + with socket.create_connection((self.HOST, self.PORT), timeout=10) as sock: + sock.settimeout(30) + if not self._available: + logger.info("JS8Call detected — bridge active") + self._available = True + buf = "" + while not self._stop.is_set(): + try: + data = sock.recv(4096).decode("utf-8", errors="replace") + except socket.timeout: + continue + if not data: + break + buf += data + while "\n" in buf: + line, buf = buf.split("\n", 1) + self._parse_message(line.strip()) + + def _parse_message(self, line: str): + """Parse a JS8Call API JSON message.""" + if not line: + return + try: + msg = json.loads(line) + msg_type = msg.get("type", "") + + # We care about RX.DIRECTED and RX.ACTIVITY messages + if msg_type not in ("RX.DIRECTED", "RX.ACTIVITY", "RX.SPOT"): + return + + params = msg.get("params", {}) + callsign = params.get("FROM", params.get("CALL", "")) + if not callsign: + return + + # Grid locator → lat/lng + grid = params.get("GRID", "") + lat, lng = self._grid_to_latlon(grid) + if lat is None: + return + + freq = params.get("FREQ", params.get("DIAL", 0)) + snr = params.get("SNR") + text = params.get("TEXT", "") + + self.signals.append( + { + "callsign": callsign[:20], + "lat": lat, + "lng": lng, + "source": "js8call", + "confidence": self.CONFIDENCE, + "timestamp": datetime.now(timezone.utc).isoformat(), + "raw_message": text[:200] if text else line[:200], + "frequency": freq, + "snr": snr, + "grid": grid, + } + ) + except (json.JSONDecodeError, KeyError): + pass + + @staticmethod + def _grid_to_latlon(grid: str) -> tuple[float | None, float | None]: + """Convert Maidenhead grid locator to lat/lng (center of grid square).""" + if not grid or len(grid) < 4: + return None, None + try: + grid = grid.upper() + lng = (ord(grid[0]) - ord("A")) * 20 - 180 + lat = (ord(grid[1]) - ord("A")) * 10 - 90 + lng += int(grid[2]) * 2 + lat += int(grid[3]) + # Add center offset for 4-char grid + if len(grid) >= 6: + lng += (ord(grid[4]) - ord("A")) * (2 / 24) + lat += (ord(grid[5]) - ord("A")) * (1 / 24) + lng += 1 / 24 + lat += 1 / 48 + else: + lng += 1 + lat += 0.5 + if -90 <= lat <= 90 and -180 <= lng <= 180: + return round(lat, 4), round(lng, 4) + except (IndexError, ValueError): + pass + return None, None + + +# ─── SIGINT Grid Orchestrator ──────────────────────────────────────────────── + + +class SIGINTGrid: + """Orchestrates all three SIGINT bridges and provides unified signal access.""" + + def __init__(self): + self.aprs = APRSBridge() + self.mesh = MeshtasticBridge() + self.js8 = JS8CallBridge() + self._started = False + + def start(self): + """Start all bridges (idempotent).""" + if self._started: + return + self._started = True + self.aprs.start() + self.mesh.start() + self.js8.start() + logger.info("SIGINT Grid started (APRS + Meshtastic + JS8Call)") + + def stop(self): + self.aprs.stop() + self.mesh.stop() + self.js8.stop() + self._started = False + + def get_all_signals(self) -> list[dict]: + """Merge signals from all bridges, deduplicate, and return newest first.""" + now = datetime.now(timezone.utc) + all_signals = [] + + for bridge in (self.aprs, self.mesh, self.js8): + for sig in list(bridge.signals): + # Filter stale signals + try: + ts = datetime.fromisoformat(sig["timestamp"]) + age = (now - ts).total_seconds() + if age > _MAX_AGE_S: + continue + except (ValueError, KeyError): + continue + all_signals.append(sig) + + # Deduplicate: keep latest per callsign+source + seen: dict[str, dict] = {} + for sig in all_signals: + key = f"{sig['callsign']}:{sig['source']}" + if key not in seen or sig["timestamp"] > seen[key]["timestamp"]: + seen[key] = sig + + result = list(seen.values()) + result.sort(key=lambda x: x["timestamp"], reverse=True) + return result + + def get_mesh_channel_stats(self, api_nodes: list[dict] | None = None) -> dict: + """Aggregate Meshtastic channel populations from live MQTT + API nodes. + + Returns { + "regions": { "US": {"nodes": 1234, "channels": {"LongFast": 45, ...}}, ... }, + "roots": { "US/rob/snd": {"nodes": 12, ...}, ... }, + "total_nodes": N, + "total_live": N, # from MQTT (last 10 min) + "total_api": N, # from map API + } + """ + now = datetime.now(timezone.utc) + regions: dict[str, dict] = {} + roots: dict[str, dict] = {} + seen_callsigns: set[str] = set() + live_count = 0 + + # Live MQTT signals (recent, have region + channel) + for sig in list(self.mesh.signals): + try: + ts = datetime.fromisoformat(sig["timestamp"]) + if (now - ts).total_seconds() > _MAX_AGE_S: + continue + except (ValueError, KeyError): + continue + + cs = sig.get("callsign", "") + region = sig.get("region", "?") + root = sig.get("root", region or "?") + channel = sig.get("channel", "LongFast") + if cs in seen_callsigns: + continue + seen_callsigns.add(cs) + live_count += 1 + + if region not in regions: + regions[region] = {"nodes": 0, "live": 0, "channels": {}} + regions[region]["nodes"] += 1 + regions[region]["live"] += 1 + regions[region]["channels"][channel] = regions[region]["channels"].get(channel, 0) + 1 + + if root not in roots: + roots[root] = {"nodes": 0, "live": 0, "region": region, "channels": {}} + roots[root]["nodes"] += 1 + roots[root]["live"] += 1 + roots[root]["channels"][channel] = roots[root]["channels"].get(channel, 0) + 1 + + # API nodes (global, no channel info but have region from topic/hardware) + api_count = 0 + if api_nodes: + for node in api_nodes: + cs = node.get("callsign", "") + if cs in seen_callsigns: + continue + seen_callsigns.add(cs) + api_count += 1 + # API nodes don't have region/channel — count as "MAP" region + region = "MAP" + if region not in regions: + regions[region] = {"nodes": 0, "live": 0, "channels": {}} + regions[region]["nodes"] += 1 + + # Also count messages per channel from the message log + channel_msgs: dict[str, int] = {} + for msg in list(self.mesh.messages): + ch = msg.get("channel", "LongFast") + channel_msgs[ch] = channel_msgs.get(ch, 0) + 1 + + return { + "regions": regions, + "roots": roots, + "known_roots": known_roots( + str(getattr(get_settings(), "MESH_MQTT_EXTRA_ROOTS", "") or ""), + include_defaults=bool(getattr(get_settings(), "MESH_MQTT_INCLUDE_DEFAULT_ROOTS", True)), + ), + "channel_messages": channel_msgs, + "total_nodes": len(seen_callsigns), + "total_live": live_count, + "total_api": api_count, + } + + @property + def status(self) -> dict: + """Return bridge status summary.""" + return { + "aprs": len(self.aprs.signals), + "meshtastic": len(self.mesh.signals), + "js8call": len(self.js8.signals), + "total": len(self.aprs.signals) + len(self.mesh.signals) + len(self.js8.signals), + } + + +# ─── APRS-IS Transmit (two-way messaging) ───────────────────────────────── + + +def send_aprs_message(callsign: str, passcode: str, target: str, message: str) -> dict: + """Send a text message to a specific callsign via APRS-IS. + + Requires a valid amateur radio callsign and passcode. + Returns {"ok": True/False, "detail": "..."}. + """ + if not callsign or not passcode or not target or not message: + return {"ok": False, "detail": "Missing required fields"} + if len(message) > 67: + message = message[:67] + + server = "rotate.aprs2.net" + port = 14580 + login = f"user {callsign} pass {passcode} vers ShadowBroker 1.0\r\n" + # APRS message format: SENDER>APRS,TCPIP*::TARGET :MESSAGE + # Target must be exactly 9 chars (padded with spaces) + packet = f"{callsign}>APRS,TCPIP*::{target.ljust(9)}:{message}\r\n" + + try: + with socket.create_connection((server, port), timeout=10) as sock: + sock.settimeout(10) + banner = sock.recv(512).decode("utf-8", errors="replace") + sock.sendall(login.encode("ascii")) + response = sock.recv(512).decode("utf-8", errors="replace") + if "verified" not in response.lower(): + return {"ok": False, "detail": "Login rejected — check callsign/passcode"} + sock.sendall(packet.encode("utf-8", errors="replace")) + logger.info(f"APRS TX: {callsign} → {target}: {message}") + return {"ok": True, "detail": f"Message sent to {target}"} + except (socket.timeout, ConnectionRefusedError, OSError) as e: + return {"ok": False, "detail": f"Connection error: {e}"} + + +# ─── Nearest KiwiSDR finder ─────────────────────────────────────────────── + + +def find_nearest_kiwisdr( + lat: float, lng: float, kiwisdr_list: list[dict], max_results: int = 3 +) -> list[dict]: + """Find the closest KiwiSDR receivers to a given coordinate. + + Uses simple Euclidean distance (fine for ranking nearby points). + Returns list of {name, url, distance_deg, bands, location}. + """ + import math + + results = [] + for sdr in kiwisdr_list: + slat = sdr.get("lat") + slng = sdr.get("lon") or sdr.get("lng") + if slat is None or slng is None: + continue + dist = math.sqrt((lat - slat) ** 2 + (lng - slng) ** 2) + results.append( + { + "name": sdr.get("name", "Unknown SDR"), + "url": sdr.get("url", ""), + "distance_deg": round(dist, 2), + "bands": sdr.get("bands", ""), + "location": sdr.get("location", ""), + "lat": slat, + "lon": slng, + } + ) + results.sort(key=lambda x: x["distance_deg"]) + return results[:max_results] + + +# Module-level singleton — bridges start on first fetch +sigint_grid = SIGINTGrid() diff --git a/backend/services/thermal_sentinel.py b/backend/services/thermal_sentinel.py new file mode 100644 index 00000000..73ea66f3 --- /dev/null +++ b/backend/services/thermal_sentinel.py @@ -0,0 +1,265 @@ +"""Thermal Sentinel — SWIR spectral anomaly detection via Sentinel-2 L2A. + +Queries Microsoft Planetary Computer for Sentinel-2 scenes near a given +coordinate and checks SWIR bands (B11 @ 1610nm, B12 @ 2190nm) for thermal +anomalies that could indicate kinetic events (explosions, fires, strikes). + +Thermal index: (B12 - B11) / (B12 + B11) — values > 0.1 suggest heat anomaly. + +Falls back to metadata-only analysis (cloud cover, scene age) when rasterio +is not available, and cross-references with FIRMS fire data for corroboration. +""" + +import logging +import requests +from datetime import datetime, timedelta +from cachetools import TTLCache + +logger = logging.getLogger(__name__) + +# Cache by rounded lat/lon (0.05° grid ~= 5km), TTL 30 min +_thermal_cache = TTLCache(maxsize=100, ttl=1800) + + +def search_thermal_anomaly( + lat: float, lng: float, radius_km: float = 10, days_back: int = 5 +) -> dict: + """Search for thermal anomalies near a coordinate using Sentinel-2 SWIR bands. + + Args: + lat, lng: Target coordinates + radius_km: Search radius in km (default 10) + days_back: How many days back to search (default 5) + + Returns: + dict with: verified (bool), confidence (float 0-1), scenes_checked (int), + thermal_index (float or None), latest_scene (str), firms_corroboration (bool) + """ + cache_key = f"{round(lat, 2)}_{round(lng, 2)}_{radius_km}_{days_back}" + if cache_key in _thermal_cache: + return _thermal_cache[cache_key] + + result = { + "verified": False, + "confidence": 0.0, + "scenes_checked": 0, + "thermal_index": None, + "latest_scene": None, + "latest_scene_date": None, + "cloud_cover": None, + "firms_corroboration": False, + "method": "metadata", # or "swir_analysis" if rasterio available + } + + try: + # Step 1: STAC search for Sentinel-2 scenes + scenes = _search_scenes(lat, lng, radius_km, days_back) + result["scenes_checked"] = len(scenes) + + if not scenes: + result["confidence"] = 0.0 + _thermal_cache[cache_key] = result + return result + + best_scene = scenes[0] + result["latest_scene"] = best_scene.get("id") + result["latest_scene_date"] = best_scene.get("datetime") + result["cloud_cover"] = best_scene.get("cloud_cover") + + # Step 2: Try SWIR band analysis if rasterio is available + swir_result = _analyze_swir_bands(best_scene, lat, lng) + if swir_result is not None: + result["thermal_index"] = swir_result["thermal_index"] + result["method"] = "swir_analysis" + if swir_result["thermal_index"] > 0.1: + result["verified"] = True + result["confidence"] = min(0.9, swir_result["thermal_index"] * 3) + elif swir_result["thermal_index"] > 0.05: + result["confidence"] = 0.3 + else: + # Fallback: metadata-only analysis + # Recent scene + low cloud cover = higher confidence that we CAN verify + scene_age_days = _scene_age_days(best_scene.get("datetime")) + if scene_age_days is not None and scene_age_days <= 2: + result["confidence"] = 0.2 # recent scene, but no SWIR analysis + else: + result["confidence"] = 0.1 + + # Step 3: Cross-reference with FIRMS fire data + firms_hit = _check_firms_corroboration(lat, lng, radius_km) + if firms_hit: + result["firms_corroboration"] = True + result["confidence"] = min(1.0, result["confidence"] + 0.4) + if not result["verified"]: + result["verified"] = True # FIRMS confirms thermal activity + + _thermal_cache[cache_key] = result + return result + + except ImportError: + logger.warning("pystac-client not installed — Thermal Sentinel unavailable") + result["confidence"] = 0.0 + return result + except Exception as e: + logger.error(f"Thermal Sentinel error for ({lat}, {lng}): {e}") + result["confidence"] = 0.0 + return result + + +def _search_scenes(lat: float, lng: float, radius_km: float, days_back: int) -> list[dict]: + """Search Planetary Computer STAC for Sentinel-2 scenes.""" + from pystac_client import Client + + catalog = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1") + end = datetime.utcnow() + start = end - timedelta(days=days_back) + + # Convert radius_km to rough bbox + dlat = radius_km / 111.0 + dlng = radius_km / ( + 111.0 + * max(0.1, abs(lat) < 89 and __import__("math").cos(__import__("math").radians(lat)) or 0.1) + ) + + bbox = [lng - dlng, lat - dlat, lng + dlng, lat + dlat] + + search = catalog.search( + collections=["sentinel-2-l2a"], + bbox=bbox, + datetime=f"{start.isoformat()}Z/{end.isoformat()}Z", + sortby=[{"field": "datetime", "direction": "desc"}], + max_items=5, + query={"eo:cloud_cover": {"lt": 50}}, + ) + + scenes = [] + for item in search.items(): + scenes.append( + { + "id": item.id, + "datetime": item.datetime.isoformat() if item.datetime else None, + "cloud_cover": item.properties.get("eo:cloud_cover"), + "b11_href": item.assets.get("B11", {}).href if "B11" in item.assets else None, + "b12_href": item.assets.get("B12", {}).href if "B12" in item.assets else None, + "item": item, + } + ) + + return scenes + + +def _analyze_swir_bands(scene: dict, lat: float, lng: float) -> dict | None: + """Analyze SWIR bands B11 and B12 for thermal anomalies. + + Returns dict with thermal_index or None if rasterio unavailable. + """ + try: + import rasterio + from rasterio.windows import from_bounds + except ImportError: + logger.debug("rasterio not installed — falling back to metadata analysis") + return None + + b11_href = scene.get("b11_href") + b12_href = scene.get("b12_href") + if not b11_href or not b12_href: + return None + + # Sign URLs for Azure blob access + item = scene.get("item") + if item: + try: + import planetary_computer + + item = planetary_computer.sign_item(item) + b11_href = item.assets["B11"].href + b12_href = item.assets["B12"].href + except (ImportError, KeyError, Exception) as e: + logger.debug(f"SWIR signing failed: {e}") + return None + + try: + # Read a small window around the target coordinate + # Sentinel-2 SWIR bands are 20m resolution + buffer_deg = 0.005 # ~500m window + + with rasterio.open(b11_href) as b11_ds: + window = from_bounds( + lng - buffer_deg, + lat - buffer_deg, + lng + buffer_deg, + lat + buffer_deg, + b11_ds.transform, + ) + b11_data = b11_ds.read(1, window=window).astype(float) + + with rasterio.open(b12_href) as b12_ds: + window = from_bounds( + lng - buffer_deg, + lat - buffer_deg, + lng + buffer_deg, + lat + buffer_deg, + b12_ds.transform, + ) + b12_data = b12_ds.read(1, window=window).astype(float) + + if b11_data.size == 0 or b12_data.size == 0: + return None + + # Compute thermal index: (B12 - B11) / (B12 + B11) + denom = b12_data + b11_data + # Avoid division by zero + valid = denom > 0 + if not valid.any(): + return None + + thermal_index = (b12_data[valid] - b11_data[valid]) / denom[valid] + max_ti = float(thermal_index.max()) + + return {"thermal_index": round(max_ti, 4)} + + except Exception as e: + logger.warning(f"SWIR band read failed: {e}") + return None + + +def _scene_age_days(dt_str: str | None) -> float | None: + """Calculate age of a scene in days.""" + if not dt_str: + return None + try: + scene_dt = datetime.fromisoformat(dt_str.replace("Z", "+00:00")) + now = datetime.now(scene_dt.tzinfo) if scene_dt.tzinfo else datetime.utcnow() + return (now - scene_dt).total_seconds() / 86400 + except (ValueError, TypeError): + return None + + +def _check_firms_corroboration(lat: float, lng: float, radius_km: float) -> bool: + """Check if FIRMS fire data corroborates thermal activity near the coordinate.""" + from services.fetchers._store import latest_data, _data_lock + + with _data_lock: + fires = list(latest_data.get("firms_fires", [])) + if not fires: + return False + + # Simple distance check (approximate, using equirectangular projection) + import math + + threshold_deg = radius_km / 111.0 + + for fire in fires: + try: + flat = fire.get("lat") or fire.get("latitude") + flng = fire.get("lng") or fire.get("longitude") + if flat is None or flng is None: + continue + dlat = abs(float(flat) - lat) + dlng = abs(float(flng) - lng) * math.cos(math.radians(lat)) + if math.sqrt(dlat**2 + dlng**2) <= threshold_deg: + return True + except (ValueError, TypeError): + continue + + return False diff --git a/backend/services/tinygs_fetcher.py b/backend/services/tinygs_fetcher.py new file mode 100644 index 00000000..7182f538 --- /dev/null +++ b/backend/services/tinygs_fetcher.py @@ -0,0 +1,385 @@ +""" +TinyGS LoRa satellite tracker — SGP4 orbit propagation + TinyGS telemetry. + +Primary position source: CelesTrak TLEs propagated via SGP4 (always available). +Secondary validation: TinyGS API confirms satellite is actively transmitting LoRa +and provides modulation/frequency/status metadata. + +CelesTrak Fair Use: TLEs fetched at most once per 24 hours, cached to disk. +TinyGS: polled every 5 minutes (their server has limited capacity). +""" + +import json +import logging +import math +import time +from datetime import datetime +from pathlib import Path + +import requests +from sgp4.api import Satrec, WGS72, jday + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# CelesTrak TLE cache (24-hour refresh, disk-backed) +# --------------------------------------------------------------------------- +_CELESTRAK_FETCH_INTERVAL = 86400 # 24 hours +_TLE_CACHE_PATH = Path(__file__).parent.parent / "data" / "tinygs_tle_cache.json" +_tle_cache: dict = {"data": None, "last_fetch": 0.0} + +# TinyGS API telemetry cache +_TINYGS_FETCH_INTERVAL = 300 # 5 minutes +_tinygs_telemetry: dict[str, dict] = {} # name_key → {modulation, frequency, status} +_tinygs_last_fetch: float = 0.0 +_tinygs_known_names: set[str] = set() # names seen from TinyGS API + +# Final result cache +_last_result: list[dict] = [] + +# CelesTrak GP groups containing LoRa / amateur cubesats +_CELESTRAK_GROUPS = ["amateur", "cubesat"] +_CELESTRAK_BASE = "https://celestrak.org/NORAD/elements/gp.php" + + +def _gmst(jd_ut1: float) -> float: + """Greenwich Mean Sidereal Time in radians from Julian Date.""" + t = (jd_ut1 - 2451545.0) / 36525.0 + gmst_sec = ( + 67310.54841 + + (876600.0 * 3600 + 8640184.812866) * t + + 0.093104 * t * t + - 6.2e-6 * t * t * t + ) + return (gmst_sec % 86400) / 86400.0 * 2 * math.pi + + +# --------------------------------------------------------------------------- +# CelesTrak TLE fetch + disk cache +# --------------------------------------------------------------------------- + + +def _load_tle_cache() -> list[dict] | None: + """Load TLE data from disk cache.""" + try: + if _TLE_CACHE_PATH.exists(): + import os + + age = time.time() - os.path.getmtime(str(_TLE_CACHE_PATH)) + if age < _CELESTRAK_FETCH_INTERVAL * 2: # accept up to 48h old cache + data = json.loads(_TLE_CACHE_PATH.read_text(encoding="utf-8")) + if isinstance(data, list) and len(data) > 0: + return data + except (IOError, json.JSONDecodeError, ValueError) as e: + logger.warning("TinyGS TLE: disk cache load failed: %s", e) + return None + + +def _save_tle_cache(data: list[dict]) -> None: + """Save TLE data to disk cache.""" + try: + _TLE_CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) + _TLE_CACHE_PATH.write_text( + json.dumps(data, ensure_ascii=False), encoding="utf-8" + ) + except (IOError, OSError) as e: + logger.warning("TinyGS TLE: disk cache save failed: %s", e) + + +def _fetch_celestrak_tles() -> list[dict]: + """Fetch GP data from CelesTrak for amateur + cubesat groups.""" + global _tle_cache + + now = time.time() + + # Return memory cache if fresh + if _tle_cache["data"] and now - _tle_cache["last_fetch"] < _CELESTRAK_FETCH_INTERVAL: + return _tle_cache["data"] + + # Try disk cache first + if not _tle_cache["data"]: + disk = _load_tle_cache() + if disk: + _tle_cache["data"] = disk + _tle_cache["last_fetch"] = now - _CELESTRAK_FETCH_INTERVAL + 3600 # re-check in 1h + logger.info("TinyGS TLE: loaded %d elements from disk cache", len(disk)) + + # Fetch fresh from CelesTrak + all_sats: dict[int, dict] = {} # keyed by NORAD_CAT_ID to deduplicate + for group in _CELESTRAK_GROUPS: + try: + resp = requests.get( + _CELESTRAK_BASE, + params={"GROUP": group, "FORMAT": "json"}, + timeout=20, + headers={ + "User-Agent": "ShadowBroker-OSINT/1.0 (CelesTrak fair-use)", + "Accept": "application/json", + }, + ) + resp.raise_for_status() + for s in resp.json(): + norad_id = s.get("NORAD_CAT_ID") + if norad_id: + all_sats[norad_id] = s + logger.info("TinyGS TLE: fetched %s group (%d sats)", group, len(resp.json())) + except (requests.RequestException, ValueError, KeyError) as e: + logger.warning("TinyGS TLE: CelesTrak %s fetch failed: %s", group, e) + + if all_sats: + result = list(all_sats.values()) + _tle_cache["data"] = result + _tle_cache["last_fetch"] = now + _save_tle_cache(result) + logger.info("TinyGS TLE: cached %d total orbital elements", len(result)) + return result + + # Fall back to whatever we have + return _tle_cache["data"] or [] + + +# --------------------------------------------------------------------------- +# SGP4 propagation +# --------------------------------------------------------------------------- + + +def _propagate_all(gp_data: list[dict]) -> dict[int, dict]: + """Propagate all satellites to current time via SGP4. + + Returns dict keyed by NORAD_CAT_ID with position/velocity data. + """ + now = datetime.utcnow() + jd, fr = jday( + now.year, now.month, now.day, + now.hour, now.minute, + now.second + now.microsecond / 1e6, + ) + + results: dict[int, dict] = {} + for s in gp_data: + try: + norad_id = s.get("NORAD_CAT_ID", 0) + mean_motion = s.get("MEAN_MOTION") + ecc = s.get("ECCENTRICITY") + incl = s.get("INCLINATION") + raan = s.get("RA_OF_ASC_NODE") + argp = s.get("ARG_OF_PERICENTER") + ma = s.get("MEAN_ANOMALY") + bstar = s.get("BSTAR", 0) + epoch_str = s.get("EPOCH") + obj_name = s.get("OBJECT_NAME", "") + + if mean_motion is None or ecc is None or incl is None or not epoch_str: + continue + + epoch_dt = datetime.strptime(epoch_str[:19], "%Y-%m-%dT%H:%M:%S") + epoch_jd, epoch_fr = jday( + epoch_dt.year, epoch_dt.month, epoch_dt.day, + epoch_dt.hour, epoch_dt.minute, epoch_dt.second, + ) + + sat_obj = Satrec() + sat_obj.sgp4init( + WGS72, "i", norad_id, + (epoch_jd + epoch_fr) - 2433281.5, + bstar, 0.0, 0.0, + ecc, + math.radians(argp), + math.radians(incl), + math.radians(ma), + mean_motion * 2 * math.pi / 1440.0, + math.radians(raan), + ) + + e, r, v = sat_obj.sgp4(jd, fr) + if e != 0: + continue + + x, y, z = r + gmst = _gmst(jd + fr) + lng_rad = math.atan2(y, x) - gmst + lat_rad = math.atan2(z, math.sqrt(x * x + y * y)) + alt_km = math.sqrt(x * x + y * y + z * z) - 6371.0 + + lat = math.degrees(lat_rad) + lng_deg = math.degrees(lng_rad) % 360 + lng = lng_deg - 360 if lng_deg > 180 else lng_deg + + # Ground-relative velocity for heading/speed + vx, vy, vz = v + omega_e = 7.2921159e-5 + vx_g = vx + omega_e * y + vy_g = vy - omega_e * x + cos_lat = math.cos(lat_rad) + sin_lat = math.sin(lat_rad) + cos_lng = math.cos(lng_rad + gmst) + sin_lng = math.sin(lng_rad + gmst) + v_east = -sin_lng * vx_g + cos_lng * vy_g + v_north = -sin_lat * cos_lng * vx_g - sin_lat * sin_lng * vy_g + cos_lat * vz + ground_speed_kms = math.sqrt(v_east**2 + v_north**2) + speed_knots = ground_speed_kms * 1943.84 + heading = math.degrees(math.atan2(v_east, v_north)) % 360 + + results[norad_id] = { + "name": obj_name, + "lat": round(lat, 4), + "lng": round(lng, 4), + "alt_km": round(alt_km, 1), + "heading": round(heading, 1), + "speed_knots": round(speed_knots, 0), + "norad_id": norad_id, + } + except (ValueError, TypeError, KeyError, OverflowError): + continue + + return results + + +# --------------------------------------------------------------------------- +# TinyGS API telemetry fetch +# --------------------------------------------------------------------------- + + +def _name_key(name: str) -> str: + """Normalise a satellite name for fuzzy matching.""" + return name.upper().replace("-", "").replace("_", "").replace(" ", "") + + +def _fetch_tinygs_telemetry() -> None: + """Fetch active satellite list from TinyGS for telemetry metadata.""" + global _tinygs_last_fetch, _tinygs_telemetry, _tinygs_known_names + + now = time.time() + if now - _tinygs_last_fetch < _TINYGS_FETCH_INTERVAL: + return + + try: + resp = requests.get( + "https://api.tinygs.com/v1/satellitesWorldmap", + timeout=15, + headers={ + "Accept": "application/json", + "User-Agent": "ShadowBroker-OSINT/1.0", + }, + ) + resp.raise_for_status() + new_telemetry: dict[str, dict] = {} + names: set[str] = set() + for s in resp.json(): + display_name = (s.get("displayName") or s.get("name") or "")[:80] + if not display_name: + continue + key = _name_key(display_name) + names.add(key) + tags = s.get("tags") or {} + new_telemetry[key] = { + "display_name": display_name, + "status": s.get("status", ""), + "modulation": ", ".join(tags.get("modulation", [])), + "frequency": ", ".join(str(f) for f in tags.get("frequency", [])), + } + _tinygs_telemetry = new_telemetry + _tinygs_known_names = names + _tinygs_last_fetch = now + logger.info("TinyGS telemetry: fetched %d active satellites", len(new_telemetry)) + except (requests.RequestException, ConnectionError, TimeoutError, ValueError, KeyError) as e: + logger.warning("TinyGS telemetry fetch failed (SGP4 still active): %s", e) + # Keep existing telemetry — don't clear on failure + + +# --------------------------------------------------------------------------- +# Merge SGP4 positions + TinyGS telemetry +# --------------------------------------------------------------------------- + + +def _match_name(celestrak_name: str) -> dict | None: + """Try to match a CelesTrak object name to TinyGS telemetry.""" + key = _name_key(celestrak_name) + # Exact match + if key in _tinygs_telemetry: + return _tinygs_telemetry[key] + # Substring match — CelesTrak name contains TinyGS name or vice versa + for tgs_key, tgs_data in _tinygs_telemetry.items(): + if tgs_key in key or key in tgs_key: + return tgs_data + return None + + +def fetch_tinygs_satellites() -> list[dict]: + """Fetch LoRa satellite positions via SGP4 + TinyGS telemetry merge. + + 1. Propagate cached CelesTrak TLEs via SGP4 (instant, no network needed) + 2. Attempt TinyGS API for telemetry (modulation, frequency, status) + 3. Merge: SGP4 provides position, TinyGS provides metadata + 4. Filter to only satellites known to TinyGS (if we have TinyGS data) + """ + global _last_result + + # Step 1: Get TLE data (from cache or CelesTrak) + gp_data = _fetch_celestrak_tles() + if not gp_data: + logger.warning("TinyGS: no TLE data available") + return _last_result or [] + + # Step 2: Try to fetch TinyGS telemetry (non-blocking, uses cache) + _fetch_tinygs_telemetry() + + # Step 3: Propagate all satellites via SGP4 + propagated = _propagate_all(gp_data) + if not propagated: + logger.warning("TinyGS: SGP4 propagation returned no results") + return _last_result or [] + + # Step 4: Merge and filter + sats: list[dict] = [] + have_tinygs = bool(_tinygs_known_names) + + for norad_id, pos in propagated.items(): + celestrak_name = pos["name"] + telemetry = _match_name(celestrak_name) + + # If we have TinyGS data, only show satellites that TinyGS knows about + # (filters out non-LoRa amateur/cubesats from the CelesTrak groups) + if have_tinygs and telemetry is None: + continue + + entry = { + "name": telemetry["display_name"] if telemetry else celestrak_name, + "lat": pos["lat"], + "lng": pos["lng"], + "heading": pos["heading"], + "speed_knots": pos["speed_knots"], + "alt_km": pos["alt_km"], + "status": telemetry.get("status", "") if telemetry else "", + "modulation": telemetry.get("modulation", "") if telemetry else "", + "frequency": telemetry.get("frequency", "") if telemetry else "", + "sgp4_propagated": True, + "tinygs_confirmed": telemetry is not None, + } + sats.append(entry) + + # If we have no TinyGS data at all (API never responded), show all propagated + # sats from the amateur group only (smaller, more relevant set) + if not have_tinygs and not sats: + for norad_id, pos in propagated.items(): + sats.append({ + "name": pos["name"], + "lat": pos["lat"], + "lng": pos["lng"], + "heading": pos["heading"], + "speed_knots": pos["speed_knots"], + "alt_km": pos["alt_km"], + "status": "", + "modulation": "", + "frequency": "", + "sgp4_propagated": True, + "tinygs_confirmed": False, + }) + + _last_result = sats + logger.info( + "TinyGS: %d satellites (SGP4 propagated, %d TinyGS confirmed)", + len(sats), + sum(1 for s in sats if s.get("tinygs_confirmed")), + ) + return sats diff --git a/backend/services/unusual_whales_connector.py b/backend/services/unusual_whales_connector.py new file mode 100644 index 00000000..f421c729 --- /dev/null +++ b/backend/services/unusual_whales_connector.py @@ -0,0 +1,316 @@ +""" +Finnhub API connector. + +Provides defense stock quotes, congressional trading data, and insider +transactions — all from Finnhub's free tier (60 calls/min). + +File kept at this path for git history; the module is referenced as +services.unusual_whales_connector in main.py imports but the public +surface is entirely Finnhub now. +""" + +from __future__ import annotations + +import logging +import os +import threading +import time +from typing import Any + +import math +import requests +from cachetools import TTLCache + +logger = logging.getLogger(__name__) + +_FINNHUB_BASE = "https://finnhub.io/api/v1" +_USER_AGENT = "ShadowBroker/0.9.6 Finnhub connector" +_REQUEST_TIMEOUT = 12 +_MIN_INTERVAL_SECONDS = 0.35 # Stay well under 60 calls/min + +# Tickers we poll for congress trades & insider activity +WATCHED_TICKERS = [ + "NVDA", "AAPL", "TSLA", "MSFT", "GOOGL", "AMZN", "META", + "RTX", "LMT", "NOC", "GD", "BA", "PLTR", +] + +# Defense + oil tickers for quotes (replaces yfinance) +QUOTE_TICKERS = [ + ("RTX", "RTX"), ("LMT", "LMT"), ("NOC", "NOC"), + ("GD", "GD"), ("BA", "BA"), ("PLTR", "PLTR"), +] +CRYPTO_TICKERS = [ + ("BTC", "BINANCE:BTCUSDT"), + ("ETH", "BINANCE:ETHUSDT"), +] + +_quote_cache: TTLCache[str, dict[str, Any]] = TTLCache(maxsize=32, ttl=300) +_congress_cache: TTLCache[str, list[dict[str, Any]]] = TTLCache(maxsize=32, ttl=600) +_insider_cache: TTLCache[str, list[dict[str, Any]]] = TTLCache(maxsize=32, ttl=600) + +_request_lock = threading.Lock() +_last_request_at = 0.0 + + +class FinnhubConnectorError(Exception): + def __init__(self, detail: str, status_code: int = 400): + super().__init__(detail) + self.detail = detail + self.status_code = status_code + + +# Keep old name as alias for main.py imports +UWConnectorError = FinnhubConnectorError + + +def _get_api_key() -> str: + api_key = os.environ.get("FINNHUB_API_KEY", "").strip() + if not api_key: + raise FinnhubConnectorError( + "Finnhub API key not configured. Add FINNHUB_API_KEY in Settings > API Keys (free at finnhub.io).", + status_code=428, + ) + return api_key + + +def _request(path: str, params: dict[str, Any] | None = None) -> Any: + """Rate-limited GET to Finnhub. Returns parsed JSON.""" + api_key = _get_api_key() + payload = dict(params or {}) + payload["token"] = api_key + + global _last_request_at + with _request_lock: + elapsed = time.monotonic() - _last_request_at + if elapsed < _MIN_INTERVAL_SECONDS: + time.sleep(_MIN_INTERVAL_SECONDS - elapsed) + try: + response = requests.get( + f"{_FINNHUB_BASE}{path}", + params=payload, + timeout=_REQUEST_TIMEOUT, + headers={"User-Agent": _USER_AGENT, "Accept": "application/json"}, + ) + finally: + _last_request_at = time.monotonic() + + if response.status_code == 401: + raise FinnhubConnectorError("Finnhub rejected the API key. Check FINNHUB_API_KEY.", 401) + if response.status_code == 403: + raise FinnhubConnectorError( + "Finnhub returned 403 — this endpoint may require a premium plan.", 403 + ) + if response.status_code == 429: + raise FinnhubConnectorError( + "Finnhub rate limit reached (60/min free). Try again shortly.", 429 + ) + if response.status_code >= 400: + detail = response.text.strip()[:240] or "Unexpected Finnhub API error." + raise FinnhubConnectorError(f"Finnhub request failed: {detail}", response.status_code) + + try: + return response.json() + except ValueError as exc: + raise FinnhubConnectorError(f"Finnhub returned invalid JSON: {exc}", 502) from exc + + +# --------------------------------------------------------------------------- +# Status +# --------------------------------------------------------------------------- +def get_uw_status() -> dict[str, Any]: + """Status check — kept as get_uw_status for route compatibility.""" + has_key = bool(os.environ.get("FINNHUB_API_KEY", "").strip()) + return { + "ok": True, + "configured": has_key, + "source": "Finnhub", + "attribution": "Data from Finnhub", + "mode": "free-tier market intelligence", + "local_only": True, + } + + +# --------------------------------------------------------------------------- +# Stock Quotes +# --------------------------------------------------------------------------- +def fetch_defense_quotes() -> dict[str, dict[str, Any]]: + """Fetch real-time quotes for defense tickers. Returns {ticker: {...}}.""" + results: dict[str, dict[str, Any]] = {} + for label, symbol in QUOTE_TICKERS: + cache_key = f"quote:{symbol}" + if cache_key in _quote_cache: + results[label] = _quote_cache[cache_key] + continue + try: + raw = _request("/quote", {"symbol": symbol}) + if raw and raw.get("c"): + entry = { + "price": round(float(raw["c"]), 2), + "change_percent": round(float(raw.get("dp") or 0), 2), + "up": float(raw.get("dp") or 0) >= 0, + } + results[label] = entry + _quote_cache[cache_key] = entry + except FinnhubConnectorError: + logger.warning(f"Finnhub quote failed for {symbol}") + except Exception as e: + logger.warning(f"Finnhub quote error for {symbol}: {e}") + # Crypto quotes via /crypto/candle + for label, symbol in CRYPTO_TICKERS: + cache_key = f"quote:{symbol}" + if cache_key in _quote_cache: + results[label] = _quote_cache[cache_key] + continue + try: + now = int(time.time()) + raw = _request("/crypto/candle", { + "symbol": symbol, + "resolution": "D", + "from": now - 172800, # 2 days back + "to": now, + }) + closes = raw.get("c") or [] + if len(closes) >= 1: + current = float(closes[-1]) + prev = float(closes[-2]) if len(closes) >= 2 else current + change_pct = ((current - prev) / prev * 100) if prev else 0 + if math.isfinite(current) and math.isfinite(change_pct): + entry = { + "price": round(current, 2), + "change_percent": round(change_pct, 2), + "up": change_pct >= 0, + "crypto": True, + } + results[label] = entry + _quote_cache[cache_key] = entry + except FinnhubConnectorError: + logger.warning(f"Finnhub crypto quote failed for {symbol}") + except Exception as e: + logger.warning(f"Finnhub crypto quote error for {symbol}: {e}") + + return results + + +# --------------------------------------------------------------------------- +# Congressional Trading +# --------------------------------------------------------------------------- +def _normalize_congress_trade(raw: dict[str, Any], symbol: str) -> dict[str, Any]: + amount_from = raw.get("amountFrom") or 0 + amount_to = raw.get("amountTo") or 0 + if amount_from and amount_to: + amount_range = f"${int(amount_from):,}–${int(amount_to):,}" + elif amount_to: + amount_range = f"Up to ${int(amount_to):,}" + else: + amount_range = "" + return { + "politician_name": str(raw.get("name") or "Unknown"), + "chamber": str(raw.get("position") or "unknown").lower(), + "filing_date": str(raw.get("filingDate") or raw.get("transactionDate") or ""), + "transaction_date": str(raw.get("transactionDate") or ""), + "ticker": symbol, + "asset_name": str(raw.get("assetName") or ""), + "transaction_type": str(raw.get("transactionType") or ""), + "amount_range": amount_range, + "owner_type": str(raw.get("ownerType") or ""), + } + + +def fetch_congress_trades() -> dict[str, Any]: + """Fetch congressional trades across watched tickers.""" + all_trades: list[dict[str, Any]] = [] + for symbol in WATCHED_TICKERS: + cache_key = f"congress:{symbol}" + if cache_key in _congress_cache: + all_trades.extend(_congress_cache[cache_key]) + continue + try: + raw = _request("/stock/congressional-trading", {"symbol": symbol}) + data = raw.get("data") or [] + if isinstance(data, list): + normalized = [_normalize_congress_trade(t, symbol) for t in data[:10] if isinstance(t, dict)] + _congress_cache[cache_key] = normalized + all_trades.extend(normalized) + except FinnhubConnectorError as e: + if e.status_code in (403, 402): + logger.info(f"Congressional trading endpoint not available on free tier for {symbol}") + # Cache empty to avoid re-hitting a premium endpoint + _congress_cache[cache_key] = [] + else: + logger.warning(f"Finnhub congress fetch failed for {symbol}: {e.detail}") + except Exception as e: + logger.warning(f"Finnhub congress error for {symbol}: {e}") + + # Sort by filing date descending + all_trades.sort(key=lambda t: t.get("filing_date", ""), reverse=True) + return { + "ok": True, + "source": "Finnhub", + "attribution": "Data from Finnhub", + "trades": all_trades[:50], + } + + +# --------------------------------------------------------------------------- +# Insider Transactions +# --------------------------------------------------------------------------- +def _normalize_insider(raw: dict[str, Any]) -> dict[str, Any]: + return { + "name": str(raw.get("name") or "Unknown"), + "ticker": str(raw.get("symbol") or ""), + "share": int(raw.get("share") or 0), + "change": int(raw.get("change") or 0), + "filing_date": str(raw.get("filingDate") or ""), + "transaction_date": str(raw.get("transactionDate") or ""), + "transaction_code": str(raw.get("transactionCode") or ""), + "transaction_price": float(raw.get("transactionPrice") or 0), + } + + +def fetch_insider_transactions() -> dict[str, Any]: + """Fetch insider transactions across watched tickers.""" + all_insiders: list[dict[str, Any]] = [] + for symbol in WATCHED_TICKERS: + cache_key = f"insider:{symbol}" + if cache_key in _insider_cache: + all_insiders.extend(_insider_cache[cache_key]) + continue + try: + raw = _request("/stock/insider-transactions", {"symbol": symbol}) + data = raw.get("data") or [] + if isinstance(data, list): + normalized = [_normalize_insider(t) for t in data[:8] if isinstance(t, dict)] + _insider_cache[cache_key] = normalized + all_insiders.extend(normalized) + except FinnhubConnectorError as e: + logger.warning(f"Finnhub insider fetch failed for {symbol}: {e.detail}") + except Exception as e: + logger.warning(f"Finnhub insider error for {symbol}: {e}") + + all_insiders.sort(key=lambda t: t.get("filing_date", ""), reverse=True) + return { + "ok": True, + "source": "Finnhub", + "attribution": "Data from Finnhub", + "transactions": all_insiders[:50], + } + + +# --------------------------------------------------------------------------- +# Aliases for backward compatibility with main.py imports +# --------------------------------------------------------------------------- +def fetch_darkpool_recent() -> dict[str, Any]: + """Replaced — now returns insider transactions instead of dark pool.""" + return fetch_insider_transactions() + + +def fetch_flow_alerts() -> dict[str, Any]: + """Replaced — now returns defense quotes formatted as alerts.""" + quotes = fetch_defense_quotes() + return { + "ok": True, + "source": "Finnhub", + "attribution": "Data from Finnhub", + "alerts": [], + "quotes": quotes, + } diff --git a/backend/services/updater.py b/backend/services/updater.py index ea2af798..ca4de95a 100644 --- a/backend/services/updater.py +++ b/backend/services/updater.py @@ -5,6 +5,7 @@ perform_update(project_root) -> dict (download + backup + extract) schedule_restart(project_root) (spawn detached start script, then exit) """ + import os import sys import logging @@ -13,6 +14,8 @@ import tempfile import time import zipfile +import hashlib +from urllib.parse import urlparse from datetime import datetime from pathlib import Path @@ -21,17 +24,41 @@ logger = logging.getLogger(__name__) GITHUB_RELEASES_URL = "https://api.github.com/repos/BigBodyCobain/Shadowbroker/releases/latest" +GITHUB_RELEASES_PAGE_URL = "https://github.com/BigBodyCobain/Shadowbroker/releases/latest" +_EXPECTED_SHA256 = os.environ.get("MESH_UPDATE_SHA256", "").strip().lower() +_ALLOWED_UPDATE_HOSTS = { + "api.github.com", + "github.com", + "objects.githubusercontent.com", + "release-assets.githubusercontent.com", + "github-releases.githubusercontent.com", +} # --------------------------------------------------------------------------- # Protected patterns — files/dirs that must NEVER be overwritten during update # --------------------------------------------------------------------------- -_PROTECTED_DIRS = {"venv", "node_modules", ".next", "__pycache__", ".git", ".github", ".claude"} -_PROTECTED_EXTENSIONS = {".db", ".sqlite"} +_PROTECTED_DIRS = { + "venv", "node_modules", ".next", "__pycache__", ".git", ".github", ".claude", + "_domain_keys", "node-local", "gate_persona", "gate_session", "dm_alias", + "root", "transport", "reputation", +} +_PROTECTED_EXTENSIONS = {".db", ".sqlite", ".key", ".pem", ".bin"} _PROTECTED_NAMES = { ".env", "ais_cache.json", "carrier_cache.json", "geocode_cache.json", + "infonet.json", + "infonet.json.bak", + "peer_store.json", + "node.json", + "wormhole.json", + "wormhole_status.json", + "wormhole_secure_store.key", + "dm_token_pepper.key", + "voter_blind_salt.bin", + "reputation_ledger.json", + "gates.json", } @@ -55,19 +82,39 @@ def _is_protected(rel_path: str) -> bool: return False +def _validate_update_url(url: str, *, allow_release_page: bool = False) -> str: + parsed = urlparse(str(url or "").strip()) + host = (parsed.hostname or "").strip().lower() + if parsed.scheme != "https": + raise RuntimeError("Updater refused a non-HTTPS release URL") + if parsed.username or parsed.password: + raise RuntimeError("Updater refused a credentialed release URL") + if not host or host not in _ALLOWED_UPDATE_HOSTS: + raise RuntimeError(f"Updater refused an untrusted release host: {host or 'unknown'}") + if parsed.port not in (None, 443): + raise RuntimeError("Updater refused a non-standard release port") + if not allow_release_page and host == "github.com" and "/releases/" not in parsed.path: + raise RuntimeError("Updater refused a non-release GitHub URL") + return parsed.geturl() + + # --------------------------------------------------------------------------- # Download # --------------------------------------------------------------------------- def _download_release(temp_dir: str) -> tuple: """Fetch latest release info and download the zip asset. - Returns (zip_path, version_tag, download_url). + Returns (zip_path, version_tag, download_url, release_url). """ logger.info("Fetching latest release info from GitHub...") + _validate_update_url(GITHUB_RELEASES_URL) resp = requests.get(GITHUB_RELEASES_URL, timeout=15) resp.raise_for_status() + _validate_update_url(resp.url) release = resp.json() tag = release.get("tag_name", "unknown") + release_url = str(release.get("html_url") or GITHUB_RELEASES_PAGE_URL).strip() + _validate_update_url(release_url, allow_release_page=True) assets = release.get("assets", []) # Find the .zip asset @@ -80,11 +127,13 @@ def _download_release(temp_dir: str) -> tuple: if not zip_url: raise RuntimeError("No .zip asset found in the latest release") + _validate_update_url(zip_url) logger.info(f"Downloading {zip_url} ...") zip_path = os.path.join(temp_dir, "update.zip") with requests.get(zip_url, stream=True, timeout=120) as dl: dl.raise_for_status() + _validate_update_url(dl.url) with open(zip_path, "wb") as f: for chunk in dl.iter_content(chunk_size=1024 * 64): f.write(chunk) @@ -94,7 +143,19 @@ def _download_release(temp_dir: str) -> tuple: size_mb = os.path.getsize(zip_path) / (1024 * 1024) logger.info(f"Downloaded {size_mb:.1f} MB — ZIP validated OK") - return zip_path, tag, zip_url + return zip_path, tag, zip_url, release_url + + +def _validate_zip_hash(zip_path: str) -> None: + if not _EXPECTED_SHA256: + return + h = hashlib.sha256() + with open(zip_path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 128), b""): + h.update(chunk) + digest = h.hexdigest().lower() + if digest != _EXPECTED_SHA256: + raise RuntimeError("Update SHA-256 mismatch") # --------------------------------------------------------------------------- @@ -142,6 +203,16 @@ def _extract_and_copy(zip_path: str, project_root: str, temp_dir: str) -> int: extract_dir = os.path.join(temp_dir, "extracted") logger.info("Extracting update zip...") with zipfile.ZipFile(zip_path, "r") as zf: + extract_root = Path(extract_dir).resolve() + for member in zf.infolist(): + try: + target = (extract_root / member.filename).resolve() + except OSError as exc: + raise RuntimeError(f"Updater refused archive entry {member.filename}: {exc}") from exc + try: + target.relative_to(extract_root) + except ValueError: + raise RuntimeError(f"Updater refused archive path traversal entry: {member.filename}") zf.extractall(extract_dir) # Detect wrapper folder: if extracted root has a single directory that @@ -245,8 +316,11 @@ def perform_update(project_root: str) -> dict: separately after the HTTP response has been sent. """ temp_dir = tempfile.mkdtemp(prefix="sb_update_") + manual_url = GITHUB_RELEASES_PAGE_URL try: - zip_path, version, url = _download_release(temp_dir) + zip_path, version, url, release_url = _download_release(temp_dir) + manual_url = release_url or manual_url + _validate_zip_hash(zip_path) backup_path = _backup_current(project_root, temp_dir) copied = _extract_and_copy(zip_path, project_root, temp_dir) @@ -255,6 +329,9 @@ def perform_update(project_root: str) -> dict: "version": version, "files_updated": copied, "backup_path": backup_path, + "manual_url": manual_url, + "release_url": release_url, + "download_url": url, "message": f"Updated to {version} — {copied} files replaced. Restarting...", } except Exception as e: @@ -262,4 +339,5 @@ def perform_update(project_root: str) -> dict: return { "status": "error", "message": str(e), + "manual_url": manual_url, } diff --git a/backend/services/wormhole_settings.py b/backend/services/wormhole_settings.py new file mode 100644 index 00000000..f3f7c939 --- /dev/null +++ b/backend/services/wormhole_settings.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +DATA_DIR = Path(__file__).parent.parent / "data" +WORMHOLE_FILE = DATA_DIR / "wormhole.json" +_cache: dict | None = None +_cache_ts: float = 0.0 +_CACHE_TTL = 5.0 # seconds +_DEFAULTS = { + "enabled": False, + "transport": "direct", + "socks_proxy": "", + "socks_dns": True, + "privacy_profile": "default", + "anonymous_mode": False, +} + + +def _safe_int(val, default=0) -> int: + try: + return int(val) + except (TypeError, ValueError): + return default + + +def read_wormhole_settings() -> dict: + global _cache, _cache_ts + now = time.monotonic() + if _cache is not None and (now - _cache_ts) < _CACHE_TTL: + return _cache + if not WORMHOLE_FILE.exists(): + result = {**_DEFAULTS, "updated_at": 0} + else: + try: + data = json.loads(WORMHOLE_FILE.read_text(encoding="utf-8")) + except Exception: + result = {**_DEFAULTS, "updated_at": 0} + else: + result = { + "enabled": bool(data.get("enabled", _DEFAULTS["enabled"])), + "transport": str(data.get("transport", _DEFAULTS["transport"]) or _DEFAULTS["transport"]), + "socks_proxy": str(data.get("socks_proxy", _DEFAULTS["socks_proxy"]) or ""), + "socks_dns": bool(data.get("socks_dns", _DEFAULTS["socks_dns"])), + "privacy_profile": str( + data.get("privacy_profile", _DEFAULTS["privacy_profile"]) or _DEFAULTS["privacy_profile"] + ), + "anonymous_mode": bool(data.get("anonymous_mode", _DEFAULTS["anonymous_mode"])), + "updated_at": _safe_int(data.get("updated_at", 0) or 0), + } + _cache = result + _cache_ts = now + return result + + +def write_wormhole_settings( + *, + enabled: bool | None = None, + transport: str | None = None, + socks_proxy: str | None = None, + socks_dns: bool | None = None, + privacy_profile: str | None = None, + anonymous_mode: bool | None = None, +) -> dict: + DATA_DIR.mkdir(parents=True, exist_ok=True) + existing = read_wormhole_settings() + payload = { + "enabled": bool(existing.get("enabled")) if enabled is None else bool(enabled), + "transport": existing.get("transport", _DEFAULTS["transport"]) + if transport is None + else str(transport), + "socks_proxy": existing.get("socks_proxy", "") + if socks_proxy is None + else str(socks_proxy), + "socks_dns": bool(existing.get("socks_dns", _DEFAULTS["socks_dns"])) + if socks_dns is None + else bool(socks_dns), + "privacy_profile": existing.get("privacy_profile", _DEFAULTS["privacy_profile"]) + if privacy_profile is None + else str(privacy_profile), + "anonymous_mode": bool(existing.get("anonymous_mode", _DEFAULTS["anonymous_mode"])) + if anonymous_mode is None + else bool(anonymous_mode), + "updated_at": int(time.time()), + } + WORMHOLE_FILE.write_text(json.dumps(payload, indent=2), encoding="utf-8") + global _cache, _cache_ts + _cache = payload + _cache_ts = time.monotonic() + return payload diff --git a/backend/services/wormhole_status.py b/backend/services/wormhole_status.py new file mode 100644 index 00000000..e5e3d82c --- /dev/null +++ b/backend/services/wormhole_status.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +DATA_DIR = Path(__file__).parent.parent / "data" +STATUS_FILE = DATA_DIR / "wormhole_status.json" + +_DEFAULTS = { + "last_restart": 0, + "last_start": 0, + "reason": "", + "transport": "", + "proxy": "", + "transport_active": "", + "proxy_active": "", + "installed": False, + "configured": False, + "running": False, + "ready": False, + "pid": 0, + "started_at": 0, + "last_error": "", + "privacy_level_effective": "default", +} + + +def _safe_int(val, default=0) -> int: + try: + return int(val) + except (TypeError, ValueError): + return default + + +def read_wormhole_status() -> dict: + if not STATUS_FILE.exists(): + return dict(_DEFAULTS) + try: + data = json.loads(STATUS_FILE.read_text(encoding="utf-8")) + except Exception: + return dict(_DEFAULTS) + return { + "last_restart": _safe_int(data.get("last_restart", 0) or 0), + "last_start": _safe_int(data.get("last_start", 0) or 0), + "reason": str(data.get("reason", "") or ""), + "transport": str(data.get("transport", "") or ""), + "proxy": str(data.get("proxy", "") or ""), + "transport_active": str(data.get("transport_active", "") or ""), + "proxy_active": str(data.get("proxy_active", "") or ""), + "installed": bool(data.get("installed", False)), + "configured": bool(data.get("configured", False)), + "running": bool(data.get("running", False)), + "ready": bool(data.get("ready", False)), + "pid": _safe_int(data.get("pid", 0) or 0), + "started_at": _safe_int(data.get("started_at", 0) or 0), + "last_error": str(data.get("last_error", "") or ""), + "privacy_level_effective": str(data.get("privacy_level_effective", "default") or "default"), + } + + +def write_wormhole_status( + *, + reason: str | None = None, + transport: str | None = None, + proxy: str | None = None, + restart: bool = False, + transport_active: str | None = None, + proxy_active: str | None = None, + installed: bool | None = None, + configured: bool | None = None, + running: bool | None = None, + ready: bool | None = None, + pid: int | None = None, + started_at: int | None = None, + last_error: str | None = None, + privacy_level_effective: str | None = None, +) -> dict: + DATA_DIR.mkdir(parents=True, exist_ok=True) + existing = read_wormhole_status() + now = int(time.time()) + payload = { + "last_start": now if not restart and reason else existing.get("last_start", now), + "last_restart": now if restart else existing.get("last_restart", 0), + "reason": existing.get("reason", "") if reason is None else reason, + "transport": existing.get("transport", "") if transport is None else transport, + "proxy": existing.get("proxy", "") if proxy is None else proxy, + "transport_active": existing.get("transport_active", "") if transport_active is None else transport_active, + "proxy_active": existing.get("proxy_active", "") if proxy_active is None else proxy_active, + "installed": existing.get("installed", False) if installed is None else bool(installed), + "configured": existing.get("configured", False) if configured is None else bool(configured), + "running": existing.get("running", False) if running is None else bool(running), + "ready": existing.get("ready", False) if ready is None else bool(ready), + "pid": existing.get("pid", 0) if pid is None else int(pid or 0), + "started_at": existing.get("started_at", 0) if started_at is None else int(started_at or 0), + "last_error": existing.get("last_error", "") if last_error is None else str(last_error), + "privacy_level_effective": ( + existing.get("privacy_level_effective", "default") + if privacy_level_effective is None + else str(privacy_level_effective) + ), + } + STATUS_FILE.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return payload diff --git a/backend/services/wormhole_supervisor.py b/backend/services/wormhole_supervisor.py new file mode 100644 index 00000000..a7d2d27d --- /dev/null +++ b/backend/services/wormhole_supervisor.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +import json +import logging +import os +import signal +import socket +import subprocess +import sys +import threading +import time +from pathlib import Path +from typing import Any +from urllib.error import URLError +from urllib.request import urlopen + +from services.wormhole_settings import read_wormhole_settings +from services.wormhole_status import read_wormhole_status, write_wormhole_status + +logger = logging.getLogger(__name__) + +_LOCK = threading.RLock() +_PROCESS: subprocess.Popen[str] | None = None +_STATE_CACHE: dict[str, Any] | None = None +_STATE_CACHE_TS = 0.0 +_STATE_CACHE_TTL_S = 2.0 +_ARTI_PROOF_CACHE: dict[str, Any] = {"port": 0, "ok": False, "ts": 0.0} +_ARTI_PROOF_CACHE_TTL_S = 30.0 +_PRIVATE_CLEARNET_FALLBACK_WINDOW_S = 300.0 + +BACKEND_DIR = Path(__file__).resolve().parent.parent +DATA_DIR = BACKEND_DIR / "data" +WORMHOLE_SCRIPT = BACKEND_DIR / "wormhole_server.py" +WORMHOLE_STDOUT = DATA_DIR / "wormhole_stdout.log" +WORMHOLE_STDERR = DATA_DIR / "wormhole_stderr.log" +WORMHOLE_HOST = "127.0.0.1" +WORMHOLE_PORT = 8787 +_WORMHOLE_ENV_ALLOWLIST = { + "APPDATA", + "COMSPEC", + "HOME", + "LOCALAPPDATA", + "PATH", + "PATHEXT", + "PROGRAMDATA", + "PYTHONHOME", + "PYTHONIOENCODING", + "PYTHONPATH", + "PYTHONUTF8", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_FILE", + "SYSTEMROOT", + "SystemRoot", + "TEMP", + "TMP", + "USERPROFILE", + "VIRTUAL_ENV", + "WINDIR", +} +_WORMHOLE_ENV_EXPLICIT = { + "ADMIN_KEY", + "ALLOW_INSECURE_ADMIN", + "CORS_ORIGINS", + "PUBLIC_API_KEY", +} + + +def transport_tier_from_state(state: dict[str, Any] | None) -> str: + snapshot = state or {} + if not bool(snapshot.get("configured")): + return "public_degraded" + if not bool(snapshot.get("ready")): + return "public_degraded" + arti = bool(snapshot.get("arti_ready")) + rns = bool(snapshot.get("rns_ready")) + if arti and rns: + return "private_strong" + if arti or rns: + return "private_transitional" + # Once Wormhole is configured and ready, the private lane is online for + # transitional gate/chat use even if the strongest transports are still warming. + return "private_transitional" + + +def _check_arti_ready() -> bool: + from services.config import get_settings + + settings = get_settings() + if not bool(settings.MESH_ARTI_ENABLED): + return False + socks_port = int(settings.MESH_ARTI_SOCKS_PORT or 9050) + try: + with socket.create_connection((WORMHOLE_HOST, socks_port), timeout=2.0) as sock: + # SOCKS5 greeting: version 5, 1 auth method, no-auth. + sock.sendall(b"\x05\x01\x00") + response = sock.recv(2) + if response != b"\x05\x00": + logger.warning("Arti SOCKS5 handshake failed: unexpected response %r", response) + return False + except Exception as exc: + logger.warning("Arti SOCKS check failed on port %s: %s", socks_port, exc) + return False + + now = time.time() + if ( + int(_ARTI_PROOF_CACHE.get("port", 0) or 0) == socks_port + and (now - float(_ARTI_PROOF_CACHE.get("ts", 0.0) or 0.0)) < _ARTI_PROOF_CACHE_TTL_S + ): + return bool(_ARTI_PROOF_CACHE.get("ok")) + + try: + import requests as _requests + + proxy = f"socks5h://127.0.0.1:{socks_port}" + response = _requests.get( + "https://check.torproject.org/api/ip", + proxies={"http": proxy, "https": proxy}, + timeout=3.0, + headers={"Accept": "application/json"}, + ) + payload = response.json() if response.ok else {} + is_tor = bool(payload.get("IsTor")) or bool(payload.get("is_tor")) + if not (response.ok and is_tor): + logger.warning( + "Arti Tor proof failed (status=%s is_tor=%s) — proxy is not trusted as Tor", + getattr(response, "status_code", "unknown"), + payload.get("IsTor", payload.get("is_tor")), + ) + _ARTI_PROOF_CACHE.update({"port": socks_port, "ok": False, "ts": now}) + return False + _ARTI_PROOF_CACHE.update({"port": socks_port, "ok": True, "ts": now}) + return True + except Exception as exc: + logger.warning("Arti Tor proof request failed on port %s: %s", socks_port, exc) + _ARTI_PROOF_CACHE.update({"port": socks_port, "ok": False, "ts": now}) + return False + + +def get_transport_tier() -> str: + return transport_tier_from_state(get_wormhole_state()) + + +def _recent_private_clearnet_fallback_warning(now: float | None = None) -> dict[str, Any]: + current = float(now if now is not None else time.time()) + try: + from services.mesh.mesh_router import mesh_router + except Exception: + return { + "recent_private_clearnet_fallback": False, + "recent_private_clearnet_fallback_at": 0, + "recent_private_clearnet_fallback_reason": "", + } + + message_log = list(getattr(mesh_router, "message_log", ()) or ()) + for entry in reversed(message_log): + routed_via = str(entry.get("routed_via", "") or "").strip().lower() + trust_tier = str(entry.get("trust_tier", "") or "").strip().lower() + ts = float(entry.get("timestamp", 0) or 0.0) + if ts > 0 and (current - ts) > _PRIVATE_CLEARNET_FALLBACK_WINDOW_S: + break + if routed_via != "internet" or not trust_tier.startswith("private_"): + continue + return { + "recent_private_clearnet_fallback": True, + "recent_private_clearnet_fallback_at": int(ts) if ts > 0 else 0, + "recent_private_clearnet_fallback_reason": ( + str(entry.get("route_reason", "") or "").strip() + or "A private-tier payload recently used internet relay instead of a hidden transport." + ), + } + + return { + "recent_private_clearnet_fallback": False, + "recent_private_clearnet_fallback_at": 0, + "recent_private_clearnet_fallback_reason": "", + } + + +def _python_bin() -> str: + venv_python = BACKEND_DIR / "venv" / ("Scripts" if os.name == "nt" else "bin") / ( + "python.exe" if os.name == "nt" else "python3" + ) + if venv_python.exists(): + return str(venv_python) + return sys.executable + + +def _wormhole_subprocess_env( + settings: dict[str, Any], + *, + settings_obj: Any | None = None, +) -> dict[str, str]: + snapshot = settings_obj + if snapshot is None: + from services.config import get_settings + + snapshot = get_settings() + + env: dict[str, str] = {} + for key in _WORMHOLE_ENV_ALLOWLIST: + value = os.environ.get(key) + if value is not None: + env[key] = value + for key, value in os.environ.items(): + if key.startswith("MESH_") or key in _WORMHOLE_ENV_EXPLICIT: + env[key] = value + env.update( + { + "MESH_ONLY": "true", + "MESH_RNS_ENABLED": "true" if bool(getattr(snapshot, "MESH_RNS_ENABLED", False)) else "false", + "WORMHOLE_TRANSPORT": str(settings.get("transport", "direct") or "direct"), + "WORMHOLE_SOCKS_PROXY": str(settings.get("socks_proxy", "") or ""), + "WORMHOLE_SOCKS_DNS": "true" if bool(settings.get("socks_dns", True)) else "false", + "WORMHOLE_HOST": WORMHOLE_HOST, + "WORMHOLE_PORT": str(WORMHOLE_PORT), + } + ) + return env + + +def _installed() -> bool: + return Path(_python_bin()).exists() and WORMHOLE_SCRIPT.exists() + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +def _probe_ready(timeout_s: float = 1.5) -> bool: + try: + with urlopen(f"http://{WORMHOLE_HOST}:{WORMHOLE_PORT}/api/health", timeout=timeout_s) as resp: + return 200 <= getattr(resp, "status", 0) < 300 + except (URLError, OSError, TimeoutError): + return False + + +def _probe_json(path: str, timeout_s: float = 1.5) -> dict[str, Any] | None: + try: + with urlopen(f"http://{WORMHOLE_HOST}:{WORMHOLE_PORT}{path}", timeout=timeout_s) as resp: + if not (200 <= getattr(resp, "status", 0) < 300): + return None + payload = resp.read().decode("utf-8", errors="replace") + data = json.loads(payload or "{}") + return data if isinstance(data, dict) else None + except (URLError, OSError, TimeoutError, json.JSONDecodeError): + return None + + +def _current_runtime_state() -> dict[str, Any]: + settings = read_wormhole_settings() + status = read_wormhole_status() + running = False + pid = int(status.get("pid", 0) or 0) + if _PROCESS and _PROCESS.poll() is None: + running = True + pid = int(_PROCESS.pid or 0) + elif _pid_alive(pid): + running = True + ready = running and _probe_ready() + transport_active = status.get("transport_active", "") if ready else "" + proxy_active = status.get("proxy_active", "") if ready else "" + effective_transport = str(transport_active or settings.get("transport", "direct") or "direct").lower() + from services.config import get_settings + settings_obj = get_settings() + arti_enabled = bool(settings_obj.MESH_ARTI_ENABLED) + arti_ready = _check_arti_ready() + if arti_ready: + try: + from services.mesh.mesh_router import mesh_router + + if mesh_router.tor_arti._consecutive_total_failures >= int( + settings_obj.MESH_RELAY_MAX_FAILURES or 3 + ): + logger.info( + "Arti SOCKS5 is up but transport has %d consecutive failures — marking degraded", + mesh_router.tor_arti._consecutive_total_failures, + ) + arti_ready = False + except Exception: + logger.warning( + "Failed to check tor_arti transport health — fail-closed, marking arti_ready=False" + ) + arti_ready = False + if arti_ready and not transport_active: + transport_active = "tor_arti" + if arti_ready: + effective_transport = "tor_arti" + rns_data = _probe_json("/api/mesh/rns/status", timeout_s=1.0) if ready else None + rns_enabled = bool(rns_data.get("enabled")) if rns_data else False + rns_ready = bool(rns_data.get("ready")) if rns_data else False + rns_configured_peers = int(rns_data.get("configured_peers", 0) or 0) if rns_data else 0 + rns_active_peers = int(rns_data.get("active_peers", 0) or 0) if rns_data else 0 + rns_private_dm_direct_ready = ( + bool(rns_data.get("private_dm_direct_ready")) if rns_data else False + ) + downgrade_warning = _recent_private_clearnet_fallback_warning() + anonymous_mode = bool(settings.get("anonymous_mode")) + anonymous_mode_ready = bool( + anonymous_mode + and settings.get("enabled") + and ready + and effective_transport in {"tor", "tor_arti", "i2p", "mixnet"} + ) + snapshot = { + "installed": _installed(), + "configured": bool(settings.get("enabled")), + "running": running, + "ready": ready, + "transport_configured": str(settings.get("transport", "direct") or "direct"), + "transport_active": transport_active, + "proxy_active": proxy_active, + "last_error": str(status.get("last_error", "") or ""), + "started_at": int(status.get("started_at", status.get("last_start", 0)) or 0), + "pid": pid, + "privacy_level_effective": str(settings.get("privacy_profile", "default") or "default"), + "reason": str(status.get("reason", "") or ""), + "last_restart": int(status.get("last_restart", 0) or 0), + "last_start": int(status.get("last_start", 0) or 0), + "transport": str(settings.get("transport", "direct") or "direct"), + "proxy": str(settings.get("socks_proxy", "") or ""), + "anonymous_mode": anonymous_mode, + "anonymous_mode_ready": anonymous_mode_ready, + "arti_ready": arti_ready, + "arti_enabled": arti_enabled, + "rns_enabled": rns_enabled, + "rns_ready": rns_ready, + "rns_configured_peers": rns_configured_peers, + "rns_active_peers": rns_active_peers, + "rns_private_dm_direct_ready": rns_private_dm_direct_ready, + **downgrade_warning, + } + snapshot["transport_tier"] = transport_tier_from_state(snapshot) + write_wormhole_status( + installed=snapshot["installed"], + configured=snapshot["configured"], + running=snapshot["running"], + ready=snapshot["ready"], + pid=snapshot["pid"], + started_at=snapshot["started_at"], + last_error=snapshot["last_error"], + privacy_level_effective=snapshot["privacy_level_effective"], + transport=snapshot["transport"], + proxy=snapshot["proxy"], + transport_active=snapshot["transport_active"], + proxy_active=snapshot["proxy_active"], + ) + return snapshot + + +def _invalidate_state_cache() -> None: + global _STATE_CACHE, _STATE_CACHE_TS + _STATE_CACHE = None + _STATE_CACHE_TS = 0.0 + + +def _store_state_cache(snapshot: dict[str, Any]) -> dict[str, Any]: + global _STATE_CACHE, _STATE_CACHE_TS + _STATE_CACHE = dict(snapshot) + _STATE_CACHE_TS = time.monotonic() + return snapshot + + +def get_wormhole_state() -> dict[str, Any]: + global _STATE_CACHE, _STATE_CACHE_TS + with _LOCK: + now = time.monotonic() + if _STATE_CACHE is not None and (now - _STATE_CACHE_TS) < _STATE_CACHE_TTL_S: + return dict(_STATE_CACHE) + snapshot = _current_runtime_state() + return _store_state_cache(snapshot) + + +def connect_wormhole(*, reason: str = "connect") -> dict[str, Any]: + with _LOCK: + _invalidate_state_cache() + settings = read_wormhole_settings() + if not settings.get("enabled"): + settings = settings.copy() + settings["enabled"] = True + current = _current_runtime_state() + if current["ready"]: + return current + if not current["installed"]: + write_wormhole_status( + reason=reason, + installed=False, + configured=True, + running=False, + ready=False, + last_error="Wormhole runtime is not installed.", + privacy_level_effective=str(settings.get("privacy_profile", "default")), + transport=str(settings.get("transport", "direct")), + proxy=str(settings.get("socks_proxy", "")), + ) + return _current_runtime_state() + + DATA_DIR.mkdir(parents=True, exist_ok=True) + stdout = open(WORMHOLE_STDOUT, "a", encoding="utf-8") + stderr = open(WORMHOLE_STDERR, "a", encoding="utf-8") + from services.config import get_settings + + env = _wormhole_subprocess_env(settings, settings_obj=get_settings()) + kwargs: dict[str, Any] = { + "cwd": str(BACKEND_DIR), + "env": env, + "stdout": stdout, + "stderr": stderr, + "text": True, + } + if os.name == "nt": + kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined] + + process = subprocess.Popen([_python_bin(), str(WORMHOLE_SCRIPT)], **kwargs) + global _PROCESS + _PROCESS = process + started_at = int(time.time()) + write_wormhole_status( + reason=reason, + restart=False, + installed=True, + configured=True, + running=True, + ready=False, + pid=int(process.pid or 0), + started_at=started_at, + last_error="", + privacy_level_effective=str(settings.get("privacy_profile", "default")), + transport=str(settings.get("transport", "direct")), + proxy=str(settings.get("socks_proxy", "")), + ) + + deadline = time.monotonic() + 8.0 + while time.monotonic() < deadline: + if process.poll() is not None: + err = f"Wormhole exited with code {process.returncode}." + write_wormhole_status( + reason="crash", + installed=True, + configured=True, + running=False, + ready=False, + pid=0, + last_error=err, + ) + return _store_state_cache(_current_runtime_state()) + if _probe_ready(timeout_s=0.75): + write_wormhole_status( + reason=reason, + installed=True, + configured=True, + running=True, + ready=True, + pid=int(process.pid or 0), + started_at=started_at, + last_error="", + privacy_level_effective=str(settings.get("privacy_profile", "default")), + transport=str(settings.get("transport", "direct")), + proxy=str(settings.get("socks_proxy", "")), + ) + break + time.sleep(0.25) + return _store_state_cache(_current_runtime_state()) + + +def disconnect_wormhole(*, reason: str = "disconnect") -> dict[str, Any]: + with _LOCK: + _invalidate_state_cache() + current = _current_runtime_state() + pid = int(current.get("pid", 0) or 0) + global _PROCESS + if _PROCESS and _PROCESS.poll() is None: + try: + _PROCESS.terminate() + _PROCESS.wait(timeout=5) + except Exception: + try: + _PROCESS.kill() + except Exception: + pass + elif _pid_alive(pid): + try: + os.kill(pid, signal.SIGTERM) + except Exception: + pass + _PROCESS = None + write_wormhole_status( + reason=reason, + running=False, + ready=False, + pid=0, + transport_active="", + proxy_active="", + last_error="", + ) + return _store_state_cache(_current_runtime_state()) + + +def restart_wormhole(*, reason: str = "restart") -> dict[str, Any]: + with _LOCK: + _invalidate_state_cache() + disconnect_wormhole(reason=f"{reason}_stop") + write_wormhole_status(reason=reason, restart=True) + return connect_wormhole(reason=reason) + + +def sync_wormhole_with_settings() -> dict[str, Any]: + settings = read_wormhole_settings() + if settings.get("enabled"): + return connect_wormhole(reason="sync") + return disconnect_wormhole(reason="sync_disabled") + + +def shutdown_wormhole_supervisor() -> None: + disconnect_wormhole(reason="backend_shutdown") diff --git a/backend/sgp_sample.json b/backend/sgp_sample.json deleted file mode 100644 index db4f4503..00000000 --- a/backend/sgp_sample.json +++ /dev/null @@ -1 +0,0 @@ -{"items":[{"timestamp":"2026-02-25T14:56:59+08:00","cameras":[{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/647cd9f0-6225-4951-a113-ebf776ded56f.jpg","location":{"latitude":1.29531332,"longitude":103.871146},"camera_id":"1001","image_metadata":{"height":240,"width":320,"md5":"0bb0e277037c088192b12052c4315e88"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/4c52361e-b566-452e-bb44-0616dc56c6fe.jpg","location":{"latitude":1.319541067,"longitude":103.8785627},"camera_id":"1002","image_metadata":{"height":240,"width":320,"md5":"5004ac49a538bf1d6c9a8757991a1be9"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/ba7c78c6-863a-4221-9854-4454efb27b1d.jpg","location":{"latitude":1.323957439,"longitude":103.8728576},"camera_id":"1003","image_metadata":{"height":240,"width":320,"md5":"09fe251afe3225d44305bdeb62d95b06"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/03775a4b-40c2-4bc9-abf2-3d881dffd632.jpg","location":{"latitude":1.319535712,"longitude":103.8750668},"camera_id":"1004","image_metadata":{"height":240,"width":320,"md5":"cb971915dffda15a2be7ef31db968805"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/9cbdd3f7-ebfe-4561-99a6-1bbbde75ee90.jpg","location":{"latitude":1.363519886,"longitude":103.905394},"camera_id":"1005","image_metadata":{"height":240,"width":320,"md5":"c80e8da0b53d7eccdb8243724b7ce2fe"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/28f2c6da-4fdd-49b5-b347-51b5bbbf4c37.jpg","location":{"latitude":1.357098686,"longitude":103.902042},"camera_id":"1006","image_metadata":{"height":240,"width":320,"md5":"349af068659b597c7912c0238bcdfe02"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/2476e3ad-cb23-4096-83b4-c834a25cf1ff.jpg","location":{"latitude":1.365434,"longitude":103.953997},"camera_id":"1111","image_metadata":{"height":1080,"width":1920,"md5":"01cb8f8a5169a6edb4ceafdfa64bd51d"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/2a4789b6-249b-4729-93cd-1b12f09eeed6.jpg","location":{"latitude":1.3605,"longitude":103.961412},"camera_id":"1112","image_metadata":{"height":1080,"width":1920,"md5":"88c037cbf99ddba20d04cbaa89f7ce79"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/1f935572-1e10-4d07-9595-c20439329a89.jpg","location":{"latitude":1.317036,"longitude":103.988598},"camera_id":"1113","image_metadata":{"height":1080,"width":1920,"md5":"37c6ac4e07c15179214a2cee305cf3f5"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/9c59d473-4cd8-4118-9cf8-a6fe6deee06c.jpg","location":{"latitude":1.27414394350065,"longitude":103.851316802547},"camera_id":"1501","image_metadata":{"height":240,"width":320,"md5":"9041591fdd0c21b4ff693db7cbc1bbfe"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/619e28d1-0865-4901-8880-62616285e80c.jpg","location":{"latitude":1.27135090682664,"longitude":103.861828440597},"camera_id":"1502","image_metadata":{"height":240,"width":320,"md5":"381291831e52a27d41942efc065c3a6c"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/95004282-fbfa-4edf-868e-eb9787bbfd6e.jpg","location":{"latitude":1.27066408655104,"longitude":103.856977943394},"camera_id":"1503","image_metadata":{"height":240,"width":320,"md5":"f7250fa571325b0b0dc12722f5a4c220"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/7e50b07d-feae-4b4a-aed7-09dfa6d7666c.jpg","location":{"latitude":1.29409891409364,"longitude":103.876056196568},"camera_id":"1504","image_metadata":{"height":240,"width":320,"md5":"ee809c0ead15971f3d24bfd36524e370"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/d3dab4c8-d495-4ebe-a050-3d296c8410e0.jpg","location":{"latitude":1.2752977149006,"longitude":103.866390381759},"camera_id":"1505","image_metadata":{"height":240,"width":320,"md5":"d569782d52227e5ef18f1ef403de7af0"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/b2a15cb8-bbfb-4121-91b1-dc1c7bd9956c.jpg","location":{"latitude":1.323604823,"longitude":103.8587802},"camera_id":"1701","image_metadata":{"height":1080,"width":1920,"md5":"818c60237225cfaf3919a7435e6be651"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/58e79987-c290-4959-b081-f8b554fca1d6.jpg","location":{"latitude":1.34355015,"longitude":103.8601984},"camera_id":"1702","image_metadata":{"height":1080,"width":1920,"md5":"d260eac8cfafcdaf062a8d62aca80125"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/b4ebc129-50a6-483a-bc5b-38254d66daf2.jpg","location":{"latitude":1.32814722194857,"longitude":103.862203282048},"camera_id":"1703","image_metadata":{"height":1080,"width":1920,"md5":"627d1134618e2a1149aba46cf0411909"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/515ba1b7-e0c8-46e8-b42d-5a76eed0accc.jpg","location":{"latitude":1.28569398886979,"longitude":103.837524510188},"camera_id":"1704","image_metadata":{"height":1080,"width":1920,"md5":"be34bb6954696df12b7f18594db248ea"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/d8f14621-cd6b-4e89-b2f8-08f81da9acac.jpg","location":{"latitude":1.375925022,"longitude":103.8587986},"camera_id":"1705","image_metadata":{"height":1080,"width":1920,"md5":"7252aab345e18cf58f19b37f86c37edb"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/1a93b935-f934-4e9f-845f-e56ce6b688b4.jpg","location":{"latitude":1.38861,"longitude":103.85806},"camera_id":"1706","image_metadata":{"height":1080,"width":1920,"md5":"510a3259ae7c13f5154b79ef0f0e7755"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/0992a5e5-3eb9-4586-995e-49b247efa888.jpg","location":{"latitude":1.28036584335876,"longitude":103.830451146503},"camera_id":"1707","image_metadata":{"height":1080,"width":1920,"md5":"f97c0671d400855e7e75ca424e72593f"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/f568e659-3731-42c8-9087-42442bf64b1a.jpg","location":{"latitude":1.31384231654635,"longitude":103.845603032574},"camera_id":"1709","image_metadata":{"height":1080,"width":1920,"md5":"ad0198cb782e68aa5543e05fa7b46923"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/0eab59e7-e166-4612-ae7c-9d3abd413013.jpg","location":{"latitude":1.35296,"longitude":103.85719},"camera_id":"1711","image_metadata":{"height":1080,"width":1920,"md5":"9b1c0d1b6173671d6a1e73f3bd80a9c8"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/ac16f6cd-273b-45b2-9cb7-ada035229a3e.jpg","location":{"latitude":1.447023728,"longitude":103.7716543},"camera_id":"2701","image_metadata":{"height":1080,"width":1920,"md5":"844f6be4187f114610a208ae5fc92d04"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/5bd9f207-ef59-4a9b-bcc2-dfebc4474645.jpg","location":{"latitude":1.445554109,"longitude":103.7683397},"camera_id":"2702","image_metadata":{"height":1080,"width":1920,"md5":"e9745302b85b5d184f231241bb1ff34d"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/ecf26425-d457-4f8e-8694-2810e90cc4b7.jpg","location":{"latitude":1.35047790791386,"longitude":103.791033581325},"camera_id":"2703","image_metadata":{"height":1080,"width":1920,"md5":"b38ffe1b0dd4beb6d85d7f0ade4286af"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/6c964abf-e339-4e81-9bf1-ce556e2434fe.jpg","location":{"latitude":1.429588536,"longitude":103.769311},"camera_id":"2704","image_metadata":{"height":1080,"width":1920,"md5":"3e87aa9b0f11c7f122a03d9cdeaa02de"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/b9e4bbc8-3936-4b03-aab5-8bbc563b13db.jpg","location":{"latitude":1.36728572,"longitude":103.7794698},"camera_id":"2705","image_metadata":{"height":1080,"width":1920,"md5":"e66963b7a32e8a2b63fa868a823e7f59"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/cd5e88bd-adfe-46b9-a574-eb4d00cb9210.jpg","location":{"latitude":1.414142,"longitude":103.771168},"camera_id":"2706","image_metadata":{"height":1080,"width":1920,"md5":"a31c1d76bfd791720c132e2239538362"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/4e0e04e0-673e-4d99-9637-92ef9bad8e57.jpg","location":{"latitude":1.3983,"longitude":103.774247},"camera_id":"2707","image_metadata":{"height":1080,"width":1920,"md5":"288b54d64509e7491f3beb4cbb0431cd"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/36775a6b-590d-4e28-b52d-f9ee79c4d2e5.jpg","location":{"latitude":1.3865,"longitude":103.7747},"camera_id":"2708","image_metadata":{"height":1080,"width":1920,"md5":"cb40a67b643a8f3d268ed77eec6a5060"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/cb137d36-6e9f-4fff-ae34-6cb37b0aea59.jpg","location":{"latitude":1.33831,"longitude":103.98032},"camera_id":"3702","image_metadata":{"height":1080,"width":1920,"md5":"caa0c9b82b8e4e4df75918b1bdad940a"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/947b8856-0ca4-4beb-80b8-2bbfe2fe1b5a.jpg","location":{"latitude":1.2958550156561,"longitude":103.880314665981},"camera_id":"3704","image_metadata":{"height":1080,"width":1920,"md5":"76382811218897f4557f3837477ec3c9"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/48b982b6-41ca-4f4d-9e79-5e75a3fc7769.jpg","location":{"latitude":1.32743,"longitude":103.97383},"camera_id":"3705","image_metadata":{"height":1080,"width":1920,"md5":"279956ba61873e5f7d8aa0249a1f2dcb"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/c2aefc7c-cd0c-424a-8164-95caa9516729.jpg","location":{"latitude":1.309330837,"longitude":103.9350504},"camera_id":"3793","image_metadata":{"height":1080,"width":1920,"md5":"9b66027e749bb359e63665e6f2647d81"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/e1caac6e-6eaf-4349-9fe3-8f4dc8de22b8.jpg","location":{"latitude":1.30145145166066,"longitude":103.910596320237},"camera_id":"3795","image_metadata":{"height":1080,"width":1920,"md5":"e7b9292c1bb0e5c3e935433180ddcce7"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/25d1b493-ec1f-4c19-bdb3-74635ebcf83b.jpg","location":{"latitude":1.297512569,"longitude":103.8983019},"camera_id":"3796","image_metadata":{"height":1080,"width":1920,"md5":"f6564076c9d04207c08009fbe10e6b97"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/09f1407a-143e-4ae6-b5aa-f6bf22249d1d.jpg","location":{"latitude":1.29565733262976,"longitude":103.885283049309},"camera_id":"3797","image_metadata":{"height":1080,"width":1920,"md5":"3fe876b2fbfb7836b3cfa4eb95d42591"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/2a2ae82e-6557-4425-af29-07f604b7b31d.jpg","location":{"latitude":1.29158484,"longitude":103.8615987},"camera_id":"3798","image_metadata":{"height":1080,"width":1920,"md5":"f52f8a9d7d3031524f69bfeb44ba8914"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/8377a1e0-514e-4aab-8e9e-8971020e0eef.jpg","location":{"latitude":1.2871,"longitude":103.79633},"camera_id":"4701","image_metadata":{"height":1080,"width":1920,"md5":"6907b046098e3022035317a05e5b6bf4"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/284dc339-e9f2-4678-839a-ac53dfc111b8.jpg","location":{"latitude":1.27237,"longitude":103.8324},"camera_id":"4702","image_metadata":{"height":1080,"width":1920,"md5":"92dedce213fd39e8abe59186026f10be"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/ccb924b0-0ba6-4e78-84cf-dee23023c416.jpg","location":{"latitude":1.348697862,"longitude":103.6350413},"camera_id":"4703","image_metadata":{"height":1080,"width":1920,"md5":"a5fd6831d87a8b7f263dfc113ee4aefc"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/5cbb3f06-ea5b-4e05-bd2d-5fce67f33dfa.jpg","location":{"latitude":1.27877,"longitude":103.82375},"camera_id":"4704","image_metadata":{"height":1080,"width":1920,"md5":"23ee7067534bfe67d37d968dd4e39808"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/1c483d65-05a6-4485-814a-3d3dbaa3f32e.jpg","location":{"latitude":1.32618,"longitude":103.73028},"camera_id":"4705","image_metadata":{"height":1080,"width":1920,"md5":"73519f44d4a91c246f23799c908a4ae9"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/af84eeab-011f-4322-a6b0-7a5aa073dfbc.jpg","location":{"latitude":1.29792,"longitude":103.78205},"camera_id":"4706","image_metadata":{"height":1080,"width":1920,"md5":"dc77f5f81a9c8815a3bed53a97a9154c"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/995bfc20-e70d-40f8-ab8f-5cea70b7e64e.jpg","location":{"latitude":1.33344648135658,"longitude":103.652700847056},"camera_id":"4707","image_metadata":{"height":1080,"width":1920,"md5":"5379b2557185eaa887d02739f3da3ed3"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/71535a87-3551-4d9e-94a0-03ccf2329d6a.jpg","location":{"latitude":1.29939,"longitude":103.7799},"camera_id":"4708","image_metadata":{"height":1080,"width":1920,"md5":"73367aab6088c92d07a2d8ba6ca87f61"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/6b0584f4-2aaa-4028-bf50-15a4d2550fd9.jpg","location":{"latitude":1.312019,"longitude":103.763002},"camera_id":"4709","image_metadata":{"height":1080,"width":1920,"md5":"aa9829150005d10007542ef630cf3435"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/266b3031-acb9-46ad-978b-f288eb319bec.jpg","location":{"latitude":1.32153,"longitude":103.75273},"camera_id":"4710","image_metadata":{"height":1080,"width":1920,"md5":"a3fb153057f06b811dbf7af7734ef7ba"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/eda070d2-887a-46f6-a2da-1ede4d1a97a2.jpg","location":{"latitude":1.341244001,"longitude":103.6439134},"camera_id":"4712","image_metadata":{"height":1080,"width":1920,"md5":"34f4db05183f55a60123c82bb1466da6"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/d63cf85f-2096-40ca-a785-f03a14d69dad.jpg","location":{"latitude":1.347645829,"longitude":103.6366955},"camera_id":"4713","image_metadata":{"height":1080,"width":1920,"md5":"427f9900709e5e7933a073b61903c302"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/831235d9-1cf2-43ed-afaa-b4e50cb4e499.jpg","location":{"latitude":1.31023,"longitude":103.76438},"camera_id":"4714","image_metadata":{"height":1080,"width":1920,"md5":"ca13af9e25ee13a0a7087c4a8f1120a0"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/080b1430-16dd-4340-92cf-263238d1c2ba.jpg","location":{"latitude":1.32227,"longitude":103.67453},"camera_id":"4716","image_metadata":{"height":360,"width":640,"md5":"3137589a4c359004b0ec1bcc58d4cf40"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/eaaa2288-54c0-4ff0-9699-e2f59c50b2e3.jpg","location":{"latitude":1.25999999687243,"longitude":103.823611110166},"camera_id":"4798","image_metadata":{"height":1080,"width":1920,"md5":"1ae3504c2ba758457b84cfb6ee15f685"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/41497c96-50e2-45bc-a7a5-a7d77f0d3d2b.jpg","location":{"latitude":1.26027777363278,"longitude":103.823888890049},"camera_id":"4799","image_metadata":{"height":1080,"width":1920,"md5":"1ddafc844dc3a32bc1d654ee75e98428"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/53f5aa17-ca67-4f09-9d2f-c72680562be5.jpg","location":{"latitude":1.3309693,"longitude":103.9168616},"camera_id":"5794","image_metadata":{"height":1080,"width":1920,"md5":"b38d98a5023b51e2da30889197c8ff2b"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/5d2bd1f9-b3a2-4504-9f19-7ea9e22d5fe3.jpg","location":{"latitude":1.326024822,"longitude":103.905625},"camera_id":"5795","image_metadata":{"height":1080,"width":1920,"md5":"fc321223489b43e54a7d699391b09de0"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/75b521b6-ccb1-45c7-a01b-a434dd53b35d.jpg","location":{"latitude":1.322875288,"longitude":103.8910793},"camera_id":"5797","image_metadata":{"height":1080,"width":1920,"md5":"67202c4cfe5dde7abd48bea7a1508dff"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/6e87cf0e-de0b-4248-91cc-c95bb9e4d81a.jpg","location":{"latitude":1.32036078126842,"longitude":103.877174116489},"camera_id":"5798","image_metadata":{"height":1080,"width":1920,"md5":"193ec601ec4e5edf6c51a074cd5d2284"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/c471ab8a-9764-4649-8aa4-31878876fba3.jpg","location":{"latitude":1.328171608,"longitude":103.8685191},"camera_id":"5799","image_metadata":{"height":1080,"width":1920,"md5":"ee3f15e4e6ed4ab226fc77ade76f751a"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/9be46717-eeb1-4f1e-b244-2164f65d09ec.jpg","location":{"latitude":1.329334,"longitude":103.858222},"camera_id":"6701","image_metadata":{"height":1080,"width":1920,"md5":"b344eff781f1a39a77bd90cf436dcaef"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/267c51d4-a9d8-43fe-ba36-af5f9dca881e.jpg","location":{"latitude":1.328899,"longitude":103.84121},"camera_id":"6703","image_metadata":{"height":1080,"width":1920,"md5":"d6ac3d3d92ae04bbe0b2efb32e8b93cc"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/574e65ee-0c6f-4659-a785-b7f49acee3f7.jpg","location":{"latitude":1.32657403632366,"longitude":103.826857295633},"camera_id":"6704","image_metadata":{"height":1080,"width":1920,"md5":"125f231f5023a0079c7e072a0c9de271"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/fbf07fed-d8c5-4c68-a1f1-f5b8a71bb5eb.jpg","location":{"latitude":1.332124,"longitude":103.81768},"camera_id":"6705","image_metadata":{"height":1080,"width":1920,"md5":"bc346990bbebadddd5ea8ee1df2cd9cf"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/a875ab4c-9120-4dfd-8edf-e993acf20cde.jpg","location":{"latitude":1.349428893,"longitude":103.7952799},"camera_id":"6706","image_metadata":{"height":1080,"width":1920,"md5":"1ae0f59bb6f634b8ef1720b6f8c9c505"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/fcc5cec3-5569-4391-8805-970df18a7cda.jpg","location":{"latitude":1.345996,"longitude":103.69016},"camera_id":"6708","image_metadata":{"height":1080,"width":1920,"md5":"5ed5e7e59f6e414b1d226128b704176a"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/5500ce19-a109-4197-afab-72170013e7fe.jpg","location":{"latitude":1.344205,"longitude":103.78577},"camera_id":"6710","image_metadata":{"height":1080,"width":1920,"md5":"abac167261f10c1f18efc0cc6407e5ff"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/3f506c21-24f7-4d80-a6f4-75f7255053a3.jpg","location":{"latitude":1.33771,"longitude":103.977827},"camera_id":"6711","image_metadata":{"height":1080,"width":1920,"md5":"a576d074834ee46273d93dea168d98e7"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/bdedc16b-78da-4f6d-95d3-01f7fcea5d1d.jpg","location":{"latitude":1.332691,"longitude":103.770278},"camera_id":"6712","image_metadata":{"height":1080,"width":1920,"md5":"a0ae636d7128bd358b7574eb4870b3f7"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/6c70ce62-7293-4866-87e5-f681e491142c.jpg","location":{"latitude":1.340298,"longitude":103.945652},"camera_id":"6713","image_metadata":{"height":1080,"width":1920,"md5":"736e3ac99b87a4193fda68fd34bbff0e"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/a0db9d37-f7f3-49c2-9e8c-4d0dd3dfc9e7.jpg","location":{"latitude":1.361742,"longitude":103.703341},"camera_id":"6714","image_metadata":{"height":1080,"width":1920,"md5":"30f1181d4c1627f9e69797c16d45f65b"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/99cdf59e-9e22-42f5-8ae8-0d546e8f9444.jpg","location":{"latitude":1.356299,"longitude":103.716071},"camera_id":"6715","image_metadata":{"height":1080,"width":1920,"md5":"4027a8b86fa3ba69e013056ab6cd87b0"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/4b70a44f-4675-4822-95c3-f506435727b2.jpg","location":{"latitude":1.322893,"longitude":103.6635051},"camera_id":"6716","image_metadata":{"height":1080,"width":1920,"md5":"065224a7abd70b0744ea086544a7b4ac"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/1a7954ae-dc49-4b33-a153-d985f8453ef9.jpg","location":{"latitude":1.354245,"longitude":103.963782},"camera_id":"7791","image_metadata":{"height":1080,"width":1920,"md5":"893e17405647c2026ec324d4fce7cf55"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/a80d7048-105b-457c-ad3f-daff267d2e1a.jpg","location":{"latitude":1.37704704,"longitude":103.92946983},"camera_id":"7793","image_metadata":{"height":1080,"width":1920,"md5":"3b3d610dd099139e3f958542f3541ae9"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/65f94ed8-0493-4492-81ab-6c7a4532a099.jpg","location":{"latitude":1.37988658,"longitude":103.92009174},"camera_id":"7794","image_metadata":{"height":1080,"width":1920,"md5":"6ff1ed46ce63cf8eed9a6526af460745"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/39732d43-f46c-48f8-bbd6-58562504f041.jpg","location":{"latitude":1.38432741,"longitude":103.91585701},"camera_id":"7795","image_metadata":{"height":1080,"width":1920,"md5":"df6eb0640094d7f5a3487e69ce836397"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/e9907ef9-890a-46b5-b196-4d4b895e778b.jpg","location":{"latitude":1.39559294,"longitude":103.90515712},"camera_id":"7796","image_metadata":{"height":1080,"width":1920,"md5":"ba0d66e62ef0784043d127c228cfaa33"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/551adb61-8906-44b0-8162-af38bbe3bcfe.jpg","location":{"latitude":1.40002575,"longitude":103.85702534},"camera_id":"7797","image_metadata":{"height":1080,"width":1920,"md5":"d7cd87f1173ea7b0d83b9568ef7a55e5"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/e5352a79-0016-4bf4-9c39-2443d4c60491.jpg","location":{"latitude":1.39748842,"longitude":103.85400467},"camera_id":"7798","image_metadata":{"height":1080,"width":1920,"md5":"22e2a3db7b6f8492b282f1528ef409b0"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/946ac562-20c9-4152-aaef-dee6f99008d8.jpg","location":{"latitude":1.38647,"longitude":103.74143},"camera_id":"8701","image_metadata":{"height":1080,"width":1920,"md5":"7b4c0c3e572104e95235e81326e321a2"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/4adaafe3-41fb-488b-a9f1-24920d04fd65.jpg","location":{"latitude":1.39059,"longitude":103.7717},"camera_id":"8702","image_metadata":{"height":1080,"width":1920,"md5":"f0fa28dd36df9e5010522cdcd4720dee"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/db393f3a-43d0-4eb7-8a00-1e2c1acbfbb3.jpg","location":{"latitude":1.3899,"longitude":103.74843},"camera_id":"8704","image_metadata":{"height":1080,"width":1920,"md5":"1b00a39cfd928149b9ba766281a75988"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/09863c3c-3ee3-4ef3-bcd8-c3199a2541bd.jpg","location":{"latitude":1.3664,"longitude":103.70899},"camera_id":"8706","image_metadata":{"height":1080,"width":1920,"md5":"fb573b45701aba2e25b5402e1db533f9"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/7d0b47b9-e2c2-4090-8b40-611f6cb7489c.jpg","location":{"latitude":1.39466333,"longitude":103.83474601},"camera_id":"9701","image_metadata":{"height":1080,"width":1920,"md5":"8155d0d6f8b974d68fc78fee31ea5a3e"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/bdfb291a-86bf-4544-9822-cbefbc20200d.jpg","location":{"latitude":1.39474081,"longitude":103.81797086},"camera_id":"9702","image_metadata":{"height":1080,"width":1920,"md5":"683c6c8e4ccff0fc3c9d0cfacc705f1a"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/96a7b783-32b4-4c86-ab84-3dc1b4034d25.jpg","location":{"latitude":1.422857,"longitude":103.773005},"camera_id":"9703","image_metadata":{"height":1080,"width":1920,"md5":"6bf3308ae97890202774c43fe7814093"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/c3d9eee5-73af-47e5-a6ff-5b844a5a5390.jpg","location":{"latitude":1.42214311,"longitude":103.79542062},"camera_id":"9704","image_metadata":{"height":1080,"width":1920,"md5":"30c8566b64c45ce7e3d76fb29384b85d"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/62263608-5b4f-4163-9dae-9f82f80f58d3.jpg","location":{"latitude":1.42627712,"longitude":103.78716637},"camera_id":"9705","image_metadata":{"height":1080,"width":1920,"md5":"52702cc3c8498b34326de7810708ab50"}},{"timestamp":"2026-02-25T14:55:59+08:00","image":"https://images.data.gov.sg/api/traffic-images/2026/02/50c2197b-4f31-4149-a1cf-2c4ff81140dd.jpg","location":{"latitude":1.41270056,"longitude":103.80642712},"camera_id":"9706","image_metadata":{"height":1080,"width":1920,"md5":"47077b766e1eeb0e86028da96e6c47b8"}}]}],"api_info":{"status":"healthy"}} \ No newline at end of file diff --git a/backend/temp.json b/backend/temp.json deleted file mode 100644 index a6aa56d0..00000000 --- a/backend/temp.json +++ /dev/null @@ -1,10 +0,0 @@ - - - - -Error - - -
Cannot GET /api/history/public/latest
- - diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 909bd766..39ca9a35 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -5,12 +5,14 @@ @pytest.fixture(autouse=True) def _suppress_background_services(): """Prevent real scheduler/stream/tracker from starting during tests.""" - with patch("services.data_fetcher.start_scheduler"), \ - patch("services.data_fetcher.stop_scheduler"), \ - patch("services.ais_stream.start_ais_stream"), \ - patch("services.ais_stream.stop_ais_stream"), \ - patch("services.carrier_tracker.start_carrier_tracker"), \ - patch("services.carrier_tracker.stop_carrier_tracker"): + with ( + patch("services.data_fetcher.start_scheduler"), + patch("services.data_fetcher.stop_scheduler"), + patch("services.ais_stream.start_ais_stream"), + patch("services.ais_stream.stop_ais_stream"), + patch("services.carrier_tracker.start_carrier_tracker"), + patch("services.carrier_tracker.stop_carrier_tracker"), + ): yield diff --git a/backend/tests/fixtures/airports.json b/backend/tests/fixtures/airports.json new file mode 100644 index 00000000..4fe0ca4d --- /dev/null +++ b/backend/tests/fixtures/airports.json @@ -0,0 +1,4 @@ +[ + {"id": "KDEN", "name": "Denver Intl", "iata": "DEN", "lat": 39.8561, "lng": -104.6737, "type": "airport"}, + {"id": "KJFK", "name": "John F Kennedy Intl", "iata": "JFK", "lat": 40.6413, "lng": -73.7781, "type": "airport"} +] diff --git a/backend/tests/mesh/__init__.py b/backend/tests/mesh/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/mesh/test_mesh_anonymous_mode.py b/backend/tests/mesh/test_mesh_anonymous_mode.py new file mode 100644 index 00000000..4fa09e5d --- /dev/null +++ b/backend/tests/mesh/test_mesh_anonymous_mode.py @@ -0,0 +1,437 @@ +import asyncio +import json + +from starlette.requests import Request +from starlette.responses import Response + + +def _request(path: str, method: str = "POST") -> Request: + return Request( + { + "type": "http", + "headers": [], + "client": ("test", 12345), + "method": method, + "path": path, + } + ) + + +def test_anonymous_mode_blocks_public_mesh_write_without_hidden_transport(monkeypatch): + import main + from services import wormhole_settings, wormhole_status + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "direct", + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + response = asyncio.run(main.enforce_high_privacy_mesh(_request("/api/mesh/send"), call_next)) + payload = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 428 + assert "hidden Wormhole transport" in payload["detail"] + + +def test_anonymous_mode_allows_public_mesh_write_when_hidden_transport_ready(monkeypatch): + import main + from services import wormhole_settings, wormhole_status + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "tor", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "tor", + }, + ) + called = {"value": False} + + async def call_next(_request: Request) -> Response: + called["value"] = True + return Response(status_code=200) + + response = asyncio.run(main.enforce_high_privacy_mesh(_request("/api/mesh/send"), call_next)) + + assert response.status_code == 200 + assert called["value"] is True + + +def test_anonymous_mode_treats_tor_arti_as_hidden_transport(monkeypatch): + import main + from services import wormhole_settings, wormhole_status + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "tor_arti", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "tor_arti", + }, + ) + called = {"value": False} + + async def call_next(_request: Request) -> Response: + called["value"] = True + return Response(status_code=200) + + response = asyncio.run(main.enforce_high_privacy_mesh(_request("/api/mesh/send"), call_next)) + + assert response.status_code == 200 + assert called["value"] is True + + +def test_anonymous_mode_does_not_block_read_only_mesh_requests(monkeypatch): + import main + from services import wormhole_settings, wormhole_status + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": False, + "ready": False, + "transport_active": "direct", + }, + ) + called = {"value": False} + + async def call_next(_request: Request) -> Response: + called["value"] = True + return Response(status_code=200) + + response = asyncio.run( + main.enforce_high_privacy_mesh(_request("/api/mesh/status", method="GET"), call_next) + ) + + assert response.status_code == 200 + assert called["value"] is True + + +def test_anonymous_mode_blocks_private_dm_actions_without_hidden_transport(monkeypatch): + import main + from services import wormhole_settings, wormhole_status, wormhole_supervisor + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "direct", + }, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": True, + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + response = asyncio.run(main.enforce_high_privacy_mesh(_request("/api/mesh/dm/send"), call_next)) + payload = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 428 + assert "private DM activity" in payload["detail"] + + +def test_anonymous_mode_allows_private_dm_actions_when_hidden_transport_ready(monkeypatch): + import main + from services import wormhole_settings, wormhole_status, wormhole_supervisor + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "tor", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "tor", + }, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": True, + }, + ) + called = {"value": False} + + async def call_next(_request: Request) -> Response: + called["value"] = True + return Response(status_code=200) + + response = asyncio.run(main.enforce_high_privacy_mesh(_request("/api/mesh/dm/poll"), call_next)) + + assert response.status_code == 200 + assert called["value"] is True + + +def test_anonymous_mode_blocks_dm_witness_and_block_without_hidden_transport(monkeypatch): + import main + from services import wormhole_settings, wormhole_status, wormhole_supervisor + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "direct", + }, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": True, + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + block_response = asyncio.run( + main.enforce_high_privacy_mesh(_request("/api/mesh/dm/block"), call_next) + ) + witness_response = asyncio.run( + main.enforce_high_privacy_mesh(_request("/api/mesh/dm/witness"), call_next) + ) + + assert block_response.status_code == 428 + assert witness_response.status_code == 428 + + +def test_anonymous_mode_blocks_public_vouch_without_hidden_transport(monkeypatch): + import main + from services import wormhole_settings, wormhole_status, wormhole_supervisor + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "direct", + }, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": False, + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + response = asyncio.run( + main.enforce_high_privacy_mesh(_request("/api/mesh/trust/vouch"), call_next) + ) + payload = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 428 + assert "hidden Wormhole transport" in payload["detail"] + + +def test_private_infonet_gate_write_requires_wormhole_ready_but_not_rns(monkeypatch): + import main + from services import wormhole_supervisor + + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": False, + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + response = asyncio.run( + main.enforce_high_privacy_mesh(_request("/api/mesh/gate/test-gate/message"), call_next) + ) + + assert response.status_code == 200 + + +def test_private_infonet_gate_write_blocks_when_wormhole_not_ready(monkeypatch): + import main + from services import wormhole_supervisor + + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": False, + "rns_ready": True, + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + response = asyncio.run( + main.enforce_high_privacy_mesh(_request("/api/mesh/gate/test-gate/message"), call_next) + ) + payload = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 428 + assert payload == { + "ok": False, + "detail": "transport tier insufficient", + "required": "private_transitional", + "current": "public_degraded", + } + + +def test_private_dm_send_blocks_at_transitional_tier(monkeypatch): + import main + from services import wormhole_settings, wormhole_supervisor + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": False, + }, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": False, + }, + ) + + async def call_next(_request: Request) -> Response: + return Response(status_code=200) + + response = asyncio.run(main.enforce_high_privacy_mesh(_request("/api/mesh/dm/send"), call_next)) + payload = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 428 + assert payload == { + "ok": False, + "detail": "transport tier insufficient", + "required": "private_strong", + "current": "private_transitional", + } diff --git a/backend/tests/mesh/test_mesh_bootstrap_manifest.py b/backend/tests/mesh/test_mesh_bootstrap_manifest.py new file mode 100644 index 00000000..967d2153 --- /dev/null +++ b/backend/tests/mesh/test_mesh_bootstrap_manifest.py @@ -0,0 +1,193 @@ +import base64 +import json + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from services.mesh.mesh_bootstrap_manifest import ( + BOOTSTRAP_MANIFEST_VERSION, + BootstrapManifestError, + bootstrap_signer_public_key_b64, + build_bootstrap_manifest_payload, + generate_bootstrap_signer, + load_bootstrap_manifest, + write_signed_bootstrap_manifest, +) +from services.mesh.mesh_crypto import canonical_json + + +def _write_signed_manifest( + path, + *, + private_key, + peers, + issued_at=1_700_000_000, + valid_until=1_800_000_000, + signer_id="bootstrap-test", +): + payload = { + "version": BOOTSTRAP_MANIFEST_VERSION, + "issued_at": issued_at, + "valid_until": valid_until, + "signer_id": signer_id, + "peers": peers, + } + signature = base64.b64encode(private_key.sign(canonical_json(payload).encode("utf-8"))).decode("utf-8") + manifest = dict(payload) + manifest["signature"] = signature + path.write_text(json.dumps(manifest), encoding="utf-8") + return manifest + + +def test_load_bootstrap_manifest_roundtrip(tmp_path): + private_key = ed25519.Ed25519PrivateKey.generate() + public_key_b64 = base64.b64encode( + private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + ).decode("utf-8") + manifest_path = tmp_path / "bootstrap.json" + _write_signed_manifest( + manifest_path, + private_key=private_key, + peers=[ + { + "peer_url": "https://seed.example", + "transport": "clearnet", + "role": "seed", + "label": "Primary seed", + }, + { + "peer_url": "http://alphaexample.onion", + "transport": "onion", + "role": "relay", + }, + ], + ) + + manifest = load_bootstrap_manifest( + manifest_path, + signer_public_key_b64=public_key_b64, + now=1_750_000_000, + ) + + assert manifest.signer_id == "bootstrap-test" + assert [peer.peer_url for peer in manifest.peers] == [ + "https://seed.example", + "http://alphaexample.onion", + ] + assert [peer.transport for peer in manifest.peers] == ["clearnet", "onion"] + + +def test_load_bootstrap_manifest_fails_on_tamper(tmp_path): + private_key = ed25519.Ed25519PrivateKey.generate() + public_key_b64 = base64.b64encode( + private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + ).decode("utf-8") + manifest_path = tmp_path / "bootstrap.json" + manifest = _write_signed_manifest( + manifest_path, + private_key=private_key, + peers=[ + { + "peer_url": "https://seed.example", + "transport": "clearnet", + "role": "seed", + } + ], + ) + manifest["peers"][0]["peer_url"] = "https://evil.example" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(BootstrapManifestError, match="signature invalid"): + load_bootstrap_manifest( + manifest_path, + signer_public_key_b64=public_key_b64, + now=1_750_000_000, + ) + + +def test_load_bootstrap_manifest_rejects_expired_manifest(tmp_path): + private_key = ed25519.Ed25519PrivateKey.generate() + public_key_b64 = base64.b64encode( + private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + ).decode("utf-8") + manifest_path = tmp_path / "bootstrap.json" + _write_signed_manifest( + manifest_path, + private_key=private_key, + peers=[ + { + "peer_url": "https://seed.example", + "transport": "clearnet", + "role": "seed", + } + ], + issued_at=100, + valid_until=200, + ) + + with pytest.raises(BootstrapManifestError, match="expired"): + load_bootstrap_manifest( + manifest_path, + signer_public_key_b64=public_key_b64, + now=500, + ) + + +def test_generate_bootstrap_signer_roundtrip(): + signer = generate_bootstrap_signer() + assert signer["private_key_b64"] + assert signer["public_key_b64"] + assert bootstrap_signer_public_key_b64(signer["private_key_b64"]) == signer["public_key_b64"] + + +def test_write_signed_bootstrap_manifest_roundtrip(tmp_path): + signer = generate_bootstrap_signer() + manifest_path = tmp_path / "bootstrap.json" + + manifest = write_signed_bootstrap_manifest( + manifest_path, + signer_id="seed-alpha", + signer_private_key_b64=signer["private_key_b64"], + peers=[ + { + "peer_url": "https://seed.example", + "transport": "clearnet", + "role": "seed", + "label": "Primary seed", + } + ], + valid_for_hours=24, + ) + + loaded = load_bootstrap_manifest( + manifest_path, + signer_public_key_b64=signer["public_key_b64"], + now=manifest.issued_at + 60, + ) + + assert loaded.signer_id == "seed-alpha" + assert [peer.peer_url for peer in loaded.peers] == ["https://seed.example"] + + +def test_build_bootstrap_manifest_payload_rejects_invalid_peers(): + with pytest.raises(BootstrapManifestError, match="clearnet bootstrap peers must use https://"): + build_bootstrap_manifest_payload( + signer_id="seed-alpha", + peers=[ + { + "peer_url": "http://seed.example", + "transport": "clearnet", + "role": "seed", + } + ], + ) diff --git a/backend/tests/mesh/test_mesh_canonical.py b/backend/tests/mesh/test_mesh_canonical.py new file mode 100644 index 00000000..84e5afb0 --- /dev/null +++ b/backend/tests/mesh/test_mesh_canonical.py @@ -0,0 +1,19 @@ +import json +from pathlib import Path + +from services.mesh.mesh_crypto import build_signature_payload + + +def test_mesh_canonical_fixtures() -> None: + root = Path(__file__).resolve().parents[3] + fixtures_path = root / "docs" / "mesh" / "mesh-canonical-fixtures.json" + fixtures = json.loads(fixtures_path.read_text(encoding="utf-8")) + + for case in fixtures: + result = build_signature_payload( + event_type=case["event_type"], + node_id=case["node_id"], + sequence=case["sequence"], + payload=case["payload"], + ) + assert result == case["expected"], f"fixture mismatch: {case['name']}" diff --git a/backend/tests/mesh/test_mesh_crypto.py b/backend/tests/mesh/test_mesh_crypto.py new file mode 100644 index 00000000..7c42305b --- /dev/null +++ b/backend/tests/mesh/test_mesh_crypto.py @@ -0,0 +1,51 @@ +import base64 + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ec, ed25519 +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat + +from services.mesh.mesh_crypto import build_signature_payload, verify_signature + + +def test_ed25519_signature_roundtrip(): + key = ed25519.Ed25519PrivateKey.generate() + pub_raw = key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + public_key_b64 = base64.b64encode(pub_raw).decode("utf-8") + + payload = {"message": "hello", "destination": "broadcast", "channel": "LongFast"} + sig_payload = build_signature_payload( + event_type="message", + node_id="!sb_test", + sequence=1, + payload=payload, + ) + signature = key.sign(sig_payload.encode("utf-8")).hex() + + assert verify_signature( + public_key_b64=public_key_b64, + public_key_algo="Ed25519", + signature_hex=signature, + payload=sig_payload, + ) + + +def test_ecdsa_signature_roundtrip(): + key = ec.generate_private_key(ec.SECP256R1()) + pub_raw = key.public_key().public_bytes(Encoding.X962, PublicFormat.UncompressedPoint) + public_key_b64 = base64.b64encode(pub_raw).decode("utf-8") + + payload = {"target_id": "!sb_abc12345", "vote": 1, "gate": ""} + sig_payload = build_signature_payload( + event_type="vote", + node_id="!sb_test", + sequence=5, + payload=payload, + ) + signature = key.sign(sig_payload.encode("utf-8"), ec.ECDSA(hashes.SHA256())).hex() + + assert verify_signature( + public_key_b64=public_key_b64, + public_key_algo="ECDSA_P256", + signature_hex=signature, + payload=sig_payload, + ) diff --git a/backend/tests/mesh/test_mesh_dm_consent_privacy.py b/backend/tests/mesh/test_mesh_dm_consent_privacy.py new file mode 100644 index 00000000..3df3f2f4 --- /dev/null +++ b/backend/tests/mesh/test_mesh_dm_consent_privacy.py @@ -0,0 +1,280 @@ +import asyncio +import json +import time + +from starlette.requests import Request + + +def _json_request(path: str, body: dict) -> Request: + payload = json.dumps(body).encode("utf-8") + sent = {"value": False} + + async def receive(): + if sent["value"]: + return {"type": "http.request", "body": b"", "more_body": False} + sent["value"] = True + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "headers": [(b"content-type", b"application/json")], + "client": ("test", 12345), + "method": "POST", + "path": path, + }, + receive, + ) + + +def test_dm_send_keeps_encrypted_payloads_off_ledger(monkeypatch): + import main + from services import wormhole_supervisor + from services.mesh import mesh_hashchain, mesh_dm_relay + + append_called = {"value": False} + + monkeypatch.setattr( + main, + "_verify_signed_event", + lambda **kwargs: (True, ""), + ) + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + + def fake_append(**kwargs): + append_called["value"] = True + return {"event_id": "unexpected"} + + monkeypatch.setattr(mesh_hashchain.infonet, "append", fake_append) + monkeypatch.setattr(mesh_hashchain.infonet, "validate_and_set_sequence", lambda *_args, **_kwargs: (True, "")) + monkeypatch.setattr(mesh_dm_relay.dm_relay, "consume_nonce", lambda *_args, **_kwargs: (True, "ok")) + monkeypatch.setattr( + mesh_dm_relay.dm_relay, + "deposit", + lambda **kwargs: { + "ok": True, + "msg_id": kwargs.get("msg_id", ""), + "detail": "stored", + }, + ) + + req = _json_request( + "/api/mesh/dm/send", + { + "sender_id": "alice", + "recipient_id": "bob", + "delivery_class": "request", + "recipient_token": "", + "ciphertext": "x3dh1:opaque", + "msg_id": "m1", + "timestamp": int(time.time()), + "public_key": "cHVi", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 1, + "protocol_version": "infonet/2", + }, + ) + + response = asyncio.run(main.dm_send(req)) + + assert response["ok"] is True + assert append_called["value"] is False + + +def test_dm_key_registration_keeps_key_material_off_ledger(monkeypatch): + import main + from services.mesh import mesh_hashchain, mesh_dm_relay + + append_called = {"value": False} + + monkeypatch.setattr( + main, + "_verify_signed_event", + lambda **kwargs: (True, ""), + ) + + def fake_append(**kwargs): + append_called["value"] = True + return {"event_id": "unexpected"} + + monkeypatch.setattr(mesh_hashchain.infonet, "append", fake_append) + monkeypatch.setattr( + mesh_dm_relay.dm_relay, + "register_dh_key", + lambda *args, **kwargs: (True, "ok", {"bundle_fingerprint": "bf", "accepted_sequence": 1}), + ) + + req = _json_request( + "/api/mesh/dm/register", + { + "agent_id": "alice", + "dh_pub_key": "dhpub", + "dh_algo": "X25519", + "timestamp": int(time.time()), + "public_key": "cHVi", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 1, + "protocol_version": "infonet/2", + }, + ) + + response = asyncio.run(main.dm_register_key(req)) + + assert response["ok"] is True + assert append_called["value"] is False + + +def test_wormhole_dm_key_registration_keeps_key_material_off_ledger(tmp_path, monkeypatch): + import main + from services.mesh import ( + mesh_hashchain, + mesh_secure_storage, + mesh_wormhole_persona, + ) + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + + append_called = {"value": False} + + def fake_append(**kwargs): + append_called["value"] = True + return {"event_id": "unexpected"} + + monkeypatch.setattr(mesh_hashchain.infonet, "append", fake_append) + monkeypatch.setattr( + main, + "register_wormhole_prekey_bundle", + lambda *args, **kwargs: {"ok": True, "bundle": {}}, + ) + + response = asyncio.run(main.api_wormhole_dm_register_key(_json_request("/api/wormhole/dm/register-key", {}))) + + assert response["ok"] is True + assert append_called["value"] is False + + +def test_dead_drop_contact_consent_helpers_round_trip(): + from services.mesh.mesh_wormhole_dead_drop import ( + build_contact_accept, + build_contact_deny, + build_contact_offer, + parse_contact_consent, + ) + + offer = build_contact_offer(dh_pub_key="dhpub", dh_algo="X25519", geo_hint="40.12,-105.27") + accept = build_contact_accept(shared_alias="dmx_pairwise") + deny = build_contact_deny(reason="declined") + + assert parse_contact_consent(offer) == { + "kind": "contact_offer", + "dh_pub_key": "dhpub", + "dh_algo": "X25519", + "geo_hint": "40.12,-105.27", + } + assert parse_contact_consent(accept) == { + "kind": "contact_accept", + "shared_alias": "dmx_pairwise", + } + assert parse_contact_consent(deny) == { + "kind": "contact_deny", + "reason": "declined", + } + + +def test_pairwise_alias_is_separate_from_gate_identities(tmp_path, monkeypatch): + from services.mesh import ( + mesh_secure_storage, + mesh_wormhole_contacts, + mesh_wormhole_dead_drop, + mesh_wormhole_persona, + ) + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + monkeypatch.setattr(mesh_wormhole_contacts, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_contacts, "CONTACTS_FILE", tmp_path / "wormhole_dm_contacts.json") + + gate_session = mesh_wormhole_persona.enter_gate_anonymously("infonet", rotate=True)["identity"] + gate_persona = mesh_wormhole_persona.create_gate_persona("infonet", label="watcher")["identity"] + dm_identity = mesh_wormhole_persona.get_dm_identity() + + issued = mesh_wormhole_dead_drop.issue_pairwise_dm_alias( + peer_id="peer_alpha", + peer_dh_pub="dhpub_alpha", + ) + + assert issued["ok"] is True + assert issued["identity_scope"] == "dm_alias" + assert issued["shared_alias"].startswith("dmx_") + assert issued["shared_alias"] != gate_session["node_id"] + assert issued["shared_alias"] != gate_persona["node_id"] + assert issued["shared_alias"] != dm_identity["node_id"] + assert issued["dm_identity_id"] == dm_identity["node_id"] + assert issued["contact"]["sharedAlias"] == issued["shared_alias"] + assert issued["contact"]["dhPubKey"] == "dhpub_alpha" + + +def test_pairwise_alias_rotation_promotes_after_grace(tmp_path, monkeypatch): + from services.mesh import ( + mesh_secure_storage, + mesh_wormhole_contacts, + mesh_wormhole_dead_drop, + mesh_wormhole_persona, + ) + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + monkeypatch.setattr(mesh_wormhole_contacts, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_contacts, "CONTACTS_FILE", tmp_path / "wormhole_dm_contacts.json") + + initial = mesh_wormhole_dead_drop.issue_pairwise_dm_alias( + peer_id="peer_beta", + peer_dh_pub="dhpub_beta", + ) + rotated = mesh_wormhole_dead_drop.rotate_pairwise_dm_alias( + peer_id="peer_beta", + peer_dh_pub="dhpub_beta", + grace_ms=5_000, + ) + + assert rotated["ok"] is True + assert rotated["active_alias"] == initial["shared_alias"] + assert rotated["pending_alias"].startswith("dmx_") + assert rotated["pending_alias"] != initial["shared_alias"] + assert rotated["contact"]["sharedAlias"] == initial["shared_alias"] + assert rotated["contact"]["pendingSharedAlias"] == rotated["pending_alias"] + assert rotated["contact"]["sharedAliasGraceUntil"] >= rotated["grace_until"] + + future = rotated["grace_until"] / 1000.0 + 1 + monkeypatch.setattr(mesh_wormhole_contacts.time, "time", lambda: future) + promoted = mesh_wormhole_contacts.list_wormhole_dm_contacts()["peer_beta"] + + assert promoted["sharedAlias"] == rotated["pending_alias"] + assert promoted["pendingSharedAlias"] == "" + assert promoted["sharedAliasGraceUntil"] == 0 + assert initial["shared_alias"] in promoted["previousSharedAliases"] diff --git a/backend/tests/mesh/test_mesh_dm_mls.py b/backend/tests/mesh/test_mesh_dm_mls.py new file mode 100644 index 00000000..58dadfd6 --- /dev/null +++ b/backend/tests/mesh/test_mesh_dm_mls.py @@ -0,0 +1,318 @@ +import asyncio +import time + +REQUEST_CLAIMS = [{"type": "requests", "token": "request-claim-token"}] + + +def _fresh_dm_mls_state(tmp_path, monkeypatch): + from services import wormhole_supervisor + from services.mesh import mesh_dm_mls, mesh_dm_relay, mesh_secure_storage, mesh_wormhole_persona + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + monkeypatch.setattr(mesh_dm_mls, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_dm_mls, "STATE_FILE", tmp_path / "wormhole_dm_mls.json") + monkeypatch.setattr(mesh_dm_relay, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_dm_relay, "RELAY_FILE", tmp_path / "dm_relay.json") + monkeypatch.setattr( + mesh_dm_mls, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + relay = mesh_dm_relay.DMRelay() + monkeypatch.setattr(mesh_dm_relay, "dm_relay", relay) + mesh_dm_mls.reset_dm_mls_state(clear_privacy_core=True, clear_persistence=True) + return mesh_dm_mls, relay + + +def test_dm_mls_initiate_accept_encrypt_decrypt_round_trip(tmp_path, monkeypatch): + dm_mls, _relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + + bob_bundle = dm_mls.export_dm_key_package_for_alias("bob") + assert bob_bundle["ok"] is True + assert bob_bundle["welcome_dh_pub"] + + initiated = dm_mls.initiate_dm_session("alice", "bob", bob_bundle) + assert initiated["ok"] is True + assert initiated["welcome"] + + accepted = dm_mls.accept_dm_session("bob", "alice", initiated["welcome"]) + assert accepted["ok"] is True + + encrypted = dm_mls.encrypt_dm("alice", "bob", "hello bob") + assert encrypted["ok"] is True + decrypted = dm_mls.decrypt_dm("bob", "alice", encrypted["ciphertext"], encrypted["nonce"]) + assert decrypted == { + "ok": True, + "plaintext": "hello bob", + "session_id": accepted["session_id"], + "nonce": encrypted["nonce"], + } + + encrypted_back = dm_mls.encrypt_dm("bob", "alice", "hello alice") + assert encrypted_back["ok"] is True + decrypted_back = dm_mls.decrypt_dm( + "alice", + "bob", + encrypted_back["ciphertext"], + encrypted_back["nonce"], + ) + assert decrypted_back["ok"] is True + assert decrypted_back["plaintext"] == "hello alice" + + +def test_dm_mls_lock_rejects_legacy_dm1_decrypt(tmp_path, monkeypatch): + import main + + dm_mls, _relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + + bob_bundle = dm_mls.export_dm_key_package_for_alias("bob") + initiated = dm_mls.initiate_dm_session("alice", "bob", bob_bundle) + accepted = dm_mls.accept_dm_session("bob", "alice", initiated["welcome"]) + assert accepted["ok"] is True + + encrypted = dm_mls.encrypt_dm("alice", "bob", "lock me in") + assert encrypted["ok"] is True + + first_decrypt = main.decrypt_wormhole_dm_envelope( + peer_id="alice-agent", + local_alias="bob", + remote_alias="alice", + ciphertext=encrypted["ciphertext"], + payload_format="mls1", + nonce=encrypted["nonce"], + ) + assert first_decrypt["ok"] is True + assert dm_mls.is_dm_locked_to_mls("bob", "alice") is True + + locked = main.decrypt_wormhole_dm_envelope( + peer_id="alice-agent", + local_alias="bob", + remote_alias="alice", + ciphertext="legacy-ciphertext", + payload_format="dm1", + nonce="legacy-nonce", + ) + assert locked == { + "ok": False, + "detail": "DM session is locked to MLS format", + "required_format": "mls1", + "current_format": "dm1", + } + + +def test_dm_mls_refuses_public_degraded_transport(tmp_path, monkeypatch): + dm_mls, _relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + + monkeypatch.setattr( + dm_mls, + "get_wormhole_state", + lambda: {"configured": False, "ready": False, "rns_ready": False}, + ) + + result = dm_mls.initiate_dm_session( + "alice", + "bob", + {"mls_key_package": "ZmFrZQ=="}, + ) + + assert result == {"ok": False, "detail": "DM MLS requires PRIVATE transport tier"} + + +def test_dm_mls_session_persistence_survives_same_process_restart(tmp_path, monkeypatch): + dm_mls, _relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + + bob_bundle = dm_mls.export_dm_key_package_for_alias("bob") + initiated = dm_mls.initiate_dm_session("alice", "bob", bob_bundle) + accepted = dm_mls.accept_dm_session("bob", "alice", initiated["welcome"]) + + dm_mls.reset_dm_mls_state(clear_privacy_core=False, clear_persistence=False) + + encrypted = dm_mls.encrypt_dm("alice", "bob", "persisted hello") + assert encrypted["ok"] is True + decrypted = dm_mls.decrypt_dm("bob", "alice", encrypted["ciphertext"], encrypted["nonce"]) + assert decrypted["ok"] is True + assert decrypted["plaintext"] == "persisted hello" + assert decrypted["session_id"] == accepted["session_id"] + + +def test_dm_mls_encrypt_detects_stale_session_after_privacy_core_reset(tmp_path, monkeypatch): + dm_mls, _relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + + bob_bundle = dm_mls.export_dm_key_package_for_alias("bob") + initiated = dm_mls.initiate_dm_session("alice", "bob", bob_bundle) + accepted = dm_mls.accept_dm_session("bob", "alice", initiated["welcome"]) + assert accepted["ok"] is True + + dm_mls.reset_dm_mls_state(clear_privacy_core=True, clear_persistence=False) + + expired = dm_mls.encrypt_dm("alice", "bob", "stale handle") + assert expired == { + "ok": False, + "detail": "session_expired", + "session_id": "alice::bob", + } + assert dm_mls.has_dm_session("alice", "bob") == { + "ok": True, + "exists": False, + "session_id": "alice::bob", + } + + +def test_dm_mls_recreates_alias_identity_when_binding_proof_is_tampered(tmp_path, monkeypatch, caplog): + import logging + + from services.mesh.mesh_secure_storage import read_domain_json, write_domain_json + + dm_mls, _relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + + first_bundle = dm_mls.export_dm_key_package_for_alias("alice") + assert first_bundle["ok"] is True + + stored = read_domain_json(dm_mls.STATE_DOMAIN, dm_mls.STATE_FILENAME, dm_mls._default_state) + original_handle = int(stored["aliases"]["alice"]["handle"]) + stored["aliases"]["alice"]["binding_proof"] = "00" * 64 + write_domain_json(dm_mls.STATE_DOMAIN, dm_mls.STATE_FILENAME, stored) + + dm_mls.reset_dm_mls_state(clear_privacy_core=False, clear_persistence=False) + + with caplog.at_level(logging.WARNING): + second_bundle = dm_mls.export_dm_key_package_for_alias("alice") + + reloaded = read_domain_json(dm_mls.STATE_DOMAIN, dm_mls.STATE_FILENAME, dm_mls._default_state) + assert second_bundle["ok"] is True + assert "dm mls alias binding invalid for alice" in caplog.text.lower() + assert int(reloaded["aliases"]["alice"]["handle"]) != original_handle + + +def test_dm_mls_http_compose_store_poll_decrypt_round_trip(tmp_path, monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_hashchain + from services import wormhole_supervisor + + dm_mls, relay = _fresh_dm_mls_state(tmp_path, monkeypatch) + bob_bundle = dm_mls.export_dm_key_package_for_alias("bob") + assert bob_bundle["ok"] is True + + monkeypatch.setattr(main, "_current_admin_key", lambda: "test-admin") + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "ok")) + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: ( + True, + "ok", + {"mailbox_claims": REQUEST_CLAIMS}, + ), + ) + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr( + dm_mls, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": True}, + ) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": True}, + ) + monkeypatch.setattr(mesh_hashchain.infonet, "validate_and_set_sequence", lambda *_args, **_kwargs: (True, "ok")) + + admin_headers = {"X-Admin-Key": main._current_admin_key()} + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + now = int(time.time()) + compose_response = await ac.post( + "/api/wormhole/dm/compose", + json={ + "peer_id": "bob-agent", + "plaintext": "hello through http", + "local_alias": "alice", + "remote_alias": "bob", + "remote_prekey_bundle": bob_bundle, + }, + headers=admin_headers, + ) + composed = compose_response.json() + send_response = await ac.post( + "/api/mesh/dm/send", + json={ + "sender_id": "alice-agent", + "recipient_id": "bob-agent", + "delivery_class": "request", + "ciphertext": composed["ciphertext"], + "format": composed["format"], + "session_welcome": composed["session_welcome"], + "msg_id": "dm-mls-http-1", + "timestamp": now, + "nonce": "http-mls-nonce-1", + "public_key": "cHVi", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 11, + "protocol_version": "infonet/2", + }, + ) + poll_response = await ac.post( + "/api/mesh/dm/poll", + json={ + "agent_id": "bob-agent", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": now + 1, + "nonce": "http-mls-nonce-2", + "public_key": "cHVi", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 12, + "protocol_version": "infonet/2", + }, + ) + polled = poll_response.json() + decrypt_response = await ac.post( + "/api/wormhole/dm/decrypt", + json={ + "peer_id": "alice-agent", + "local_alias": "bob", + "remote_alias": "alice", + "ciphertext": polled["messages"][0]["ciphertext"], + "format": polled["messages"][0]["format"], + "nonce": "", + "session_welcome": polled["messages"][0]["session_welcome"], + }, + headers=admin_headers, + ) + return composed, send_response.json(), polled, decrypt_response.json() + + composed, sent, polled, decrypted = asyncio.run(_run()) + + assert composed["ok"] is True + assert composed["format"] == "mls1" + assert sent["ok"] is True + assert sent["msg_id"] == "dm-mls-http-1" + assert polled["ok"] is True + assert polled["count"] == 1 + assert polled["messages"][0]["format"] == "mls1" + assert polled["messages"][0]["session_welcome"] == composed["session_welcome"] + assert decrypted == { + "ok": True, + "peer_id": "alice-agent", + "local_alias": "bob", + "remote_alias": "alice", + "plaintext": "hello through http", + "format": "mls1", + } + assert relay.count_claims("bob-agent", REQUEST_CLAIMS) == 0 diff --git a/backend/tests/mesh/test_mesh_dm_security.py b/backend/tests/mesh/test_mesh_dm_security.py new file mode 100644 index 00000000..6bac4967 --- /dev/null +++ b/backend/tests/mesh/test_mesh_dm_security.py @@ -0,0 +1,343 @@ +import json +import time + +from services.config import get_settings +from services.mesh import mesh_dm_relay, mesh_schema, mesh_secure_storage + +REQUEST_CLAIM = [{"type": "requests", "token": "request-claim-token"}] + + +def _fresh_relay(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_dm_relay, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_dm_relay, "RELAY_FILE", tmp_path / "dm_relay.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + get_settings.cache_clear() + return mesh_dm_relay.DMRelay() + + +def test_dm_key_registration_is_monotonic(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + ok, reason, meta = relay.register_dh_key( + "alice", + "pub1", + "X25519", + 100, + "sig1", + "nodepub", + "Ed25519", + "infonet/2", + 1, + ) + assert ok, reason + assert meta["accepted_sequence"] == 1 + assert meta["bundle_fingerprint"] + + ok, reason, _ = relay.register_dh_key( + "alice", + "pub1", + "X25519", + 100, + "sig1", + "nodepub", + "Ed25519", + "infonet/2", + 1, + ) + assert not ok + assert "rollback" in reason.lower() or "replay" in reason.lower() + + ok, reason, _ = relay.register_dh_key( + "alice", + "pub2", + "X25519", + 99, + "sig2", + "nodepub", + "Ed25519", + "infonet/2", + 2, + ) + assert not ok + assert "older" in reason.lower() + + ok, reason, meta = relay.register_dh_key( + "alice", + "pub3", + "X25519", + 101, + "sig3", + "nodepub", + "Ed25519", + "infonet/2", + 2, + ) + assert ok, reason + assert meta["accepted_sequence"] == 2 + + +def test_secure_mailbox_claims_split_requests_and_shared(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + request_result = relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher_req", + msg_id="msg_req", + delivery_class="request", + ) + shared_result = relay.deposit( + sender_id="carol", + recipient_id="bob", + ciphertext="cipher_shared", + msg_id="msg_shared", + delivery_class="shared", + recipient_token="sharedtoken", + ) + + assert request_result["ok"] + assert shared_result["ok"] + assert relay.count_legacy(agent_id="bob") == 0 + + request_claims = REQUEST_CLAIM + shared_claims = [{"type": "shared", "token": "sharedtoken"}] + + assert relay.count_claims("bob", request_claims) == 1 + assert relay.count_claims("bob", shared_claims) == 1 + + request_messages = relay.collect_claims("bob", request_claims) + assert [msg["msg_id"] for msg in request_messages] == ["msg_req"] + assert request_messages[0]["delivery_class"] == "request" + assert relay.count_claims("bob", request_claims) == 0 + assert relay.count_claims("bob", [{"type": "requests"}]) == 0 + + shared_messages = relay.collect_claims("bob", shared_claims) + assert [msg["msg_id"] for msg in shared_messages] == ["msg_shared"] + assert shared_messages[0]["delivery_class"] == "shared" + assert relay.count_claims("bob", shared_claims) == 0 + + +def test_legacy_collect_and_count_require_agent_token(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + relay._mailboxes["legacy-token"].append( + mesh_dm_relay.DMMessage( + sender_id="alice", + ciphertext="cipher", + timestamp=time.time(), + msg_id="legacy-1", + delivery_class="request", + ) + ) + + assert relay.collect_legacy(agent_id="bob") == [] + assert relay.count_legacy(agent_id="bob") == 0 + assert relay.count_legacy(agent_token="legacy-token") == 1 + + +def test_nonce_replay_and_memory_only_spool(tmp_path, monkeypatch): + monkeypatch.setenv("MESH_DM_PERSIST_SPOOL", "false") + relay = _fresh_relay(tmp_path, monkeypatch) + + result = relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher", + msg_id="msg1", + delivery_class="request", + ) + assert result["ok"] + assert mesh_dm_relay.RELAY_FILE.exists() + + payload = json.loads(mesh_dm_relay.RELAY_FILE.read_text(encoding="utf-8")) + assert payload.get("kind") == "sb_secure_json" + + restored = mesh_secure_storage.read_secure_json(mesh_dm_relay.RELAY_FILE, lambda: {}) + assert "mailboxes" not in restored + + ok, reason = relay.consume_nonce("bob", "nonce-1", 100) + assert ok, reason + ok, reason = relay.consume_nonce("bob", "nonce-1", 100) + assert not ok + assert reason == "nonce replay detected" + + +def test_request_mailbox_token_binding_requires_presented_token(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + legacy_key = relay.mailbox_key_for_delivery( + recipient_id="bob", + delivery_class="request", + ) + presented_token = "mailbox-token-bob" + hashed = relay._hashed_mailbox_token(presented_token) + + assert legacy_key != hashed + claimed = relay.claim_mailbox_keys("bob", [{"type": "requests", "token": presented_token}]) + assert claimed[0] == hashed + assert legacy_key in claimed + assert relay.mailbox_key_for_delivery(recipient_id="bob", delivery_class="request") == hashed + + +def test_shared_delivery_uses_hashed_mailbox_token(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + result = relay.deposit( + sender_id="alice", + recipient_id="", + ciphertext="cipher_shared", + msg_id="msg_shared_hash", + delivery_class="shared", + recipient_token="shared-mailbox-token", + sender_token_hash="abc123", + ) + + assert result["ok"] is True + mailbox_key = relay._hashed_mailbox_token("shared-mailbox-token") + assert list(relay._mailboxes.keys()) == [mailbox_key] + assert relay._mailboxes[mailbox_key][0].sender_id == "sender_token:abc123" + + +def test_request_and_shared_claims_freeze_current_sender_identity_contract(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + request_result = relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher-req", + msg_id="msg-req-1", + delivery_class="request", + ) + shared_result = relay.deposit( + sender_id="alice", + recipient_id="", + ciphertext="cipher-shared", + msg_id="msg-shared-1", + delivery_class="shared", + recipient_token="shared-mailbox-token", + sender_token_hash="abc123", + sender_seal="v3:sealed", + ) + + assert request_result["ok"] is True + assert shared_result["ok"] is True + + request_messages = relay.collect_claims("bob", [{"type": "requests", "token": "request-claim-token"}]) + shared_messages = relay.collect_claims("bob", [{"type": "shared", "token": "shared-mailbox-token"}]) + + assert request_messages == [ + { + "sender_id": "alice", + "ciphertext": "cipher-req", + "timestamp": request_messages[0]["timestamp"], + "msg_id": "msg-req-1", + "delivery_class": "request", + "sender_seal": "", + "format": "dm1", + "session_welcome": "", + } + ] + assert shared_messages == [ + { + "sender_id": "sender_token:abc123", + "ciphertext": "cipher-shared", + "timestamp": shared_messages[0]["timestamp"], + "msg_id": "msg-shared-1", + "delivery_class": "shared", + "sender_seal": "v3:sealed", + "format": "dm1", + "session_welcome": "", + } + ] + + +def test_block_purges_and_rejects_reduced_sender_handles(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + + first = relay.deposit( + sender_id="sealed:first", + raw_sender_id="alice", + recipient_id="bob", + ciphertext="cipher-req", + msg_id="msg-sealed-1", + delivery_class="request", + sender_seal="v3:test-seal", + ) + + assert first["ok"] is True + relay.block("bob", "alice") + assert relay.count_claims("bob", REQUEST_CLAIM) == 0 + + second = relay.deposit( + sender_id="sealed:second", + raw_sender_id="alice", + recipient_id="bob", + ciphertext="cipher-req-2", + msg_id="msg-sealed-2", + delivery_class="request", + sender_seal="v3:test-seal", + ) + + assert second == {"ok": False, "detail": "Recipient is not accepting your messages"} + assert relay.count_claims("bob", REQUEST_CLAIM) == 0 + + +def test_nonce_cache_is_bounded_and_expires_entries(tmp_path, monkeypatch): + monkeypatch.setenv("MESH_DM_NONCE_CACHE_MAX", "2") + relay = _fresh_relay(tmp_path, monkeypatch) + current = {"value": 1_000.0} + monkeypatch.setattr(mesh_dm_relay.time, "time", lambda: current["value"]) + + assert relay.consume_nonce("bob", "nonce-1", 1_000)[0] is True + assert relay.consume_nonce("bob", "nonce-2", 1_000)[0] is True + assert len(relay._nonce_cache) == 2 + + ok, reason = relay.consume_nonce("bob", "nonce-3", 1_000) + assert ok is False + assert reason == "nonce cache at capacity" + assert len(relay._nonce_cache) == 2 + assert "bob:nonce-1" in relay._nonce_cache + assert "bob:nonce-2" in relay._nonce_cache + + current["value"] = 1_000.0 + 301.0 + assert relay.consume_nonce("bob", "nonce-2", 1_000)[0] is True + + +def test_dm_schema_requires_tokens_for_all_mailbox_claims(): + ok, reason = mesh_schema.validate_event_payload( + "dm_poll", + { + "mailbox_claims": [{"type": "requests", "token": ""}], + "timestamp": 123, + "nonce": "abc", + }, + ) + assert not ok + assert "token" in reason.lower() + + ok, reason = mesh_schema.validate_event_payload( + "dm_count", + { + "mailbox_claims": [{"type": "shared", "token": ""}], + "timestamp": 123, + "nonce": "abc", + }, + ) + assert not ok + assert "token" in reason.lower() + + ok, reason = mesh_schema.validate_event_payload( + "dm_message", + { + "recipient_id": "bob", + "delivery_class": "shared", + "recipient_token": "", + "ciphertext": "cipher", + "format": "mls1", + "msg_id": "m1", + "timestamp": 123, + }, + ) + assert not ok + assert "recipient_token" in reason.lower() diff --git a/backend/tests/mesh/test_mesh_dm_witness_fix.py b/backend/tests/mesh/test_mesh_dm_witness_fix.py new file mode 100644 index 00000000..03dfafb8 --- /dev/null +++ b/backend/tests/mesh/test_mesh_dm_witness_fix.py @@ -0,0 +1,86 @@ +"""Tests verifying the /api/mesh/dm/witness endpoint uses correct identifiers. + +The bug: _preflight_signed_event_integrity was called with event_type="trust_vouch" +and node_id=voucher_id (undefined NameError). Fixed to event_type="dm_key_witness" +and node_id=witness_id. +""" + +import ast +import textwrap +from pathlib import Path + +MAIN_PY = Path(__file__).resolve().parents[2] / "main.py" + + +def _find_dm_key_witness_func(source: str) -> ast.AsyncFunctionDef | None: + """Parse main.py AST and find the dm_key_witness POST handler.""" + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, ast.AsyncFunctionDef) and node.name == "dm_key_witness": + return node + return None + + +def test_witness_endpoint_uses_correct_event_type(): + """The preflight call must use event_type='dm_key_witness', not 'trust_vouch'.""" + source = MAIN_PY.read_text(encoding="utf-8") + func = _find_dm_key_witness_func(source) + assert func is not None, "dm_key_witness function not found in main.py" + + # Extract the function source and look for _preflight_signed_event_integrity call + func_source = ast.get_source_segment(source, func) + assert func_source is not None + + assert 'event_type="dm_key_witness"' in func_source, ( + "preflight call should use event_type='dm_key_witness', " + f"but the function contains: {func_source[:500]}" + ) + assert 'event_type="trust_vouch"' not in func_source, ( + "preflight call still uses the wrong event_type='trust_vouch'" + ) + + +def test_witness_endpoint_uses_witness_id_not_voucher_id(): + """The preflight call must use node_id=witness_id, not voucher_id.""" + source = MAIN_PY.read_text(encoding="utf-8") + func = _find_dm_key_witness_func(source) + assert func is not None, "dm_key_witness function not found in main.py" + + func_source = ast.get_source_segment(source, func) + assert func_source is not None + + # Find all _preflight_signed_event_integrity calls in the function + lines = func_source.splitlines() + in_preflight = False + preflight_block = [] + for line in lines: + if "_preflight_signed_event_integrity" in line: + in_preflight = True + if in_preflight: + preflight_block.append(line) + if ")" in line and line.strip().endswith(")"): + break + + preflight_text = "\n".join(preflight_block) + assert "node_id=witness_id" in preflight_text, ( + f"preflight call should use node_id=witness_id, got:\n{preflight_text}" + ) + assert "node_id=voucher_id" not in preflight_text, ( + "preflight call still references undefined voucher_id" + ) + + +def test_verify_signed_event_also_uses_dm_key_witness(): + """The _verify_signed_event call should also use event_type='dm_key_witness'.""" + source = MAIN_PY.read_text(encoding="utf-8") + func = _find_dm_key_witness_func(source) + assert func is not None + + func_source = ast.get_source_segment(source, func) + assert func_source is not None + + # Count occurrences of the correct event_type + assert func_source.count('event_type="dm_key_witness"') == 2, ( + "Both _verify_signed_event and _preflight_signed_event_integrity " + "should use event_type='dm_key_witness'" + ) diff --git a/backend/tests/mesh/test_mesh_endpoint_integrity.py b/backend/tests/mesh/test_mesh_endpoint_integrity.py new file mode 100644 index 00000000..2927ba11 --- /dev/null +++ b/backend/tests/mesh/test_mesh_endpoint_integrity.py @@ -0,0 +1,1129 @@ +import asyncio +import base64 +import json +import time +from types import SimpleNamespace + +import pytest +from cryptography.hazmat.primitives.asymmetric import ed25519 +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat +from httpx import ASGITransport, AsyncClient + + +class _DummyBreaker: + def check_and_record(self, _priority): + return True, "ok" + + +class _FakeMeshtasticTransport: + NAME = "meshtastic" + + def __init__(self): + self.sent = [] + + def can_reach(self, _envelope): + return True + + def send(self, envelope, _credentials): + from services.mesh.mesh_router import TransportResult + + self.sent.append(envelope) + return TransportResult(True, self.NAME, "sent") + + +class _FakeMeshRouter: + def __init__(self): + self.meshtastic = _FakeMeshtasticTransport() + self.breakers = {"meshtastic": _DummyBreaker()} + self.route_called = False + + def route(self, _envelope, _credentials): + self.route_called = True + return [] + + +class _FakeReputationLedger: + def __init__(self): + self.registered = [] + self.votes = [] + self.reputation: dict[str, dict] = {} + + def register_node(self, *args): + self.registered.append(args) + + def cast_vote(self, *args): + self.votes.append(args) + return True, "ok" + + def get_reputation(self, node_id): + return self.reputation.get(node_id, {"overall": 0, "gates": {}, "upvotes": 0, "downvotes": 0}) + + def get_reputation_log(self, node_id, detailed=False): + rep = self.get_reputation(node_id) + result = {"node_id": node_id, **rep} + if detailed: + result["recent_votes"] = [] + return result + + +class _FakeGateManager: + def __init__(self): + self.recorded = [] + self.enter_checks = [] + + def can_enter(self, sender_id, gate_id): + self.enter_checks.append((sender_id, gate_id)) + return True, "ok" + + def record_message(self, gate_id): + self.recorded.append(gate_id) + + +def test_recent_private_clearnet_fallback_warning_tracks_private_internet_route(monkeypatch): + from collections import deque + + from services import wormhole_supervisor + from services.mesh import mesh_router + + now = 1_700_000_000.0 + monkeypatch.setattr( + mesh_router, + "mesh_router", + SimpleNamespace( + message_log=deque( + [ + { + "trust_tier": "private_transitional", + "routed_via": "internet", + "route_reason": "Payload too large for radio or radio transports failed — internet relay", + "timestamp": now - 15, + } + ], + maxlen=500, + ) + ), + ) + + warning = wormhole_supervisor._recent_private_clearnet_fallback_warning(now=now) + + assert warning["recent_private_clearnet_fallback"] is True + assert warning["recent_private_clearnet_fallback_at"] == int(now - 15) + assert "internet relay" in warning["recent_private_clearnet_fallback_reason"].lower() + + +def test_mesh_reputation_batch_returns_overall_scores(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_reputation as mesh_reputation_mod + + fake_ledger = _FakeReputationLedger() + fake_ledger.reputation = { + "!alpha": {"overall": 7, "gates": {}, "upvotes": 4, "downvotes": 1}, + "!bravo": {"overall": -2, "gates": {}, "upvotes": 1, "downvotes": 3}, + } + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.get("/api/mesh/reputation/batch?node_id=!alpha&node_id=!bravo") + return response.json() + + result = asyncio.run(_run()) + + assert result == {"ok": True, "reputations": {"!alpha": 7, "!bravo": -2}} + + +def test_wormhole_gate_message_batch_decrypt_preserves_order(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services import wormhole_supervisor + + monkeypatch.setattr(main, "_debug_mode_enabled", lambda: True) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + calls = [] + + def fake_decrypt(**kwargs): + calls.append(kwargs) + return { + "ok": True, + "gate_id": kwargs["gate_id"], + "epoch": int(kwargs.get("epoch", 0) or 0) + 1, + "plaintext": f"plain:{kwargs['ciphertext']}", + } + + monkeypatch.setattr(main, "decrypt_gate_message_for_local_identity", fake_decrypt) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/wormhole/gate/messages/decrypt", + json={ + "messages": [ + {"gate_id": "ops", "epoch": 2, "ciphertext": "ct-1", "nonce": "", "sender_ref": ""}, + {"gate_id": "ops", "epoch": 3, "ciphertext": "ct-2", "nonce": "", "sender_ref": ""}, + ] + }, + headers={"X-Admin-Key": main._current_admin_key()}, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result == { + "ok": True, + "results": [ + {"ok": True, "gate_id": "ops", "epoch": 3, "plaintext": "plain:ct-1"}, + {"ok": True, "gate_id": "ops", "epoch": 4, "plaintext": "plain:ct-2"}, + ], + } + assert [call["ciphertext"] for call in calls] == ["ct-1", "ct-2"] + + +def _gate_proof_identity(): + from services.mesh.mesh_crypto import derive_node_id + + signing_key = ed25519.Ed25519PrivateKey.generate() + private_raw = signing_key.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()) + public_raw = signing_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + public_key = base64.b64encode(public_raw).decode("ascii") + private_key = base64.b64encode(private_raw).decode("ascii") + return { + "node_id": derive_node_id(public_key), + "public_key": public_key, + "public_key_algo": "Ed25519", + "private_key": private_key, + "signing_key": signing_key, + } + + +def _send_body(**overrides): + body = { + "destination": "!a0cc7a80", + "message": "hello mesh", + "sender_id": "!sb_sender", + "node_id": "!sb_sender", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 11, + "protocol_version": "1", + "channel": "LongFast", + "priority": "normal", + "ephemeral": False, + "transport_lock": "meshtastic", + "credentials": {"mesh_region": "US"}, + } + body.update(overrides) + return body + + +def test_preflight_integrity_rejects_replay(monkeypatch): + import main + from services.mesh import mesh_hashchain as mesh_hashchain_mod + + fake_infonet = SimpleNamespace( + check_replay=lambda node_id, sequence: True, + node_sequences={"!node": 9}, + public_key_bindings={}, + _revocation_status=lambda public_key: (False, None), + ) + monkeypatch.setattr(mesh_hashchain_mod, "infonet", fake_infonet) + + ok, reason = main._preflight_signed_event_integrity( + event_type="vote", + node_id="!node", + sequence=9, + public_key="pub", + public_key_algo="Ed25519", + signature="sig", + protocol_version="1", + ) + + assert ok is False + assert "Replay detected" in reason + + +def test_signed_event_verification_always_requires_signature_fields(): + import main + + ok, reason = main._verify_signed_event( + event_type="dm_message", + node_id="!node", + sequence=1, + public_key="", + public_key_algo="", + signature="", + payload={"ciphertext": "c"}, + protocol_version="", + ) + assert ok is False + assert reason == "Missing protocol_version" + + ok, reason = main._preflight_signed_event_integrity( + event_type="dm_poll", + node_id="!node", + sequence=1, + public_key="", + public_key_algo="", + signature="", + protocol_version="", + ) + assert ok is False + assert reason == "Missing signature or public key" + + +def test_scoped_auth_uses_timing_safe_compare(monkeypatch): + import main + + compare_calls = [] + + def _fake_compare(left, right): + compare_calls.append((left, right)) + return True + + monkeypatch.setattr(main, "_current_admin_key", lambda: "top-secret") + monkeypatch.setattr(main, "_scoped_admin_tokens", lambda: {}) + monkeypatch.setattr(main.hmac, "compare_digest", _fake_compare) + + request = SimpleNamespace( + headers={"X-Admin-Key": "top-secret"}, + client=SimpleNamespace(host="203.0.113.10"), + url=SimpleNamespace(path="/api/wormhole/status"), + ) + + ok, detail = main._check_scoped_auth(request, "wormhole") + + assert ok is True + assert detail == "ok" + assert compare_calls == [(b"top-secret", b"top-secret")] + + +def test_scoped_auth_uses_timing_safe_compare_for_scoped_tokens(monkeypatch): + import main + + compare_calls = [] + + def _fake_compare(left, right): + compare_calls.append((left, right)) + return left == right + + monkeypatch.setattr(main, "_current_admin_key", lambda: "") + monkeypatch.setattr(main, "_scoped_admin_tokens", lambda: {"gate-token": ["gate"]}) + monkeypatch.setattr(main.hmac, "compare_digest", _fake_compare) + + request = SimpleNamespace( + headers={"X-Admin-Key": "gate-token"}, + client=SimpleNamespace(host="203.0.113.10"), + url=SimpleNamespace(path="/api/wormhole/gate/demo/message"), + ) + + ok, detail = main._check_scoped_auth(request, "gate") + + assert ok is True + assert detail == "ok" + assert compare_calls == [(b"gate-token", b"gate-token")] + + +def test_invalid_json_body_returns_422(): + import main + from httpx import ASGITransport, AsyncClient + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/send", + content="{", + headers={"content-type": "application/json"}, + ) + return response.status_code, response.json() + + status_code, payload = asyncio.run(_run()) + + assert status_code == 422 + assert payload == {"ok": False, "detail": "invalid JSON body"} + + +def test_arti_ready_requires_no_auth_socks5_response(monkeypatch): + from services import config as config_mod + from services import wormhole_supervisor + + class _FakeSocket: + def __init__(self, response: bytes): + self.response = response + self.sent = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def sendall(self, data: bytes): + self.sent.append(data) + + def recv(self, _size: int) -> bytes: + return self.response + + monkeypatch.setattr( + config_mod, + "get_settings", + lambda: SimpleNamespace(MESH_ARTI_ENABLED=True, MESH_ARTI_SOCKS_PORT=9050), + ) + monkeypatch.setattr( + wormhole_supervisor.socket, + "create_connection", + lambda *_args, **_kwargs: _FakeSocket(b"\x05\x02"), + ) + + assert wormhole_supervisor._check_arti_ready() is False + + +def test_gate_router_private_push_uses_opaque_gate_ref(monkeypatch): + from services import config as config_mod + from services.mesh.mesh_router import InternetTransport, MeshEnvelope + + monkeypatch.setattr( + config_mod, + "get_settings", + lambda: SimpleNamespace(MESH_PEER_PUSH_SECRET="peer-secret"), + ) + + envelope = MeshEnvelope( + sender_id="!sb_sender", + destination="broadcast", + payload=json.dumps( + { + "event_type": "gate_message", + "timestamp": 1710000000, + "payload": { + "gate": "finance", + "ciphertext": "abc123", + "format": "mls1", + }, + } + ), + ) + endpoint, body = InternetTransport()._build_peer_push_request(envelope, "internet") + payload = json.loads(body.rstrip(b" ").decode("utf-8")) + + assert endpoint == "/api/mesh/gate/peer-push" + gate_payload = payload["events"][0]["payload"] + assert "gate" not in gate_payload + assert gate_payload["gate_ref"] + + +def test_gate_router_private_push_freezes_current_v1_signer_bundle(monkeypatch): + from services import config as config_mod + from services.mesh.mesh_router import InternetTransport, MeshEnvelope + + monkeypatch.setattr( + config_mod, + "get_settings", + lambda: SimpleNamespace(MESH_PEER_PUSH_SECRET="peer-secret"), + ) + + envelope = MeshEnvelope( + sender_id="!sb_sender", + destination="broadcast", + payload=json.dumps( + { + "event_type": "gate_message", + "timestamp": 1710000000, + "event_id": "gate-evt-1", + "node_id": "!gate-persona-1", + "sequence": 19, + "signature": "deadbeef", + "public_key": "pubkey-1", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "payload": { + "gate": "finance", + "ciphertext": "abc123", + "format": "mls1", + "nonce": "nonce-7", + "sender_ref": "sender-ref-7", + "epoch": 4, + }, + } + ), + ) + + endpoint, body = InternetTransport()._build_peer_push_request(envelope, "internet") + payload = json.loads(body.rstrip(b" ").decode("utf-8")) + event = payload["events"][0] + + assert endpoint == "/api/mesh/gate/peer-push" + assert set(event.keys()) == { + "event_type", + "timestamp", + "payload", + "event_id", + "node_id", + "sequence", + "signature", + "public_key", + "public_key_algo", + "protocol_version", + } + assert event["event_id"] == "gate-evt-1" + assert event["node_id"] == "!gate-persona-1" + assert event["sequence"] == 19 + assert event["signature"] == "deadbeef" + assert event["public_key"] == "pubkey-1" + assert event["public_key_algo"] == "Ed25519" + assert event["protocol_version"] == "infonet/2" + assert set(event["payload"].keys()) == {"ciphertext", "format", "gate_ref", "nonce", "sender_ref", "epoch"} + assert event["payload"]["ciphertext"] == "abc123" + assert event["payload"]["format"] == "mls1" + assert event["payload"]["nonce"] == "nonce-7" + assert event["payload"]["sender_ref"] == "sender-ref-7" + assert event["payload"]["epoch"] == 4 + assert event["payload"]["gate_ref"] + assert "gate" not in event["payload"] + + +def test_gate_access_proof_round_trip_verifies_fresh_member_signature(monkeypatch): + import main + + identity = _gate_proof_identity() + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr(main, "_resolve_gate_proof_identity", lambda gate_id: dict(identity) if gate_id == "finance" else None) + monkeypatch.setattr( + main, + "_lookup_gate_member_binding", + lambda gate_id, node_id: (identity["public_key"], "Ed25519") + if gate_id == "finance" and node_id == identity["node_id"] + else None, + ) + + proof = main._sign_gate_access_proof("finance") + request = SimpleNamespace( + headers={ + "x-wormhole-node-id": identity["node_id"], + "x-wormhole-gate-proof": proof["proof"], + "x-wormhole-gate-ts": str(proof["ts"]), + } + ) + + assert proof["ok"] is True + assert main._verify_gate_access(request, "finance") is True + + +def test_gate_access_proof_rejects_stale_timestamp(monkeypatch): + import main + + identity = _gate_proof_identity() + stale_ts = int(time.time()) - 120 + signature = identity["signing_key"].sign(f"finance:{stale_ts}".encode("utf-8")) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr( + main, + "_lookup_gate_member_binding", + lambda gate_id, node_id: (identity["public_key"], "Ed25519") + if gate_id == "finance" and node_id == identity["node_id"] + else None, + ) + request = SimpleNamespace( + headers={ + "x-wormhole-node-id": identity["node_id"], + "x-wormhole-gate-proof": base64.b64encode(signature).decode("ascii"), + "x-wormhole-gate-ts": str(stale_ts), + } + ) + + assert main._verify_gate_access(request, "finance") is False + + +def test_gate_proof_endpoint_returns_signed_proof(monkeypatch): + import main + + identity = _gate_proof_identity() + monkeypatch.setattr(main, "_resolve_gate_proof_identity", lambda gate_id: dict(identity) if gate_id == "finance" else None) + monkeypatch.setattr(main, "_current_admin_key", lambda: "test-admin") + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/wormhole/gate/proof", + json={"gate_id": "finance"}, + headers={"x-admin-key": "test-admin"}, + ) + return response.status_code, response.json() + + status_code, result = asyncio.run(_run()) + + assert status_code == 200 + assert result["ok"] is True + assert result["gate_id"] == "finance" + assert result["node_id"] == identity["node_id"] + assert result["proof"] + + +def test_private_infonet_policy_marks_gate_actions_transitional(): + import main + + assert main._private_infonet_required_tier("/api/mesh/vote", "POST") == "transitional" + assert ( + main._private_infonet_required_tier("/api/mesh/gate/infonet/message", "POST") + == "transitional" + ) + assert main._private_infonet_required_tier("/api/mesh/dm/send", "POST") == "strong" + assert main._private_infonet_required_tier("/api/mesh/dm/poll", "GET") == "strong" + assert main._private_infonet_required_tier("/api/mesh/status", "GET") == "" + + +def test_current_private_lane_tier_reflects_runtime_readiness(): + import main + + assert main._current_private_lane_tier({"configured": False, "ready": False, "rns_ready": False}) == "public_degraded" + assert main._current_private_lane_tier({"configured": True, "ready": False, "rns_ready": True}) == "public_degraded" + assert main._current_private_lane_tier({"configured": True, "ready": True, "rns_ready": False}) == "private_transitional" + assert main._current_private_lane_tier({"configured": True, "ready": True, "rns_ready": True}) == "private_transitional" + assert main._current_private_lane_tier({"configured": True, "ready": True, "arti_ready": True, "rns_ready": True}) == "private_strong" + + +def test_message_payload_normalization_keeps_transport_lock(): + from services.mesh.mesh_protocol import normalize_message_payload + + normalized = normalize_message_payload( + { + "message": "hello mesh", + "destination": "broadcast", + "channel": "LongFast", + "priority": "normal", + "ephemeral": False, + "transport_lock": "Meshtastic", + } + ) + + assert normalized["transport_lock"] == "meshtastic" + + +def test_public_ledger_rejects_transport_lock(): + from services.mesh.mesh_schema import validate_public_ledger_payload + + ok, reason = validate_public_ledger_payload( + "message", + { + "message": "hello mesh", + "destination": "broadcast", + "channel": "LongFast", + "priority": "normal", + "ephemeral": False, + "transport_lock": "meshtastic", + }, + ) + + assert ok is False + assert "transport_lock" in reason + + +def test_preflight_integrity_rejects_public_key_binding_conflict(monkeypatch): + import main + from services.mesh import mesh_hashchain as mesh_hashchain_mod + + fake_infonet = SimpleNamespace( + check_replay=lambda node_id, sequence: False, + node_sequences={}, + public_key_bindings={"pub": "!other-node"}, + _revocation_status=lambda public_key: (False, None), + ) + monkeypatch.setattr(mesh_hashchain_mod, "infonet", fake_infonet) + + ok, reason = main._preflight_signed_event_integrity( + event_type="gate_message", + node_id="!node", + sequence=10, + public_key="pub", + public_key_algo="Ed25519", + signature="sig", + protocol_version="1", + ) + + assert ok is False + assert reason == "public key already bound to !other-node" + + +def test_mesh_send_blocks_before_transport_side_effect_when_integrity_fails(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_router as mesh_router_mod + + fake_router = _FakeMeshRouter() + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr( + main, + "_preflight_signed_event_integrity", + lambda **_: (False, "Replay detected: sequence 11 <= last 11"), + ) + monkeypatch.setattr(main, "_check_throttle", lambda *_: (True, "ok")) + monkeypatch.setattr(mesh_router_mod, "mesh_router", fake_router) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post("/api/mesh/send", json=_send_body()) + return response.json() + + result = asyncio.run(_run()) + + assert result == {"ok": False, "detail": "Replay detected: sequence 11 <= last 11"} + assert fake_router.route_called is False + assert fake_router.meshtastic.sent == [] + + +def test_mesh_vote_blocks_before_vote_side_effect_when_integrity_fails(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_reputation as mesh_reputation_mod + from services import wormhole_supervisor + + fake_ledger = _FakeReputationLedger() + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr( + main, + "_preflight_signed_event_integrity", + lambda **_: (False, "public key is revoked"), + ) + monkeypatch.setattr(main, "_validate_gate_vote_context", lambda *_: (True, "")) + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/vote", + json={ + "voter_id": "!voter", + "target_id": "!target", + "vote": 1, + "voter_pubkey": "pub", + "public_key_algo": "Ed25519", + "voter_sig": "sig", + "sequence": 4, + "protocol_version": "1", + }, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result == {"ok": False, "detail": "public key is revoked"} + assert fake_ledger.registered == [] + assert fake_ledger.votes == [] + + +def test_gate_message_blocks_before_gate_side_effect_when_integrity_fails(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_reputation as mesh_reputation_mod + from services import wormhole_supervisor + + fake_ledger = _FakeReputationLedger() + fake_gate_manager = _FakeGateManager() + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr( + main, + "_preflight_signed_event_integrity", + lambda **_: (False, "Replay detected: sequence 7 <= last 7"), + ) + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr(mesh_reputation_mod, "gate_manager", fake_gate_manager, raising=False) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/gate/infonet/message", + json={ + "sender_id": "!sender", + "epoch": 1, + "ciphertext": "opaque-ciphertext", + "nonce": "nonce-1", + "sender_ref": "gate-session-1", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 7, + "protocol_version": "1", + }, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result == {"ok": False, "detail": "Replay detected: sequence 7 <= last 7"} + assert fake_ledger.registered == [] + assert fake_gate_manager.enter_checks == [] + assert fake_gate_manager.recorded == [] + + +def test_gate_message_rejects_plaintext_payload_shape(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services import wormhole_supervisor + + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/gate/infonet/message", + json={ + "sender_id": "!sender", + "message": "hello gate", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 7, + "protocol_version": "1", + }, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result == { + "ok": False, + "detail": "Plaintext gate messages are no longer accepted. Submit an encrypted gate envelope.", + } + + +def test_gate_message_accepts_encrypted_envelope(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_hashchain as mesh_hashchain_mod + from services.mesh import mesh_reputation as mesh_reputation_mod + from services import wormhole_supervisor + + fake_ledger = _FakeReputationLedger() + fake_gate_manager = _FakeGateManager() + append_calls = [] + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr(mesh_reputation_mod, "gate_manager", fake_gate_manager, raising=False) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + + def fake_append(gate_id, event): + append_calls.append({"gate_id": gate_id, "event": event}) + return event + + monkeypatch.setattr(mesh_hashchain_mod.gate_store, "append", fake_append) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/gate/infonet/message", + json={ + "sender_id": "!sender", + "epoch": 3, + "ciphertext": "opaque-ciphertext", + "nonce": "nonce-3", + "sender_ref": "persona-ops-1", + "format": "mls1", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 9, + "protocol_version": "1", + }, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is True + assert result["detail"] == "Message posted to gate 'infonet'" + assert result["gate_id"] == "infonet" + assert result["event_id"] == append_calls[0]["event"]["event_id"] + assert fake_ledger.registered == [("!sender", "pub", "Ed25519")] + assert fake_gate_manager.enter_checks == [("!sender", "infonet")] + assert fake_gate_manager.recorded == ["infonet"] + assert append_calls[0]["gate_id"] == "infonet" + assert append_calls[0]["event"]["payload"] == { + "gate": "infonet", + "epoch": 3, + "ciphertext": "opaque-ciphertext", + "nonce": "nonce-3", + "sender_ref": "persona-ops-1", + "format": "mls1", + } + + +def test_gate_message_enforces_30_second_sender_cooldown(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_hashchain as mesh_hashchain_mod + from services.mesh import mesh_reputation as mesh_reputation_mod + from services import wormhole_supervisor + + class _Clock: + def __init__(self): + self.current = 1_000.0 + + def time(self): + return self.current + + clock = _Clock() + fake_ledger = _FakeReputationLedger() + fake_gate_manager = _FakeGateManager() + append_calls = [] + + monkeypatch.setattr(main.time, "time", clock.time) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr(mesh_reputation_mod, "gate_manager", fake_gate_manager, raising=False) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + monkeypatch.setattr( + mesh_hashchain_mod.gate_store, + "append", + lambda gate_id, event: append_calls.append({"gate_id": gate_id, "event": event}) or event, + ) + monkeypatch.setattr( + mesh_hashchain_mod.infonet, + "validate_and_set_sequence", + lambda node_id, sequence: (True, "ok"), + ) + main._gate_post_cooldown.clear() + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + first = await ac.post( + "/api/mesh/gate/infonet/message", + json={ + "sender_id": "!sender", + "epoch": 3, + "ciphertext": "opaque-ciphertext", + "nonce": "nonce-3", + "sender_ref": "persona-ops-1", + "format": "mls1", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 9, + "protocol_version": "1", + }, + ) + clock.current += 12 + second = await ac.post( + "/api/mesh/gate/infonet/message", + json={ + "sender_id": "!sender", + "epoch": 3, + "ciphertext": "opaque-ciphertext-2", + "nonce": "nonce-4", + "sender_ref": "persona-ops-1", + "format": "mls1", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 10, + "protocol_version": "1", + }, + ) + return first.json(), second.json() + + first_result, second_result = asyncio.run(_run()) + + assert first_result["ok"] is True + assert second_result == { + "ok": False, + "detail": "Gate post cooldown: wait 18s before posting again.", + } + assert fake_gate_manager.recorded == ["infonet"] + assert len(append_calls) == 1 + + +def test_infonet_status_reports_lane_tier_and_policy(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services import wormhole_supervisor + + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.get("/api/mesh/infonet/status") + return response.json() + + result = asyncio.run(_run()) + + assert result["private_lane_tier"] == "private_transitional" + assert result["private_lane_policy"]["gate_actions"]["post_message"] == "private_transitional" + assert result["private_lane_policy"]["gate_chat"]["content_private"] is True + assert ( + result["private_lane_policy"]["gate_chat"]["storage_model"] + == "private_gate_store_encrypted_envelope" + ) + assert result["private_lane_policy"]["dm_lane"]["public_transports_excluded"] is True + assert result["private_lane_policy"]["reserved_for_private_strong"] == [] + + +def test_wormhole_status_reports_transport_tier(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + + monkeypatch.setattr(main, "_debug_mode_enabled", lambda: True) + monkeypatch.setattr( + main, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": False, + "transport": "direct", + }, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.get("/api/wormhole/status") + return response.json() + + result = asyncio.run(_run()) + + assert result["transport_tier"] == "private_transitional" + + +def test_wormhole_status_reports_private_strong_when_arti_ready(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + + monkeypatch.setattr(main, "_debug_mode_enabled", lambda: True) + monkeypatch.setattr( + main, + "get_wormhole_state", + lambda: { + "configured": True, + "ready": True, + "arti_ready": True, + "rns_ready": True, + "transport": "tor_arti", + }, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.get("/api/wormhole/status") + return response.json() + + result = asyncio.run(_run()) + + assert result["transport_tier"] == "private_strong" + + +def test_rns_status_reports_lane_tier_and_policy(monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services import wormhole_supervisor + from services.mesh import mesh_rns + + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": True}, + ) + monkeypatch.setattr( + mesh_rns, + "rns_bridge", + SimpleNamespace(status=lambda: {"enabled": True, "ready": True, "configured_peers": 1, "active_peers": 1}), + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.get("/api/mesh/rns/status") + return response.json() + + result = asyncio.run(_run()) + + assert result["private_lane_tier"] == "private_strong" + assert result["private_lane_policy"]["gate_chat"]["trust_tier"] == "private_transitional" + + +def test_scoped_gate_token_cannot_access_dm_endpoints(tmp_path, monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.config import get_settings + from services import wormhole_supervisor + from services.mesh import mesh_gate_mls, mesh_secure_storage, mesh_wormhole_persona + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_gate_mls, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_gate_mls, "STATE_FILE", tmp_path / "wormhole_gate_mls.json") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + mesh_gate_mls.reset_gate_mls_state() + mesh_wormhole_persona.bootstrap_wormhole_persona_state(force=True) + mesh_wormhole_persona.create_gate_persona("infonet", label="scribe") + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + monkeypatch.setenv("MESH_SCOPED_TOKENS", '{"gate-only":["gate"]}') + get_settings.cache_clear() + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + gate_response = await ac.post( + "/api/wormhole/gate/message/compose", + json={"gate_id": "infonet", "plaintext": "gate scoped"}, + headers={"X-Admin-Key": "gate-only"}, + ) + dm_response = await ac.post( + "/api/wormhole/dm/compose", + json={"peer_id": "bob", "peer_dh_pub": "deadbeef", "plaintext": "blocked"}, + headers={"X-Admin-Key": "gate-only"}, + ) + return gate_response.json(), dm_response.status_code, dm_response.json() + + try: + gate_result, dm_status, dm_result = asyncio.run(_run()) + finally: + get_settings.cache_clear() + + assert gate_result["ok"] is True + assert dm_status == 403 + assert dm_result == {"detail": "Forbidden — insufficient scope"} diff --git a/backend/tests/mesh/test_mesh_env_security_audit.py b/backend/tests/mesh/test_mesh_env_security_audit.py new file mode 100644 index 00000000..7a66fccd --- /dev/null +++ b/backend/tests/mesh/test_mesh_env_security_audit.py @@ -0,0 +1,178 @@ +"""Tests for security config guardrails in env_check._audit_security_config.""" + +import logging +import os +from unittest.mock import patch + +import pytest + +# Reset pydantic settings cache before importing, so env overrides take effect +os.environ.pop("MESH_DM_TOKEN_PEPPER", None) + +from services.config import get_settings, Settings + + +@pytest.fixture(autouse=True) +def _clear_settings_cache(): + """Bust the lru_cache so each test gets fresh Settings.""" + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +@pytest.fixture(autouse=True) +def _clean_pepper_env(): + """Remove any auto-generated pepper between tests.""" + os.environ.pop("MESH_DM_TOKEN_PEPPER", None) + yield + os.environ.pop("MESH_DM_TOKEN_PEPPER", None) + + +class TestInsecureAdminWarning: + def test_allow_insecure_admin_without_key_logs_critical(self, caplog): + with patch.dict(os.environ, {"ALLOW_INSECURE_ADMIN": "true", "ADMIN_KEY": ""}): + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.CRITICAL): + _audit_security_config(get_settings()) + + assert "ALLOW_INSECURE_ADMIN=true with no ADMIN_KEY" in caplog.text + assert "completely unauthenticated" in caplog.text + + def test_admin_key_present_no_warning(self, caplog): + with patch.dict( + os.environ, {"ALLOW_INSECURE_ADMIN": "true", "ADMIN_KEY": "secret123"} + ): + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.CRITICAL): + _audit_security_config(get_settings()) + + assert "ALLOW_INSECURE_ADMIN=true with no ADMIN_KEY" not in caplog.text + + +class TestSignatureConfigWarnings: + def test_non_strict_logs_warning(self, caplog): + with patch.dict(os.environ, {"MESH_STRICT_SIGNATURES": "false"}): + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.WARNING): + _audit_security_config(get_settings()) + + assert "MESH_STRICT_SIGNATURES=false" in caplog.text + + +class TestTokenPepperAutoGeneration: + def test_empty_pepper_auto_generates(self, caplog): + os.environ.pop("MESH_DM_TOKEN_PEPPER", None) + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.WARNING): + _audit_security_config(get_settings()) + + generated = os.environ.get("MESH_DM_TOKEN_PEPPER", "") + assert len(generated) == 64 # 32 bytes hex + assert "Auto-generated a random pepper" in caplog.text + + def test_existing_pepper_preserved(self, caplog): + os.environ["MESH_DM_TOKEN_PEPPER"] = "my-secret-pepper" + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.WARNING): + _audit_security_config(get_settings()) + + assert os.environ["MESH_DM_TOKEN_PEPPER"] == "my-secret-pepper" + assert "Auto-generated" not in caplog.text + + +class TestPeerSecretWarnings: + def test_missing_peer_secret_only_warns_and_does_not_fail_validation(self, caplog): + with patch.dict( + os.environ, + { + "MESH_RELAY_PEERS": "https://peer.example", + "MESH_PEER_PUSH_SECRET": "", + }, + clear=False, + ): + get_settings.cache_clear() + from services.env_check import validate_env + + with caplog.at_level(logging.WARNING): + result = validate_env(strict=True) + + assert result is True + assert "MESH_PEER_PUSH_SECRET is invalid (empty)" in caplog.text + + def test_security_posture_warnings_include_missing_peer_secret(self): + with patch.dict( + os.environ, + { + "MESH_RELAY_PEERS": "https://peer.example", + "MESH_PEER_PUSH_SECRET": "", + }, + clear=False, + ): + get_settings.cache_clear() + from services.env_check import get_security_posture_warnings + + warnings = get_security_posture_warnings(get_settings()) + + assert any("MESH_PEER_PUSH_SECRET is invalid (empty)" in item for item in warnings) + + def test_placeholder_peer_secret_is_flagged(self, caplog): + with patch.dict( + os.environ, + { + "MESH_RELAY_PEERS": "https://peer.example", + "MESH_PEER_PUSH_SECRET": "change-me", + }, + clear=False, + ): + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.WARNING): + _audit_security_config(get_settings()) + + assert "MESH_PEER_PUSH_SECRET is invalid (placeholder)" in caplog.text + + +class TestCoverTrafficWarnings: + def test_disabled_cover_traffic_logs_warning_when_rns_enabled(self, caplog): + with patch.dict( + os.environ, + { + "MESH_RNS_ENABLED": "true", + "MESH_RNS_COVER_INTERVAL_S": "0", + }, + clear=False, + ): + get_settings.cache_clear() + from services.env_check import _audit_security_config + + with caplog.at_level(logging.WARNING): + _audit_security_config(get_settings()) + + assert "MESH_RNS_COVER_INTERVAL_S<=0 disables background RNS cover traffic" in caplog.text + + def test_security_posture_warnings_include_disabled_cover_traffic(self): + with patch.dict( + os.environ, + { + "MESH_RNS_ENABLED": "true", + "MESH_RNS_COVER_INTERVAL_S": "0", + }, + clear=False, + ): + get_settings.cache_clear() + from services.env_check import get_security_posture_warnings + + warnings = get_security_posture_warnings(get_settings()) + + assert any("MESH_RNS_COVER_INTERVAL_S<=0" in item for item in warnings) diff --git a/backend/tests/mesh/test_mesh_gate_catalog.py b/backend/tests/mesh/test_mesh_gate_catalog.py new file mode 100644 index 00000000..67f546e2 --- /dev/null +++ b/backend/tests/mesh/test_mesh_gate_catalog.py @@ -0,0 +1,177 @@ +import asyncio +import json + +from starlette.requests import Request +from httpx import ASGITransport, AsyncClient + + +def _json_request(path: str, body: dict) -> Request: + raw = json.dumps(body).encode("utf-8") + sent = {"value": False} + + async def receive(): + if sent["value"]: + return {"type": "http.request", "body": b"", "more_body": False} + sent["value"] = True + return {"type": "http.request", "body": raw, "more_body": False} + + return Request( + { + "type": "http", + "headers": [(b"content-type", b"application/json")], + "client": ("test", 12345), + "method": "POST", + "path": path, + }, + receive, + ) + + +def test_fixed_launch_gate_catalog_contains_private_rooms(): + from services.mesh.mesh_reputation import gate_manager + + gate_ids = [gate["gate_id"] for gate in gate_manager.list_gates()] + + assert "infonet" in gate_ids + assert "finance" in gate_ids + assert "prediction-markets" in gate_ids + assert "cryptography" in gate_ids + assert "opsec-lab" in gate_ids + assert "public-square" not in gate_ids + + +def test_fixed_launch_gate_catalog_exposes_descriptions_and_welcome_text(): + from services.mesh.mesh_reputation import gate_manager + + finance = next(g for g in gate_manager.list_gates() if g["gate_id"] == "finance") + + assert finance["fixed"] is True + assert "Macro" in finance["description"] + assert "WELCOME TO FINANCE" in finance["welcome"] + + +def test_gate_create_endpoint_is_disabled_for_fixed_launch_catalog(): + import main + + response = asyncio.run( + main.gate_create( + _json_request( + "/api/mesh/gate/create", + { + "creator_id": "!sb_test", + "gate_id": "new-gate", + "display_name": "New Gate", + "rules": {"min_overall_rep": 0}, + }, + ) + ) + ) + + assert response["ok"] is False + assert "fixed private launch catalog" in response["detail"] + + +def test_infonet_messages_returns_seed_notice_for_empty_fixed_gate(monkeypatch): + import main + from services.mesh import mesh_hashchain + + monkeypatch.setattr(mesh_hashchain.infonet, "get_messages", lambda **kwargs: []) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + + response = asyncio.run( + main.infonet_messages( + Request( + { + "type": "http", + "headers": [(b"x-admin-key", b"test-admin")], + "client": ("test", 12345), + "method": "GET", + "path": "/api/mesh/infonet/messages", + } + ), + gate="finance", + limit=20, + offset=0, + ) + ) + + assert response["count"] == 1 + assert response["messages"][0]["system_seed"] is True + assert response["messages"][0]["gate"] == "finance" + assert "WELCOME TO FINANCE" in response["messages"][0]["message"] + + +def test_gate_scoped_vote_rejects_unknown_gate(monkeypatch): + import main + from services import wormhole_supervisor + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "ok")) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True}, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/vote", + json={ + "voter_id": "!sb_voter", + "target_id": "!sb_target", + "vote": 1, + "gate": "nonexistent-gate", + "voter_pubkey": "pub", + "public_key_algo": "Ed25519", + "voter_sig": "sig", + "sequence": 1, + "protocol_version": "1", + }, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is False + assert "does not exist" in result["detail"] + + +def test_gate_scoped_vote_requires_voter_gate_access(monkeypatch): + import main + from services import wormhole_supervisor + from services.mesh.mesh_reputation import gate_manager + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "ok")) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True}, + ) + monkeypatch.setattr( + gate_manager, + "can_enter", + lambda voter_id, gate_id: (False, "Need 10 overall rep (you have 0)"), + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/mesh/vote", + json={ + "voter_id": "!sb_voter", + "target_id": "!sb_target", + "vote": -1, + "gate": "finance", + "voter_pubkey": "pub", + "public_key_algo": "Ed25519", + "voter_sig": "sig", + "sequence": 1, + "protocol_version": "1", + }, + ) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is False + assert "Gate vote denied" in result["detail"] diff --git a/backend/tests/mesh/test_mesh_gate_mls.py b/backend/tests/mesh/test_mesh_gate_mls.py new file mode 100644 index 00000000..e91e35cf --- /dev/null +++ b/backend/tests/mesh/test_mesh_gate_mls.py @@ -0,0 +1,667 @@ +import asyncio +import base64 +import json + + +def _embedded_gate_event_wire_size(gate_mls_mod, persona_id: str, gate_id: str, plaintext: str) -> int: + from services.mesh.mesh_hashchain import build_gate_wire_ref + from services.mesh.mesh_rns import RNSMessage + + binding = gate_mls_mod._sync_binding(gate_id) + member = binding.members[persona_id] + proof = { + "proof_version": "embedded-proof-v1", + "node_id": "!sb_embeddedproof", + "public_key": "A" * 44, + "public_key_algo": "Ed25519", + "sequence": 7, + "protocol_version": "infonet/2", + "content_hash": "b" * 64, + "transport_hash": "c" * 64, + "signature": "d" * 128, + } + plaintext_with_proof = json.dumps( + { + "m": plaintext, + "e": int(binding.epoch), + "proof": proof, + }, + separators=(",", ":"), + ensure_ascii=False, + ) + ciphertext = gate_mls_mod._privacy_client().encrypt_group_message( + member.group_handle, + plaintext_with_proof.encode("utf-8"), + ) + padded = gate_mls_mod._pad_ciphertext_raw(ciphertext) + event = { + "gate_contract_version": "gate-v2-embedded-origin-v1", + "event_type": "gate_message", + "timestamp": 1710000000, + "event_id": "e" * 64, + "payload": { + "ciphertext": gate_mls_mod._b64(padded), + "format": gate_mls_mod.MLS_GATE_FORMAT, + "nonce": "n" * 16, + "sender_ref": "s" * 16, + "epoch": int(binding.epoch), + }, + } + event["payload"]["gate_ref"] = build_gate_wire_ref(gate_id, event) + return len( + RNSMessage( + msg_type="gate_event", + body={"event": event}, + meta={"message_id": "mid", "dandelion": {"phase": "stem", "hops": 0, "max_hops": 3}}, + ).encode() + ) + + +def _fresh_gate_state(tmp_path, monkeypatch): + from services import wormhole_supervisor + from services.mesh import mesh_gate_mls, mesh_secure_storage, mesh_wormhole_persona + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_gate_mls, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_gate_mls, "STATE_FILE", tmp_path / "wormhole_gate_mls.json") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + mesh_gate_mls.reset_gate_mls_state() + return mesh_gate_mls, mesh_wormhole_persona + + +def test_gate_message_schema_accepts_mls1_format(): + from services.mesh.mesh_protocol import normalize_payload + from services.mesh.mesh_schema import validate_event_payload + + payload = normalize_payload( + "gate_message", + { + "gate": "infonet", + "epoch": 1, + "ciphertext": "ZmFrZQ==", + "nonce": "bWxzMS1lbnZlbG9wZQ==", + "sender_ref": "persona-1", + "format": "mls1", + }, + ) + + assert validate_event_payload("gate_message", payload) == (True, "ok") + + +def test_compose_and_decrypt_gate_message_round_trip_via_mls(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.create_gate_persona("finance", label="scribe") + + composed = gate_mls_mod.compose_encrypted_gate_message("finance", "hello mls gate") + decrypted = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id="finance", + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + nonce=str(composed["nonce"]), + sender_ref=str(composed["sender_ref"]), + ) + + assert composed["ok"] is True + assert composed["format"] == "mls1" + assert composed["ciphertext"] != "hello mls gate" + assert decrypted == { + "ok": True, + "gate_id": "finance", + "epoch": 1, + "plaintext": "hello mls gate", + "identity_scope": "persona", + } + + +def test_anonymous_gate_session_can_compose_and_decrypt_round_trip(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.enter_gate_anonymously("finance", rotate=True) + + status = gate_mls_mod.get_local_gate_key_status("finance") + composed = gate_mls_mod.compose_encrypted_gate_message("finance", "hello from anonymous gate") + decrypted = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id="finance", + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + nonce=str(composed["nonce"]), + sender_ref=str(composed["sender_ref"]), + ) + + assert status["ok"] is True + assert status["identity_scope"] == "anonymous" + assert status["has_local_access"] is True + assert composed["ok"] is True + assert composed["identity_scope"] == "anonymous" + assert decrypted == { + "ok": True, + "gate_id": "finance", + "epoch": 1, + "plaintext": "hello from anonymous gate", + "identity_scope": "anonymous", + } + + +def test_self_echo_decrypt_uses_local_plaintext_cache_fast_path(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.create_gate_persona("finance", label="scribe") + composed = gate_mls_mod.compose_encrypted_gate_message("finance", "cache hit") + + def fail_sync(_gate_id: str): + raise AssertionError("self-echo cache should bypass MLS sync/decrypt") + + monkeypatch.setattr(gate_mls_mod, "_sync_binding", fail_sync) + + decrypted = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id="finance", + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + nonce=str(composed["nonce"]), + sender_ref=str(composed["sender_ref"]), + ) + + assert decrypted == { + "ok": True, + "gate_id": "finance", + "epoch": 1, + "plaintext": "cache hit", + "identity_scope": "persona", + } + + +def test_verifier_open_does_not_require_active_gate_persona(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "finance" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + second = persona_mod.create_gate_persona(gate_id, label="second") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + composed = gate_mls_mod.compose_encrypted_gate_message(gate_id, "verifier open") + assert composed["ok"] is True + + persona_mod.enter_gate_anonymously(gate_id, rotate=True) + + opened = gate_mls_mod.open_gate_ciphertext_for_verifier( + gate_id=gate_id, + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + format=str(composed["format"]), + ) + + assert opened["ok"] is True + assert opened["plaintext"] == "verifier open" + assert opened["identity_scope"] == "verifier" + assert opened["opened_by_persona_id"] in { + first["identity"]["persona_id"], + second["identity"]["persona_id"], + } + + +def test_verifier_open_does_not_use_self_echo_cache(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "finance" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + second = persona_mod.create_gate_persona(gate_id, label="second") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + composed = gate_mls_mod.compose_encrypted_gate_message(gate_id, "no cache authority") + assert composed["ok"] is True + + monkeypatch.setattr( + gate_mls_mod, + "_peek_cached_plaintext", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("verifier must not peek cache")), + ) + monkeypatch.setattr( + gate_mls_mod, + "_consume_cached_plaintext", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("verifier must not consume cache")), + ) + monkeypatch.setattr(gate_mls_mod, "_active_gate_persona", lambda *_args, **_kwargs: None) + + opened = gate_mls_mod.open_gate_ciphertext_for_verifier( + gate_id=gate_id, + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + format=str(composed["format"]), + ) + + assert opened == { + "ok": True, + "gate_id": gate_id, + "epoch": 1, + "plaintext": "no cache authority", + "opened_by_persona_id": second["identity"]["persona_id"], + "identity_scope": "verifier", + } + + +def test_removed_member_cannot_decrypt_new_messages(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "opsec-lab" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + second = persona_mod.create_gate_persona(gate_id, label="second") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + before_removal = gate_mls_mod.compose_encrypted_gate_message(gate_id, "before removal") + + persona_mod.activate_gate_persona(gate_id, second["identity"]["persona_id"]) + readable_before = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(before_removal["epoch"]), + ciphertext=str(before_removal["ciphertext"]), + nonce=str(before_removal["nonce"]), + sender_ref=str(before_removal["sender_ref"]), + ) + + persona_mod.retire_gate_persona(gate_id, second["identity"]["persona_id"]) + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + after_removal = gate_mls_mod.compose_encrypted_gate_message(gate_id, "after removal") + + persona_mod.enter_gate_anonymously(gate_id, rotate=True) + blocked_after = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(after_removal["epoch"]), + ciphertext=str(after_removal["ciphertext"]), + nonce=str(after_removal["nonce"]), + sender_ref=str(after_removal["sender_ref"]), + ) + + assert readable_before["ok"] is True + assert readable_before["plaintext"] == "before removal" + assert blocked_after == { + "ok": True, + "gate_id": gate_id, + "epoch": int(after_removal["epoch"]), + "plaintext": "after removal", + "identity_scope": "anonymous", + } + + +def test_gate_mls_state_survives_simulated_restart(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "infonet" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + second = persona_mod.create_gate_persona(gate_id, label="second") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + initial = gate_mls_mod.compose_encrypted_gate_message(gate_id, "before restart") + + gate_mls_mod.reset_gate_mls_state() + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + after_restart = gate_mls_mod.compose_encrypted_gate_message(gate_id, "after restart") + + persona_mod.activate_gate_persona(gate_id, second["identity"]["persona_id"]) + decrypted = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(after_restart["epoch"]), + ciphertext=str(after_restart["ciphertext"]), + nonce=str(after_restart["nonce"]), + sender_ref=str(after_restart["sender_ref"]), + ) + + assert initial["ok"] is True + assert after_restart["ok"] is True + assert after_restart["epoch"] == initial["epoch"] + assert decrypted["ok"] is True + assert decrypted["plaintext"] == "after restart" + + +def test_pre_restart_gate_message_fails_to_decrypt_after_reset(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "restart-blackout" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + second = persona_mod.create_gate_persona(gate_id, label="second") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + before_reset = gate_mls_mod.compose_encrypted_gate_message(gate_id, "before reset") + assert before_reset["ok"] is True + + persona_mod.activate_gate_persona(gate_id, second["identity"]["persona_id"]) + readable_before = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(before_reset["epoch"]), + ciphertext=str(before_reset["ciphertext"]), + nonce=str(before_reset["nonce"]), + sender_ref=str(before_reset["sender_ref"]), + ) + assert readable_before["ok"] is True + assert readable_before["plaintext"] == "before reset" + + gate_mls_mod.reset_gate_mls_state() + + persona_mod.activate_gate_persona(gate_id, second["identity"]["persona_id"]) + blocked_after = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(before_reset["epoch"]), + ciphertext=str(before_reset["ciphertext"]), + nonce=str(before_reset["nonce"]), + sender_ref=str(before_reset["sender_ref"]), + ) + + assert blocked_after == { + "ok": False, + "detail": "gate_mls_decrypt_failed", + } + + +def test_embedded_proof_budget_exceeds_rns_limit_before_6144_bucket_for_large_messages(tmp_path, monkeypatch): + from services.config import get_settings + + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "budget-gate" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + persona_id = first["identity"]["persona_id"] + persona_mod.activate_gate_persona(gate_id, persona_id) + + medium_wire = _embedded_gate_event_wire_size(gate_mls_mod, persona_id, gate_id, "x" * 1000) + large_wire = _embedded_gate_event_wire_size(gate_mls_mod, persona_id, gate_id, "x" * 2000) + + assert medium_wire < get_settings().MESH_RNS_MAX_PAYLOAD + assert large_wire > get_settings().MESH_RNS_MAX_PAYLOAD + + +def test_sync_binding_skips_persist_when_membership_is_unchanged(tmp_path, monkeypatch): + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "quiet-room" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="first") + second = persona_mod.create_gate_persona(gate_id, label="second") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + composed = gate_mls_mod.compose_encrypted_gate_message(gate_id, "steady state") + + persist_calls = [] + original_persist = gate_mls_mod._persist_binding + + def track_persist(binding): + persist_calls.append(binding.gate_id) + return original_persist(binding) + + monkeypatch.setattr(gate_mls_mod, "_persist_binding", track_persist) + persona_mod.activate_gate_persona(gate_id, second["identity"]["persona_id"]) + + decrypted = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + nonce=str(composed["nonce"]), + sender_ref=str(composed["sender_ref"]), + ) + + assert decrypted["ok"] is True + assert decrypted["plaintext"] == "steady state" + assert persist_calls == [] + + +def test_tampered_binding_is_rejected_on_sync(tmp_path, monkeypatch, caplog): + from services.mesh.mesh_secure_storage import read_domain_json, write_domain_json + import logging + + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "cryptography" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona = persona_mod.create_gate_persona(gate_id, label="scribe") + composed = gate_mls_mod.compose_encrypted_gate_message(gate_id, "tamper target") + assert composed["ok"] is True + + stored = read_domain_json( + gate_mls_mod.STATE_DOMAIN, + gate_mls_mod.STATE_FILENAME, + gate_mls_mod._default_binding_store, + ) + persona_id = persona["identity"]["persona_id"] + stored["gates"][gate_id]["members"][persona_id]["binding_signature"] = "00" * 64 + write_domain_json(gate_mls_mod.STATE_DOMAIN, gate_mls_mod.STATE_FILENAME, stored) + + gate_mls_mod.reset_gate_mls_state() + with caplog.at_level(logging.WARNING): + retry = gate_mls_mod.compose_encrypted_gate_message(gate_id, "should rebuild") + + assert retry["ok"] is True + assert "corrupted binding for gate#" in caplog.text.lower() + assert "member persona#" in caplog.text.lower() + + +def test_mls_compose_refuses_public_degraded_transport(tmp_path, monkeypatch): + from services import wormhole_supervisor + + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.create_gate_persona("finance", label="scribe") + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "public_degraded") + + result = gate_mls_mod.compose_encrypted_gate_message("finance", "should fail closed") + + assert result == { + "ok": False, + "detail": "MLS gate compose requires PRIVATE transport tier", + } + + +def test_compose_endpoint_can_use_mls_without_changing_gate_post_envelope(tmp_path, monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.mesh import mesh_hashchain, mesh_reputation + from services import wormhole_supervisor + + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.create_gate_persona("infonet", label="scribe") + monkeypatch.setattr(main, "_debug_mode_enabled", lambda: True) + + class _Ledger: + def __init__(self): + self.registered = [] + + def register_node(self, *args): + self.registered.append(args) + + class _GateManager: + def __init__(self): + self.recorded = [] + self.enter_checks = [] + + def can_enter(self, sender_id, gate_id): + self.enter_checks.append((sender_id, gate_id)) + return True, "ok" + + def record_message(self, gate_id): + self.recorded.append(gate_id) + + fake_ledger = _Ledger() + fake_gate_manager = _GateManager() + append_calls = [] + + def fake_append(gate_id, event): + append_calls.append({"gate_id": gate_id, "event": event}) + return event + + admin_headers = {"X-Admin-Key": main._current_admin_key()} + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + monkeypatch.setattr(mesh_reputation, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr(mesh_reputation, "gate_manager", fake_gate_manager, raising=False) + monkeypatch.setattr(mesh_hashchain.gate_store, "append", fake_append) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + compose_response = await ac.post( + "/api/wormhole/gate/message/compose", + json={"gate_id": "infonet", "plaintext": "field report"}, + headers=admin_headers, + ) + composed = compose_response.json() + send_response = await ac.post( + "/api/wormhole/gate/message/post", + json={"gate_id": "infonet", "plaintext": "field report"}, + headers=admin_headers, + ) + decrypt_response = await ac.post( + "/api/wormhole/gate/message/decrypt", + json={ + "gate_id": "infonet", + "epoch": composed["epoch"], + "ciphertext": composed["ciphertext"], + "nonce": composed["nonce"], + "sender_ref": composed["sender_ref"], + "format": composed["format"], + }, + headers=admin_headers, + ) + return composed, send_response.json(), decrypt_response.json() + + try: + composed, sent, decrypted = asyncio.run(_run()) + finally: + gate_mls_mod.reset_gate_mls_state() + + assert composed["ok"] is True + assert composed["format"] == "mls1" + assert len(base64.b64decode(composed["nonce"])) == 12 + assert sent["ok"] is True + assert sent["detail"] == "Message posted to gate 'infonet'" + assert sent["gate_id"] == "infonet" + assert sent["event_id"] == append_calls[0]["event"]["event_id"] + assert decrypted["ok"] is True + assert decrypted["plaintext"] == "field report" + assert fake_gate_manager.enter_checks == [(append_calls[0]["event"]["node_id"], "infonet")] + assert fake_gate_manager.recorded == ["infonet"] + assert fake_ledger.registered == [ + ( + append_calls[0]["event"]["node_id"], + append_calls[0]["event"]["public_key"], + append_calls[0]["event"]["public_key_algo"], + ) + ] + assert append_calls[0]["gate_id"] == "infonet" + assert append_calls[0]["event"]["payload"]["gate"] == "infonet" + assert append_calls[0]["event"]["payload"]["format"] == "mls1" + assert append_calls[0]["event"]["payload"]["ciphertext"] + assert append_calls[0]["event"]["payload"]["nonce"] + assert append_calls[0]["event"]["payload"]["sender_ref"] + + +def test_receive_only_mls_decrypt_locks_gate_format(tmp_path, monkeypatch): + from services.mesh.mesh_secure_storage import read_domain_json, write_domain_json + + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "receive-only-lab" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona(gate_id, label="sender") + second = persona_mod.create_gate_persona(gate_id, label="receiver") + + persona_mod.activate_gate_persona(gate_id, first["identity"]["persona_id"]) + composed = gate_mls_mod.compose_encrypted_gate_message(gate_id, "receiver should lock gate") + + stored = read_domain_json( + gate_mls_mod.STATE_DOMAIN, + gate_mls_mod.STATE_FILENAME, + gate_mls_mod._default_binding_store, + ) + stored.setdefault("gate_format_locks", {}).pop(gate_id, None) + write_domain_json(gate_mls_mod.STATE_DOMAIN, gate_mls_mod.STATE_FILENAME, stored) + + assert gate_mls_mod.is_gate_locked_to_mls(gate_id) is True + + persona_mod.activate_gate_persona(gate_id, second["identity"]["persona_id"]) + decrypted = gate_mls_mod.decrypt_gate_message_for_local_identity( + gate_id=gate_id, + epoch=int(composed["epoch"]), + ciphertext=str(composed["ciphertext"]), + nonce=str(composed["nonce"]), + sender_ref=str(composed["sender_ref"]), + ) + + assert decrypted["ok"] is True + assert decrypted["plaintext"] == "receiver should lock gate" + assert gate_mls_mod.is_gate_locked_to_mls(gate_id) is True + + +def test_mls_locked_gate_rejects_legacy_g1_decrypt(tmp_path, monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services import wormhole_supervisor + + gate_mls_mod, persona_mod = _fresh_gate_state(tmp_path, monkeypatch) + gate_id = "lockout-lab" + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.create_gate_persona(gate_id, label="scribe") + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + + composed = gate_mls_mod.compose_encrypted_gate_message(gate_id, "mls only") + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post( + "/api/wormhole/gate/message/decrypt", + json={ + "gate_id": gate_id, + "epoch": composed["epoch"], + "ciphertext": composed["ciphertext"], + "nonce": composed["nonce"], + "sender_ref": composed["sender_ref"], + "format": "g1", + }, + headers={"X-Admin-Key": main._current_admin_key()}, + ) + return response.json() + + try: + result = asyncio.run(_run()) + finally: + gate_mls_mod.reset_gate_mls_state() + + assert composed["ok"] is True + assert gate_mls_mod.is_gate_locked_to_mls(gate_id) is True + assert result == { + "ok": False, + "detail": "gate is locked to MLS format", + "gate_id": gate_id, + "required_format": "mls1", + "current_format": "g1", + } diff --git a/backend/tests/mesh/test_mesh_hashchain_sequence.py b/backend/tests/mesh/test_mesh_hashchain_sequence.py new file mode 100644 index 00000000..d0606da9 --- /dev/null +++ b/backend/tests/mesh/test_mesh_hashchain_sequence.py @@ -0,0 +1,122 @@ +import pytest +import base64 + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from services.mesh import mesh_crypto, mesh_hashchain, mesh_protocol + + +def _signed_event_fields(event_type: str, payload: dict, sequence: int, private_key=None): + priv = private_key or ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + public_key = base64.b64encode(pub).decode("utf-8") + node_id = mesh_crypto.derive_node_id(public_key) + normalized = mesh_protocol.normalize_payload(event_type, payload) + sig_payload = mesh_crypto.build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=normalized, + ) + signature = priv.sign(sig_payload.encode("utf-8")).hex() + return { + "node_id": node_id, + "payload": normalized, + "signature": signature, + "public_key": public_key, + "public_key_algo": "Ed25519", + "protocol_version": mesh_protocol.PROTOCOL_VERSION, + "private_key": priv, + } + + +def test_infonet_sequence_enforced(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + evt1_fields = _signed_event_fields( + "message", + {"message": "hello", "destination": "broadcast"}, + 1, + ) + + evt = inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=evt1_fields["payload"], + signature=evt1_fields["signature"], + sequence=1, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + assert evt["payload"]["channel"] == "LongFast" + assert evt["payload"]["priority"] == "normal" + + replay_fields = _signed_event_fields( + "message", + {"message": "replay", "destination": "broadcast", "channel": "LongFast"}, + 1, + ) + with pytest.raises(ValueError): + inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=replay_fields["payload"], + signature=replay_fields["signature"], + sequence=1, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + + out_of_order_fields = _signed_event_fields( + "message", + {"message": "out-of-order", "destination": "broadcast", "channel": "LongFast"}, + 1, + ) + with pytest.raises(ValueError): + inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=out_of_order_fields["payload"], + signature=out_of_order_fields["signature"], + sequence=1, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + + evt2_fields = _signed_event_fields( + "message", + {"message": "next", "destination": "broadcast", "channel": "LongFast"}, + 2, + private_key=evt1_fields["private_key"], + ) + inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=evt2_fields["payload"], + signature=evt2_fields["signature"], + sequence=2, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + + with pytest.raises(ValueError): + inf.append( + event_type="not_valid", + node_id=evt1_fields["node_id"], + payload={"message": "nope", "destination": "broadcast"}, + signature=evt1_fields["signature"], + sequence=1, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) diff --git a/backend/tests/mesh/test_mesh_ibf.py b/backend/tests/mesh/test_mesh_ibf.py new file mode 100644 index 00000000..3c7e8fb5 --- /dev/null +++ b/backend/tests/mesh/test_mesh_ibf.py @@ -0,0 +1,38 @@ +import hashlib + +from services.mesh.mesh_ibf import IBLT, build_iblt + + +def _key(seed: str) -> bytes: + return hashlib.sha256(seed.encode("utf-8")).digest() + + +def test_iblt_reconcile_diff() -> None: + keys_a = [_key(f"a{i}") for i in range(20)] + keys_b = [_key(f"a{i}") for i in range(12)] + [_key(f"b{i}") for i in range(6)] + + iblt_a = build_iblt(keys_a, size=64) + iblt_b = build_iblt(keys_b, size=64) + + diff = iblt_a.subtract(iblt_b) + ok, plus, minus = diff.decode() + assert ok + + plus_set = {p for p in plus} + minus_set = {m for m in minus} + + assert plus_set == set(keys_a) - set(keys_b) + assert minus_set == set(keys_b) - set(keys_a) + + +def test_iblt_compact_roundtrip() -> None: + keys = [_key(f"x{i}") for i in range(15)] + iblt = build_iblt(keys, size=64) + packed = iblt.to_compact_dict() + iblt2 = IBLT.from_compact_dict(packed) + + diff = iblt.subtract(iblt2) + ok, plus, minus = diff.decode() + assert ok + assert plus == [] + assert minus == [] diff --git a/backend/tests/mesh/test_mesh_infonet_ingest.py b/backend/tests/mesh/test_mesh_infonet_ingest.py new file mode 100644 index 00000000..a0509865 --- /dev/null +++ b/backend/tests/mesh/test_mesh_infonet_ingest.py @@ -0,0 +1,263 @@ +import base64 +import json +import pytest + +from cryptography.hazmat.primitives.asymmetric import ed25519 +from cryptography.hazmat.primitives import serialization + +from services.mesh import mesh_hashchain, mesh_crypto, mesh_protocol, mesh_schema + + +def test_infonet_ingest_accepts_valid_event(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + + priv = ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + pub_b64 = base64.b64encode(pub).decode("utf-8") + node_id = mesh_crypto.derive_node_id(pub_b64) + + payload = mesh_protocol.normalize_payload( + "message", + {"message": "hello", "destination": "broadcast", "channel": "LongFast", "priority": "normal", "ephemeral": False}, + ) + sig_payload = mesh_crypto.build_signature_payload( + event_type="message", node_id=node_id, sequence=1, payload=payload + ) + signature = priv.sign(sig_payload.encode("utf-8")).hex() + + evt = mesh_hashchain.ChainEvent( + prev_hash=mesh_hashchain.GENESIS_HASH, + event_type="message", + node_id=node_id, + payload=payload, + sequence=1, + signature=signature, + public_key=pub_b64, + public_key_algo="Ed25519", + protocol_version=mesh_protocol.PROTOCOL_VERSION, + network_id=mesh_protocol.NETWORK_ID, + ) + result = inf.ingest_events([evt.to_dict()]) + + assert result["accepted"] == 1 + assert inf.head_hash == evt.event_id + + +def test_verify_node_binding_rejects_legacy_and_accepts_current_ids(): + priv = ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + pub_b64 = base64.b64encode(pub).decode("utf-8") + + current = mesh_crypto.derive_node_id(pub_b64) + legacy = f"{mesh_crypto.NODE_ID_PREFIX}{current[len(mesh_crypto.NODE_ID_PREFIX):len(mesh_crypto.NODE_ID_PREFIX) + 8]}" + + assert mesh_crypto.verify_node_binding(current, pub_b64) + assert not mesh_crypto.verify_node_binding(legacy, pub_b64) + + +def test_infonet_append_rejects_missing_signature_fields(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + payload = mesh_protocol.normalize_payload( + "message", + {"message": "hello", "destination": "broadcast", "channel": "LongFast", "priority": "normal", "ephemeral": False}, + ) + + try: + inf.append( + event_type="message", + node_id="!sb_test", + payload=payload, + sequence=1, + ) + assert False, "Expected missing signature fields to be rejected" + except ValueError as exc: + assert "signature" in str(exc).lower() + + +def test_infonet_load_fails_closed_on_hash_mismatch(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + priv = ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + pub_b64 = base64.b64encode(pub).decode("utf-8") + node_id = mesh_crypto.derive_node_id(pub_b64) + payload = mesh_protocol.normalize_payload( + "message", + {"message": "hello", "destination": "broadcast", "channel": "LongFast", "priority": "normal", "ephemeral": False}, + ) + sig_payload = mesh_crypto.build_signature_payload( + event_type="message", node_id=node_id, sequence=1, payload=payload + ) + signature = priv.sign(sig_payload.encode("utf-8")).hex() + evt = mesh_hashchain.ChainEvent( + prev_hash=mesh_hashchain.GENESIS_HASH, + event_type="message", + node_id=node_id, + payload=payload, + sequence=1, + signature=signature, + public_key=pub_b64, + public_key_algo="Ed25519", + protocol_version=mesh_protocol.PROTOCOL_VERSION, + network_id=mesh_protocol.NETWORK_ID, + ).to_dict() + evt["event_id"] = "tampered" + mesh_hashchain.CHAIN_FILE.write_text( + json.dumps({"events": [evt], "head_hash": "tampered", "node_sequences": {node_id: 1}}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Hash mismatch on event load"): + mesh_hashchain.Infonet() + + +def test_validate_gate_message_payload_rejects_plaintext_shape(): + payload = mesh_protocol.normalize_payload( + "gate_message", + {"gate": "infonet", "message": "plaintext should fail"}, + ) + + valid, reason = mesh_schema.validate_event_payload("gate_message", payload) + + assert valid is False + assert reason == "epoch must be a positive integer" + + +def test_gate_store_accepts_encrypted_gate_payload(tmp_path, monkeypatch): + priv = ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + pub_b64 = base64.b64encode(pub).decode("utf-8") + node_id = mesh_crypto.derive_node_id(pub_b64) + + payload = mesh_protocol.normalize_payload( + "gate_message", + { + "gate": "infonet", + "epoch": 2, + "ciphertext": "opaque-ciphertext", + "nonce": "nonce-2", + "sender_ref": "persona-ops-1", + "format": "mls1", + }, + ) + sig_payload = mesh_crypto.build_signature_payload( + event_type="gate_message", node_id=node_id, sequence=1, payload=payload + ) + signature = priv.sign(sig_payload.encode("utf-8")).hex() + + store = mesh_hashchain.GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + evt = store.append( + "infonet", + { + "event_id": "evt_gate_cipher", + "event_type": "gate_message", + "node_id": node_id, + "payload": payload, + "timestamp": 1_700_000_000.0, + "sequence": 1, + "signature": signature, + "public_key": pub_b64, + "public_key_algo": "Ed25519", + "protocol_version": mesh_protocol.PROTOCOL_VERSION, + }, + ) + + assert evt["payload"]["gate"] == "infonet" + assert evt["payload"]["ciphertext"] == "opaque-ciphertext" + assert evt["payload"]["epoch"] == 2 + assert evt["payload"]["nonce"] == "nonce-2" + assert evt["payload"]["sender_ref"] == "persona-ops-1" + + +def test_gate_store_rejects_replayed_ciphertext_across_append_and_peer_ingest(tmp_path): + store = mesh_hashchain.GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + gate_id = "infonet" + first = store.append( + gate_id, + { + "event_id": "a" * 64, + "event_type": "gate_message", + "payload": { + "gate": gate_id, + "ciphertext": "opaque-ciphertext", + "format": "mls1", + }, + "timestamp": float(int(mesh_hashchain.time.time() / 60) * 60), + }, + ) + + replayed = store.append( + gate_id, + { + "event_id": "b" * 64, + "event_type": "gate_message", + "payload": { + "gate": gate_id, + "ciphertext": "opaque-ciphertext", + "format": "mls1", + }, + "timestamp": float(int(mesh_hashchain.time.time() / 60) * 60) + 60.0, + }, + ) + peer_result = store.ingest_peer_events( + gate_id, + [ + { + "event_type": "gate_message", + "timestamp": mesh_hashchain.time.time(), + "payload": { + "gate": gate_id, + "ciphertext": "opaque-ciphertext", + "format": "mls1", + }, + } + ], + ) + + assert replayed is first + assert peer_result == {"accepted": 0, "duplicates": 1, "rejected": 0} + assert len(store.get_messages(gate_id, limit=10)) == 1 + + +def test_gate_store_prunes_stale_replay_fingerprints(tmp_path): + store = mesh_hashchain.GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + old_ts = mesh_hashchain.time.time() - mesh_hashchain.GATE_REPLAY_WINDOW_S - 10 + + store.append( + "infonet", + { + "event_id": "c" * 64, + "event_type": "gate_message", + "payload": { + "gate": "infonet", + "ciphertext": "old-cipher", + "format": "mls1", + }, + "timestamp": old_ts, + }, + ) + + assert len(store._replay_index) == 1 + removed = store._prune_replay_index(now=mesh_hashchain.time.time()) + + assert removed == 1 + assert store._replay_index == {} diff --git a/backend/tests/mesh/test_mesh_infonet_sync_support.py b/backend/tests/mesh/test_mesh_infonet_sync_support.py new file mode 100644 index 00000000..b4b140e4 --- /dev/null +++ b/backend/tests/mesh/test_mesh_infonet_sync_support.py @@ -0,0 +1,75 @@ +from services.mesh.mesh_infonet_sync_support import ( + SyncWorkerState, + begin_sync, + eligible_sync_peers, + finish_sync, + should_run_sync, +) +from services.mesh.mesh_peer_store import make_bootstrap_peer_record, make_sync_peer_record + + +def test_eligible_sync_peers_filters_bucket_and_cooldown(): + records = [ + make_bootstrap_peer_record( + peer_url="https://seed.example", + transport="clearnet", + role="seed", + signer_id="bootstrap-a", + now=100, + ), + make_sync_peer_record( + peer_url="https://active.example", + transport="clearnet", + now=100, + ), + make_sync_peer_record( + peer_url="https://cooldown.example", + transport="clearnet", + now=100, + ), + ] + cooled = records[2] + records[2] = type(cooled)(**{**cooled.to_dict(), "cooldown_until": 500}) + + candidates = eligible_sync_peers(records, now=200) + + assert [record.peer_url for record in candidates] == ["https://active.example"] + + +def test_finish_sync_success_updates_schedule(): + state = begin_sync(SyncWorkerState(), peer_url="https://seed.example", now=100) + finished = finish_sync( + state, + ok=True, + peer_url="https://seed.example", + current_head="abc123", + now=110, + interval_s=300, + ) + + assert finished.last_outcome == "ok" + assert finished.last_sync_ok_at == 110 + assert finished.next_sync_due_at == 410 + assert finished.current_head == "abc123" + assert not finished.fork_detected + + +def test_finish_sync_failure_surfaces_fork_without_auto_merging(): + state = begin_sync(SyncWorkerState(), peer_url="https://seed.example", now=100) + finished = finish_sync( + state, + ok=False, + peer_url="https://seed.example", + error="fork detected", + fork_detected=True, + now=120, + failure_backoff_s=45, + ) + + assert finished.last_outcome == "fork" + assert finished.fork_detected is True + assert finished.last_error == "fork detected" + assert finished.consecutive_failures == 1 + assert finished.next_sync_due_at == 165 + assert should_run_sync(finished, now=150) is False + assert should_run_sync(finished, now=165) is True diff --git a/backend/tests/mesh/test_mesh_invariants.py b/backend/tests/mesh/test_mesh_invariants.py new file mode 100644 index 00000000..6aacf9c8 --- /dev/null +++ b/backend/tests/mesh/test_mesh_invariants.py @@ -0,0 +1,163 @@ +import base64 + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from services.mesh import mesh_crypto, mesh_hashchain, mesh_protocol + + +def _signed_event_fields( + event_type: str, + payload: dict, + sequence: int, + *, + private_key: ed25519.Ed25519PrivateKey | None = None, +): + priv = private_key or ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + pub_b64 = base64.b64encode(pub).decode("utf-8") + node_id = mesh_crypto.derive_node_id(pub_b64) + normalized = mesh_protocol.normalize_payload(event_type, payload) + sig_payload = mesh_crypto.build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=normalized, + ) + signature = priv.sign(sig_payload.encode("utf-8")).hex() + return { + "node_id": node_id, + "payload": normalized, + "signature": signature, + "public_key": pub_b64, + "public_key_algo": "Ed25519", + "protocol_version": mesh_protocol.PROTOCOL_VERSION, + } + + +def test_chain_linkage_and_head(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + evt1_fields = _signed_event_fields( + "message", + {"message": "one", "destination": "broadcast", "channel": "LongFast", "priority": "normal", "ephemeral": False}, + 1, + ) + evt1 = inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=evt1_fields["payload"], + signature=evt1_fields["signature"], + sequence=1, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + evt2_fields = _signed_event_fields( + "message", + {"message": "two", "destination": "broadcast", "channel": "LongFast", "priority": "normal", "ephemeral": False}, + 2, + ) + evt2 = inf.append( + event_type="message", + node_id=evt2_fields["node_id"], + payload=evt2_fields["payload"], + signature=evt2_fields["signature"], + sequence=2, + public_key=evt2_fields["public_key"], + public_key_algo=evt2_fields["public_key_algo"], + protocol_version=evt2_fields["protocol_version"], + ) + + assert evt1["prev_hash"] == mesh_hashchain.GENESIS_HASH + assert evt2["prev_hash"] == evt1["event_id"] + assert inf.head_hash == evt2["event_id"] + + +def test_ingest_rejects_non_normalized_payload(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + evt = mesh_hashchain.ChainEvent( + prev_hash=mesh_hashchain.GENESIS_HASH, + event_type="message", + node_id="!sb_test", + payload={"message": "hi", "destination": "broadcast"}, + sequence=1, + signature="deadbeef", + public_key="pub", + public_key_algo="Ed25519", + protocol_version=mesh_protocol.PROTOCOL_VERSION, + network_id=mesh_protocol.NETWORK_ID, + ) + result = inf.ingest_events([evt.to_dict()]) + + assert result["accepted"] == 0 + assert result["rejected"] + assert "normalized" in result["rejected"][0]["reason"].lower() + + +def test_revoked_key_rejects_future_events(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + now = int(mesh_hashchain.time.time()) + revoke_priv = ed25519.Ed25519PrivateKey.generate() + revoked_payload = { + "revoked_public_key": "", + "revoked_public_key_algo": "Ed25519", + "revoked_at": now - 10, + "grace_until": now - 10, + "reason": "compromised", + } + revoke_fields = _signed_event_fields( + "key_revoke", + revoked_payload, + 1, + private_key=revoke_priv, + ) + revoked_payload["revoked_public_key"] = revoke_fields["public_key"] + revoke_fields = _signed_event_fields( + "key_revoke", + revoked_payload, + 1, + private_key=revoke_priv, + ) + inf.append( + event_type="key_revoke", + node_id=revoke_fields["node_id"], + payload=revoked_payload, + signature=revoke_fields["signature"], + sequence=1, + public_key=revoke_fields["public_key"], + public_key_algo=revoke_fields["public_key_algo"], + protocol_version=revoke_fields["protocol_version"], + ) + + msg_fields = _signed_event_fields( + "message", + {"message": "blocked", "destination": "broadcast", "channel": "LongFast", "priority": "normal", "ephemeral": False}, + 2, + private_key=revoke_priv, + ) + try: + inf.append( + event_type="message", + node_id=revoke_fields["node_id"], + payload=msg_fields["payload"], + signature=msg_fields["signature"], + sequence=2, + public_key=revoke_fields["public_key"], + public_key_algo=revoke_fields["public_key_algo"], + protocol_version=revoke_fields["protocol_version"], + ) + assert False, "Expected revocation to block new events" + except ValueError as exc: + assert "revoked" in str(exc).lower() diff --git a/backend/tests/mesh/test_mesh_locator.py b/backend/tests/mesh/test_mesh_locator.py new file mode 100644 index 00000000..e2c4708f --- /dev/null +++ b/backend/tests/mesh/test_mesh_locator.py @@ -0,0 +1,79 @@ +import base64 + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from services.mesh import mesh_crypto, mesh_hashchain, mesh_protocol + + +def _signed_event_fields(event_type: str, payload: dict, sequence: int, private_key=None): + priv = private_key or ed25519.Ed25519PrivateKey.generate() + pub = priv.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + public_key = base64.b64encode(pub).decode("utf-8") + node_id = mesh_crypto.derive_node_id(public_key) + normalized = mesh_protocol.normalize_payload(event_type, payload) + sig_payload = mesh_crypto.build_signature_payload( + event_type=event_type, + node_id=node_id, + sequence=sequence, + payload=normalized, + ) + signature = priv.sign(sig_payload.encode("utf-8")).hex() + return { + "node_id": node_id, + "payload": normalized, + "signature": signature, + "public_key": public_key, + "public_key_algo": "Ed25519", + "protocol_version": mesh_protocol.PROTOCOL_VERSION, + "private_key": priv, + } + + +def test_locator_sync(tmp_path, monkeypatch) -> None: + monkeypatch.setattr(mesh_hashchain, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_hashchain, "CHAIN_FILE", tmp_path / "infonet.json") + + inf = mesh_hashchain.Infonet() + evt1_fields = _signed_event_fields( + "message", + {"message": "hello", "destination": "broadcast"}, + 1, + ) + evt1 = inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=evt1_fields["payload"], + signature=evt1_fields["signature"], + sequence=1, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + evt2_fields = _signed_event_fields( + "message", + {"message": "world", "destination": "broadcast"}, + 2, + private_key=evt1_fields["private_key"], + ) + evt2 = inf.append( + event_type="message", + node_id=evt1_fields["node_id"], + payload=evt2_fields["payload"], + signature=evt2_fields["signature"], + sequence=2, + public_key=evt1_fields["public_key"], + public_key_algo=evt1_fields["public_key_algo"], + protocol_version=evt1_fields["protocol_version"], + ) + + locator = inf.get_locator() + assert locator[0] == evt2["event_id"] + + matched, _start, events = inf.get_events_after_locator([evt1["event_id"]], limit=10) + assert matched == evt1["event_id"] + assert len(events) == 1 + assert events[0]["event_id"] == evt2["event_id"] diff --git a/backend/tests/mesh/test_mesh_merkle.py b/backend/tests/mesh/test_mesh_merkle.py new file mode 100644 index 00000000..9389c0da --- /dev/null +++ b/backend/tests/mesh/test_mesh_merkle.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path + +from services.mesh.mesh_merkle import merkle_root, verify_merkle_proof + + +def test_merkle_fixtures() -> None: + root_dir = Path(__file__).resolve().parents[3] + fixture_path = root_dir / "docs" / "mesh" / "mesh-merkle-fixtures.json" + fixtures = json.loads(fixture_path.read_text(encoding="utf-8")) + + leaves = fixtures["leaves"] + root = fixtures["root"] + assert merkle_root(leaves) == root + + for idx_str, proof in fixtures["proofs"].items(): + idx = int(idx_str) + assert verify_merkle_proof(leaves[idx], idx, proof, root) diff --git a/backend/tests/mesh/test_mesh_node_bootstrap_runtime.py b/backend/tests/mesh/test_mesh_node_bootstrap_runtime.py new file mode 100644 index 00000000..dff982ed --- /dev/null +++ b/backend/tests/mesh/test_mesh_node_bootstrap_runtime.py @@ -0,0 +1,136 @@ +import asyncio +import base64 +import json +from types import SimpleNamespace + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 +from httpx import ASGITransport, AsyncClient + + +def _write_signed_manifest(path, *, private_key): + from services.mesh.mesh_bootstrap_manifest import BOOTSTRAP_MANIFEST_VERSION + from services.mesh.mesh_crypto import canonical_json + + payload = { + "version": BOOTSTRAP_MANIFEST_VERSION, + "issued_at": 1_700_000_000, + "valid_until": 1_800_000_000, + "signer_id": "bootstrap-a", + "peers": [ + { + "peer_url": "https://seed.example", + "transport": "clearnet", + "role": "seed", + "label": "Seed A", + } + ], + } + signature = base64.b64encode(private_key.sign(canonical_json(payload).encode("utf-8"))).decode("utf-8") + path.write_text(json.dumps({**payload, "signature": signature}), encoding="utf-8") + + +def test_refresh_node_peer_store_promotes_manifest_peers_to_sync_only(tmp_path, monkeypatch): + import main + from services.config import get_settings + from services.mesh import mesh_bootstrap_manifest as manifest_mod + from services.mesh import mesh_peer_store as peer_store_mod + + manifest_key = ed25519.Ed25519PrivateKey.generate() + manifest_pub = base64.b64encode( + manifest_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + ).decode("utf-8") + manifest_path = tmp_path / "bootstrap.json" + peer_store_path = tmp_path / "peer_store.json" + _write_signed_manifest(manifest_path, private_key=manifest_key) + + monkeypatch.setattr(manifest_mod, "DEFAULT_BOOTSTRAP_MANIFEST_PATH", manifest_path) + monkeypatch.setattr(peer_store_mod, "DEFAULT_PEER_STORE_PATH", peer_store_path) + monkeypatch.setenv("MESH_BOOTSTRAP_SIGNER_PUBLIC_KEY", manifest_pub) + monkeypatch.setenv("MESH_BOOTSTRAP_MANIFEST_PATH", str(manifest_path)) + monkeypatch.setenv("MESH_RELAY_PEERS", "https://operator.example") + get_settings.cache_clear() + + try: + snapshot = main._refresh_node_peer_store(now=1_750_000_000) + store = peer_store_mod.PeerStore(peer_store_path) + store.load() + finally: + get_settings.cache_clear() + + assert snapshot["manifest_loaded"] is True + assert snapshot["bootstrap_peer_count"] == 1 + assert snapshot["sync_peer_count"] == 2 + assert snapshot["push_peer_count"] == 1 + assert [record.peer_url for record in store.records_for_bucket("bootstrap")] == ["https://seed.example"] + assert sorted(record.peer_url for record in store.records_for_bucket("sync")) == [ + "https://operator.example", + "https://seed.example", + ] + assert [record.peer_url for record in store.records_for_bucket("push")] == ["https://operator.example"] + + +def test_verify_peer_push_hmac_requires_allowlisted_peer(monkeypatch): + import hashlib + import hmac + + import main + from services.config import get_settings + from services.mesh.mesh_crypto import _derive_peer_key + + monkeypatch.setenv("MESH_PEER_PUSH_SECRET", "shared-secret") + get_settings.cache_clear() + monkeypatch.setattr(main, "authenticated_push_peer_urls", lambda *args, **kwargs: ["https://good.example"]) + + try: + body = b'{"events":[]}' + peer_url = "https://bad.example" + peer_key = _derive_peer_key("shared-secret", peer_url) + signature = hmac.new(peer_key, body, hashlib.sha256).hexdigest() + request = SimpleNamespace( + headers={"x-peer-url": peer_url, "x-peer-hmac": signature}, + url=SimpleNamespace(scheme="https", netloc="bad.example"), + ) + assert main._verify_peer_push_hmac(request, body) is False + finally: + get_settings.cache_clear() + + +def test_infonet_status_includes_node_runtime_snapshot(monkeypatch): + import main + from services import wormhole_supervisor + + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": False}, + ) + monkeypatch.setattr( + main, + "_node_runtime_snapshot", + lambda: { + "node_mode": "participant", + "node_enabled": True, + "bootstrap": {"sync_peer_count": 2, "push_peer_count": 1}, + "sync_runtime": {"last_outcome": "ok"}, + "push_runtime": {"last_event_id": "evt-1"}, + }, + ) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.get("/api/mesh/infonet/status") + return response.json() + + result = asyncio.run(_run()) + + assert result["node_mode"] == "participant" + assert result["node_enabled"] is True + assert result["bootstrap"]["sync_peer_count"] == 2 + assert result["bootstrap"]["push_peer_count"] == 1 + assert result["sync_runtime"]["last_outcome"] == "ok" + assert result["push_runtime"]["last_event_id"] == "evt-1" diff --git a/backend/tests/mesh/test_mesh_peer_store.py b/backend/tests/mesh/test_mesh_peer_store.py new file mode 100644 index 00000000..67bd72a0 --- /dev/null +++ b/backend/tests/mesh/test_mesh_peer_store.py @@ -0,0 +1,98 @@ +from services.mesh.mesh_peer_store import ( + PeerStore, + make_bootstrap_peer_record, + make_push_peer_record, + make_sync_peer_record, +) + + +def test_peer_store_preserves_provenance_across_buckets(tmp_path): + store = PeerStore(tmp_path / "peer_store.json") + bootstrap = make_bootstrap_peer_record( + peer_url="https://seed.example", + transport="clearnet", + role="seed", + signer_id="bootstrap-a", + now=100, + ) + sync_peer = make_sync_peer_record( + peer_url="https://seed.example", + transport="clearnet", + role="seed", + source="bootstrap_promoted", + signer_id="bootstrap-a", + now=101, + ) + push_peer = make_push_peer_record( + peer_url="https://seed.example", + transport="clearnet", + role="seed", + now=102, + ) + + store.upsert(bootstrap) + store.upsert(sync_peer) + store.upsert(push_peer) + + assert [record.bucket for record in store.records()] == ["bootstrap", "push", "sync"] + assert [record.source for record in store.records_for_bucket("sync")] == ["bootstrap_promoted"] + assert [record.source for record in store.records_for_bucket("push")] == ["operator"] + + +def test_peer_store_save_load_roundtrip(tmp_path): + path = tmp_path / "peer_store.json" + store = PeerStore(path) + store.upsert( + make_bootstrap_peer_record( + peer_url="https://seed.example", + transport="clearnet", + role="seed", + signer_id="bootstrap-a", + now=100, + ) + ) + store.upsert( + make_sync_peer_record( + peer_url="http://alphaexample.onion", + transport="onion", + role="relay", + source="operator", + now=101, + ) + ) + store.save() + + loaded = PeerStore(path) + records = loaded.load() + + assert len(records) == 2 + assert [record.bucket for record in records] == ["bootstrap", "sync"] + assert records[0].signer_id == "bootstrap-a" + assert records[1].peer_url == "http://alphaexample.onion" + + +def test_peer_store_failure_and_success_lifecycle(tmp_path): + store = PeerStore(tmp_path / "peer_store.json") + store.upsert( + make_sync_peer_record( + peer_url="https://sync.example", + transport="clearnet", + now=100, + ) + ) + failed = store.mark_failure( + "https://sync.example", + "sync", + error="timeout", + cooldown_s=30, + now=200, + ) + assert failed.failure_count == 1 + assert failed.cooldown_until == 230 + assert failed.last_error == "timeout" + + recovered = store.mark_sync_success("https://sync.example", now=250) + assert recovered.failure_count == 0 + assert recovered.cooldown_until == 0 + assert recovered.last_error == "" + assert recovered.last_sync_ok_at == 250 diff --git a/backend/tests/mesh/test_mesh_privacy_hardening.py b/backend/tests/mesh/test_mesh_privacy_hardening.py new file mode 100644 index 00000000..be132bf7 --- /dev/null +++ b/backend/tests/mesh/test_mesh_privacy_hardening.py @@ -0,0 +1,1232 @@ +import asyncio +import base64 +import json +import logging +import time +from collections import deque + +from cryptography.hazmat.primitives.asymmetric import ed25519 +from starlette.requests import Request + +import main +from services.mesh.mesh_crypto import build_signature_payload, derive_node_id +from services.mesh.mesh_hashchain import GateMessageStore +from services.mesh import mesh_reputation + + +def _json_request(path: str, body: dict) -> Request: + payload = json.dumps(body).encode("utf-8") + sent = {"value": False} + + async def receive(): + if sent["value"]: + return {"type": "http.request", "body": b"", "more_body": False} + sent["value"] = True + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "headers": [(b"content-type", b"application/json")], + "client": ("test", 12345), + "method": "POST", + "path": path, + }, + receive, + ) + + +def _request(path: str, method: str = "GET") -> Request: + sent = {"value": False} + + async def receive(): + if sent["value"]: + return {"type": "http.request", "body": b"", "more_body": False} + sent["value"] = True + return {"type": "http.request", "body": b"", "more_body": False} + + return Request( + { + "type": "http", + "headers": [], + "client": ("test", 12345), + "method": method, + "path": path, + }, + receive, + ) + + +def _signed_gate_event( + gate_id: str = "finance", + *, + private_key: ed25519.Ed25519PrivateKey | None = None, + sequence: int = 7, + ciphertext: str = "opaque-ciphertext", + nonce: str = "nonce-1", + sender_ref: str = "sender-ref-1", +) -> dict: + private_key = private_key or ed25519.Ed25519PrivateKey.generate() + public_key = base64.b64encode(private_key.public_key().public_bytes_raw()).decode("ascii") + node_id = derive_node_id(public_key) + payload = { + "gate": gate_id, + "ciphertext": ciphertext, + "nonce": nonce, + "sender_ref": sender_ref, + "format": "mls1", + } + signature = private_key.sign( + build_signature_payload( + event_type="gate_message", + node_id=node_id, + sequence=sequence, + payload=payload, + ).encode("utf-8") + ).hex() + return { + "event_type": "gate_message", + "timestamp": float(int(time.time() / 60) * 60), + "node_id": node_id, + "sequence": sequence, + "signature": signature, + "public_key": public_key, + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "payload": payload, + } + + +def _vote_event() -> dict: + return { + "event_id": "vote-1", + "event_type": "vote", + "node_id": "!node-1", + "payload": {"gate": "finance", "vote": 1}, + "timestamp": 100.0, + "sequence": 3, + "signature": "sig", + "public_key": "pub", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + } + + +def _key_rotate_event() -> dict: + return { + "event_id": "rotate-1", + "event_type": "key_rotate", + "node_id": "!node-2", + "payload": { + "old_node_id": "!old-node", + "old_public_key": "old-pub", + "old_public_key_algo": "Ed25519", + "old_signature": "old-sig", + "timestamp": 123, + }, + "timestamp": 101.0, + "sequence": 4, + "signature": "sig", + "public_key": "pub", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + } + + +def _public_gate_message_event() -> dict: + return { + "event_id": "gate-1", + "event_type": "gate_message", + "node_id": "!node-3", + "payload": { + "gate": "finance", + "ciphertext": "opaque", + "epoch": 2, + "nonce": "nonce-1", + "sender_ref": "sender-ref-1", + "format": "mls1", + }, + "timestamp": 102.0, + "sequence": 5, + "signature": "sig", + "public_key": "pub", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + } + + +def test_private_gate_timestamp_is_stably_jittered_backward(monkeypatch): + class _Settings: + MESH_GATE_TIMESTAMP_JITTER_S = 60 + + monkeypatch.setattr(main, "get_settings", lambda: _Settings()) + event = { + "event_id": "gate-event-stable-jitter", + "event_type": "gate_message", + "timestamp": 120.0, + "payload": { + "gate": "finance", + "ciphertext": "opaque", + "format": "mls1", + }, + } + + first = main._strip_gate_identity(event) + second = main._strip_gate_identity(dict(event)) + + assert first["timestamp"] == second["timestamp"] + assert 60.0 <= float(first["timestamp"]) < 120.0 + assert first["public_key"] == "" + assert first["node_id"] == "" + + +def test_gate_identity_redaction_keeps_gate_member_visible_fields(): + event = { + "event_id": "gate-event-visible-fields", + "event_type": "gate_message", + "timestamp": 120.0, + "node_id": "!sb_gate_member", + "sequence": 7, + "signature": "sig", + "public_key": "pub", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "payload": { + "gate": "finance", + "ciphertext": "opaque", + "format": "mls1", + "nonce": "nonce-1", + "sender_ref": "sender-ref-1", + }, + } + + stripped = main._strip_gate_identity(event) + + assert stripped["node_id"] == "!sb_gate_member" + assert stripped["public_key"] == "pub" + assert stripped["public_key_algo"] == "Ed25519" + assert stripped["sequence"] == 7 + assert stripped["signature"] == "sig" + assert stripped["protocol_version"] == "infonet/2" + assert stripped["payload"]["nonce"] == "nonce-1" + assert stripped["payload"]["sender_ref"] == "sender-ref-1" + + +class _FakePublicInfonet: + def __init__(self): + self.head_hash = "head-1" + self.events = [_vote_event(), _key_rotate_event(), _public_gate_message_event()] + + @staticmethod + def _limit_value(limit: int) -> int: + try: + return int(limit) + except Exception: + return int(getattr(limit, "default", 100) or 100) + + def decorate_event(self, evt: dict) -> dict: + return dict(evt) + + def decorate_events(self, events: list[dict]) -> list[dict]: + return [dict(evt) for evt in events] + + def get_event(self, event_id: str): + for evt in self.events: + if evt["event_id"] == event_id: + return dict(evt) + return None + + def get_messages(self, gate_id: str = "", limit: int = 50, offset: int = 0): + resolved_limit = self._limit_value(limit) + return [dict(evt) for evt in self.events[offset : offset + resolved_limit]] + + def get_events_by_node(self, node_id: str, limit: int = 50): + return [dict(evt) for evt in self.events if evt["node_id"] == node_id][:limit] + + def get_events_by_type(self, event_type: str, limit: int = 50, offset: int = 0): + resolved_limit = self._limit_value(limit) + filtered = [dict(evt) for evt in self.events if evt["event_type"] == event_type] + return filtered[offset : offset + resolved_limit] + + def get_events_after(self, after_hash: str, limit: int = 100): + resolved_limit = self._limit_value(limit) + return [dict(evt) for evt in self.events[:resolved_limit]] + + def get_events_after_locator(self, locator: list[str], limit: int = 100): + resolved_limit = self._limit_value(limit) + return self.head_hash, 0, [dict(evt) for evt in self.events[:resolved_limit]] + + def get_merkle_proofs(self, start_index: int, count: int): + return {"root": "merkle-root", "total": len(self.events), "start": start_index, "proofs": []} + + def get_merkle_root(self): + return "merkle-root" + + +def test_gate_store_rejects_unverified_peer_events(tmp_path): + store = GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + + result = store.ingest_peer_events( + "finance", + [ + { + "event_type": "gate_message", + "timestamp": time.time(), + "payload": { + "gate": "finance", + "ciphertext": "opaque-ciphertext", + "format": "mls1", + "nonce": "nonce-1", + "sender_ref": "sender-ref-1", + }, + } + ], + ) + + assert result == {"accepted": 0, "duplicates": 0, "rejected": 1} + assert store.get_messages("finance", limit=10) == [] + + +def test_gate_store_forwarded_peer_ingest_allows_out_of_order_signed_sequences_today(tmp_path, monkeypatch): + class _GateManager: + def can_enter(self, sender_id, gate_id): + assert sender_id + assert gate_id == "finance" + return True, "Access granted" + + monkeypatch.setattr(mesh_reputation, "gate_manager", _GateManager(), raising=False) + store = GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + author_key = ed25519.Ed25519PrivateKey.generate() + + first = _signed_gate_event( + "finance", + private_key=author_key, + sequence=7, + ciphertext="opaque-ciphertext-7", + nonce="nonce-7", + sender_ref="sender-ref-7", + ) + second = _signed_gate_event( + "finance", + private_key=author_key, + sequence=3, + ciphertext="opaque-ciphertext-3", + nonce="nonce-3", + sender_ref="sender-ref-3", + ) + + result = store.ingest_peer_events("finance", [first, second]) + + assert result == {"accepted": 2, "duplicates": 0, "rejected": 0} + stored = store.get_messages("finance", limit=10) + assert {msg["payload"]["ciphertext"] for msg in stored} == {"opaque-ciphertext-7", "opaque-ciphertext-3"} + + +def test_gate_store_accepts_verified_peer_events_and_persists_sanitized_shape(tmp_path, monkeypatch): + class _GateManager: + def can_enter(self, sender_id, gate_id): + assert sender_id + assert gate_id == "finance" + return True, "Access granted" + + monkeypatch.setattr(mesh_reputation, "gate_manager", _GateManager(), raising=False) + store = GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + + result = store.ingest_peer_events("finance", [_signed_gate_event("finance")]) + + assert result == {"accepted": 1, "duplicates": 0, "rejected": 0} + stored = store.get_messages("finance", limit=1)[0] + assert stored["payload"] == { + "gate": "finance", + "ciphertext": "opaque-ciphertext", + "nonce": "nonce-1", + "sender_ref": "sender-ref-1", + "format": "mls1", + } + assert "node_id" not in stored + assert "signature" not in stored + assert "sequence" not in stored + + +def test_gate_store_rejects_verified_peer_events_from_unauthorized_authors(tmp_path, monkeypatch): + class _GateManager: + def can_enter(self, sender_id, gate_id): + assert sender_id + assert gate_id == "finance" + return False, "Need 10 overall rep" + + monkeypatch.setattr(mesh_reputation, "gate_manager", _GateManager(), raising=False) + store = GateMessageStore(data_dir=str(tmp_path / "gate_messages")) + + result = store.ingest_peer_events("finance", [_signed_gate_event("finance")]) + + assert result == {"accepted": 0, "duplicates": 0, "rejected": 1} + assert store.get_messages("finance", limit=10) == [] + + +def test_mesh_log_hides_private_entries_from_public_callers(monkeypatch): + from services.mesh.mesh_router import mesh_router + + monkeypatch.setattr( + mesh_router, + "message_log", + deque( + [ + { + "sender": "alice", + "destination": "broadcast", + "routed_via": "internet", + "priority": "normal", + "route_reason": "public", + "timestamp": 123.0, + "trust_tier": "public_degraded", + }, + { + "message_id": "m-private", + "routed_via": "tor_arti", + "priority": "normal", + "route_reason": "private", + "timestamp": 456.0, + "trust_tier": "private_strong", + }, + ], + maxlen=8, + ), + ) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + + response = asyncio.run(main.mesh_log(_request("/api/mesh/log"))) + + assert response == { + "log": [ + { + "sender": "alice", + "destination": "broadcast", + "routed_via": "internet", + "priority": "normal", + "route_reason": "public", + "timestamp": 123.0, + } + ] + } + + +def test_mesh_status_public_hides_private_activity_volume(monkeypatch): + from services.mesh.mesh_router import mesh_router + from services import sigint_bridge + + monkeypatch.setattr( + mesh_router, + "message_log", + deque( + [ + { + "sender": "alice", + "destination": "broadcast", + "routed_via": "internet", + "priority": "normal", + "route_reason": "public", + "timestamp": 123.0, + "trust_tier": "public_degraded", + }, + { + "message_id": "m-private", + "routed_via": "tor_arti", + "priority": "normal", + "route_reason": "private", + "timestamp": 456.0, + "trust_tier": "private_strong", + }, + ], + maxlen=8, + ), + ) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr( + sigint_bridge, + "sigint_grid", + type("FakeSigintGrid", (), {"get_all_signals": staticmethod(lambda: [])})(), + ) + + response = asyncio.run(main.mesh_status(_request("/api/mesh/status"))) + + assert response == {"message_log_size": 1} + + +def test_mesh_status_admin_gets_full_log_size_and_warning_details(monkeypatch): + from services.mesh.mesh_router import mesh_router + from services import sigint_bridge + now = time.time() + + monkeypatch.setattr( + mesh_router, + "message_log", + deque( + [ + {"timestamp": now, "trust_tier": "public_degraded"}, + {"timestamp": now, "trust_tier": "private_transitional"}, + ], + maxlen=8, + ), + ) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + monkeypatch.setattr( + sigint_bridge, + "sigint_grid", + type("FakeSigintGrid", (), {"get_all_signals": staticmethod(lambda: [])})(), + ) + + response = asyncio.run(main.mesh_status(_request("/api/mesh/status"))) + + assert response["message_log_size"] == 2 + assert response["public_message_log_size"] == 1 + assert "private_log_retention_seconds" in response + assert isinstance(response["security_warnings"], list) + + +def test_public_oracle_profile_hides_behavioral_lists(monkeypatch): + from services.mesh import mesh_oracle + + fake_profile = { + "node_id": "!oracle", + "oracle_rep": 2.5, + "oracle_rep_total": 3.0, + "oracle_rep_locked": 0.5, + "predictions_won": 4, + "predictions_lost": 1, + "win_rate": 80, + "farming_pct": 25, + "active_stakes": [{"message_id": "msg-1", "side": "truth", "amount": 0.5, "expires": 123}], + "prediction_history": [{"market": "Test", "side": "yes", "rep_earned": 0.7}], + } + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr( + mesh_oracle, + "oracle_ledger", + type("FakeOracleLedger", (), {"get_oracle_profile": staticmethod(lambda *_args, **_kwargs: dict(fake_profile))})(), + raising=False, + ) + + response = asyncio.run(main.oracle_profile(_request("/api/mesh/oracle/profile"), node_id="!oracle")) + + assert response["node_id"] == "!oracle" + assert response["oracle_rep"] == 2.5 + assert response["active_stakes"] == [] + assert response["prediction_history"] == [] + + +def test_public_oracle_predictions_hide_active_positions(monkeypatch): + from services.mesh import mesh_oracle + + active_predictions = [ + {"prediction_id": "pred-1", "market_title": "Alpha", "side": "yes", "mode": "free"}, + {"prediction_id": "pred-2", "market_title": "Bravo", "side": "no", "mode": "staked"}, + ] + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr( + mesh_oracle, + "oracle_ledger", + type( + "FakeOracleLedger", + (), + {"get_active_predictions": staticmethod(lambda *_args, **_kwargs: list(active_predictions))}, + )(), + raising=False, + ) + + response = asyncio.run( + main.oracle_predictions(_request("/api/mesh/oracle/predictions"), node_id="!oracle") + ) + + assert response == {"predictions": [], "count": 2} + + +def test_public_oracle_stakes_hide_staker_lists(monkeypatch): + from services.mesh import mesh_oracle + + fake_stakes = { + "message_id": "msg-1", + "truth_total": 1.5, + "false_total": 0.75, + "truth_stakers": [{"node_id": "!truth", "amount": 1.5, "expires": 123}], + "false_stakers": [{"node_id": "!false", "amount": 0.75, "expires": 456}], + "earliest_expiry": 123, + } + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr( + mesh_oracle, + "oracle_ledger", + type( + "FakeOracleLedger", + (), + {"get_stakes_for_message": staticmethod(lambda *_args, **_kwargs: dict(fake_stakes))}, + )(), + raising=False, + ) + + response = asyncio.run( + main.oracle_stakes_for_message(_request("/api/mesh/oracle/stakes/msg-1"), message_id="msg-1") + ) + + assert response == { + "message_id": "msg-1", + "truth_total": 1.5, + "false_total": 0.75, + "truth_stakers": [], + "false_stakers": [], + "earliest_expiry": 123, + } + + +def test_public_wormhole_settings_redact_sensitive_fields(monkeypatch): + monkeypatch.setattr( + main, + "read_wormhole_settings", + lambda: { + "enabled": True, + "transport": "tor", + "anonymous_mode": True, + "socks_proxy": "127.0.0.1:9050", + "socks_dns": True, + "privacy_profile": "high", + }, + ) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + + response = asyncio.run(main.api_get_wormhole_settings(_request("/api/settings/wormhole"))) + + assert response == { + "enabled": True, + "transport": "tor", + "anonymous_mode": True, + } + + +def test_admin_wormhole_settings_keep_sensitive_fields(monkeypatch): + monkeypatch.setattr( + main, + "read_wormhole_settings", + lambda: { + "enabled": True, + "transport": "tor", + "anonymous_mode": True, + "socks_proxy": "127.0.0.1:9050", + "socks_dns": True, + "privacy_profile": "high", + }, + ) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + + response = asyncio.run(main.api_get_wormhole_settings(_request("/api/settings/wormhole"))) + + assert response["enabled"] is True + assert response["socks_proxy"] == "127.0.0.1:9050" + assert response["socks_dns"] is True + assert response["privacy_profile"] == "high" + + +def test_public_privacy_profile_hides_transport_metadata(monkeypatch): + monkeypatch.setattr( + main, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "high", + "transport": "tor", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + + response = asyncio.run(main.api_get_privacy_profile(_request("/api/settings/privacy-profile"))) + + assert response == { + "profile": "high", + "wormhole_enabled": True, + } + + +def test_public_settings_wormhole_status_uses_redacted_shape(monkeypatch): + monkeypatch.setattr( + main, + "get_wormhole_state", + lambda: { + "installed": True, + "configured": True, + "running": True, + "ready": True, + "last_error": "sensitive", + "proxy_active": "tor", + "arti_ready": True, + }, + ) + monkeypatch.setattr(main, "_current_private_lane_tier", lambda *_args, **_kwargs: "private_strong") + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + + response = asyncio.run(main.api_get_wormhole_status(_request("/api/settings/wormhole-status"))) + + assert response == { + "installed": True, + "configured": True, + "running": True, + "ready": True, + } + + +def test_public_infonet_status_hides_private_lane_policy(monkeypatch): + monkeypatch.setattr( + main, + "_check_scoped_auth", + lambda *_args, **_kwargs: (False, "no"), + ) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + + response = asyncio.run(main.infonet_status(_request("/api/mesh/infonet/status"))) + + assert "private_lane_tier" not in response + assert "private_lane_policy" not in response + assert "network_id" in response + + +def test_public_rns_status_hides_private_lane_policy(monkeypatch): + from services import wormhole_supervisor + from services.mesh import mesh_rns + + monkeypatch.setattr( + main, + "_check_scoped_auth", + lambda *_args, **_kwargs: (False, "no"), + ) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + monkeypatch.setattr( + main, + "_current_private_lane_tier", + lambda *_args, **_kwargs: "private_strong", + ) + + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": True}, + ) + monkeypatch.setattr( + mesh_rns, + "rns_bridge", + type( + "FakeRnsBridge", + (), + { + "status": staticmethod( + lambda: { + "enabled": True, + "ready": True, + "configured_peers": 3, + "active_peers": 2, + "local_hash": "abc123", + "session_identities": 4, + "destination_age_s": 90, + "private_dm_direct_ready": True, + } + ) + }, + )(), + ) + + response = asyncio.run(main.mesh_rns_status(_request("/api/mesh/rns/status"))) + + assert response == { + "enabled": True, + "ready": True, + "configured_peers": 3, + "active_peers": 2, + } + + +def test_public_dm_witness_hides_graph_details(monkeypatch): + from services.mesh import mesh_dm_relay + + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + monkeypatch.setattr( + mesh_dm_relay, + "dm_relay", + type( + "FakeRelay", + (), + {"get_witnesses": staticmethod(lambda *_args, **_kwargs: [{"witness_id": "!alpha"}])}, + )(), + raising=False, + ) + + response = asyncio.run( + main.dm_key_witness_get( + _request("/api/mesh/dm/witness"), + target_id="!target", + dh_pub_key="dh-pub", + ) + ) + + assert response == {"ok": True, "count": 1} + + +def test_audit_dm_witness_keeps_graph_details(monkeypatch): + from services.mesh import mesh_dm_relay + + witnesses = [{"witness_id": "!alpha"}, {"witness_id": "!bravo"}] + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + monkeypatch.setattr( + mesh_dm_relay, + "dm_relay", + type( + "FakeRelay", + (), + {"get_witnesses": staticmethod(lambda *_args, **_kwargs: witnesses)}, + )(), + raising=False, + ) + + response = asyncio.run( + main.dm_key_witness_get( + _request("/api/mesh/dm/witness"), + target_id="!target", + dh_pub_key="dh-pub", + ) + ) + + assert response == { + "ok": True, + "target_id": "!target", + "dh_pub_key": "dh-pub", + "count": 2, + "witnesses": witnesses, + } + + +def test_public_gate_compose_redacts_signer_fields(monkeypatch): + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + monkeypatch.setattr( + main, + "compose_encrypted_gate_message", + lambda **_kwargs: { + "ok": True, + "gate_id": "finance", + "identity_scope": "gate_persona", + "sender_id": "!gate-sender", + "public_key": "public-key", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "sequence": 42, + "signature": "deadbeef", + "ciphertext": "ciphertext", + "nonce": "nonce", + "sender_ref": "sender-ref", + "format": "mls1", + "timestamp": 123.0, + "epoch": 3, + }, + ) + + response = asyncio.run( + main.api_wormhole_gate_message_compose( + _request("/api/wormhole/gate/message/compose", method="POST"), + main.WormholeGateComposeRequest(gate_id="finance", plaintext="hello"), + ) + ) + + assert response == { + "ok": True, + "gate_id": "finance", + "identity_scope": "gate_persona", + "ciphertext": "ciphertext", + "nonce": "nonce", + "sender_ref": "sender-ref", + "format": "mls1", + "timestamp": 123.0, + "epoch": 3, + } + + +def test_dm_relay_auto_msg_id_omits_sender_suffix(tmp_path, monkeypatch): + from services.mesh import mesh_dm_relay + + monkeypatch.setattr(mesh_dm_relay, "DATA_DIR", tmp_path, raising=False) + monkeypatch.setattr(mesh_dm_relay, "RELAY_FILE", tmp_path / "dm_relay.json", raising=False) + relay = mesh_dm_relay.DMRelay() + + result = relay.deposit( + sender_id="!alice-1234", + recipient_id="!bob", + ciphertext="ciphertext", + delivery_class="request", + ) + + assert result["ok"] is True + assert str(result["msg_id"]).startswith("dm_") + assert "1234" not in str(result["msg_id"]) + + +def test_public_event_endpoints_preserve_redactions(client, monkeypatch): + from services.mesh import mesh_hashchain + + fake_infonet = _FakePublicInfonet() + monkeypatch.setattr(mesh_hashchain, "infonet", fake_infonet, raising=False) + monkeypatch.setattr( + mesh_hashchain, + "gate_store", + type("FakeGateStore", (), {"get_event": staticmethod(lambda *_args, **_kwargs: None)})(), + raising=False, + ) + + collection_responses = [ + client.get("/api/mesh/infonet/messages").json()["messages"], + client.get("/api/mesh/infonet/events").json()["events"], + client.get("/api/mesh/infonet/sync").json()["events"], + asyncio.run( + main.infonet_sync_post( + _json_request("/api/mesh/infonet/sync", {"locator": ["head-1"]}) + ) + )["events"], + ] + node_response = client.get("/api/mesh/infonet/node/!node-1").json()["events"] + single_event = client.get("/api/mesh/infonet/event/rotate-1").json() + + for events in collection_responses: + assert all(evt.get("event_type") != "gate_message" for evt in events) + vote = next(evt for evt in events if evt.get("event_type") == "vote") + rotate = next(evt for evt in events if evt.get("event_type") == "key_rotate") + assert "gate" not in vote.get("payload", {}) + assert "old_node_id" not in rotate.get("payload", {}) + assert "old_public_key" not in rotate.get("payload", {}) + assert "old_signature" not in rotate.get("payload", {}) + + assert len(node_response) == 1 + assert node_response[0]["event_type"] == "vote" + assert set(node_response[0].keys()) == {"event_id", "event_type", "timestamp"} + + assert single_event["event_type"] == "key_rotate" + assert "old_node_id" not in single_event.get("payload", {}) + + +def test_mesh_router_private_log_entries_age_out(monkeypatch): + from services import config as config_mod + from services.mesh.mesh_router import MeshRouter + + monkeypatch.setattr( + config_mod, + "get_settings", + lambda: type("Settings", (), {"MESH_PRIVATE_LOG_TTL_S": 60})(), + ) + + router = MeshRouter() + router.message_log = deque( + [ + {"timestamp": 100.0, "trust_tier": "private_strong"}, + {"timestamp": 100.0, "trust_tier": "public_degraded"}, + {"timestamp": 180.0, "trust_tier": "private_transitional"}, + ], + maxlen=8, + ) + + router.prune_message_log(now=200.0) + + assert list(router.message_log) == [ + {"timestamp": 100.0, "trust_tier": "public_degraded"}, + {"timestamp": 180.0, "trust_tier": "private_transitional"}, + ] + + +def test_mesh_router_private_log_entries_strip_metadata(caplog): + from services.mesh.mesh_router import MeshEnvelope, MeshRouter, Priority, TransportResult + + router = MeshRouter() + envelope = MeshEnvelope( + sender_id="alice", + destination="bob", + channel="shadow", + priority=Priority.NORMAL, + trust_tier="private_strong", + payload="secret payload", + ) + envelope.routed_via = "tor_arti" + envelope.route_reason = "PRIVATE_STRONG — Tor required" + + with caplog.at_level(logging.INFO, logger="services.mesh_router"): + router._log(envelope, [TransportResult(True, "tor_arti", "Delivered to 1 peer via Tor")]) + + entry = list(router.message_log)[0] + assert entry["trust_tier"] == "private_strong" + assert entry["routed_via"] == "tor_arti" + assert entry["transport_outcomes"] == [{"transport": "tor_arti", "ok": True}] + assert "message_id" not in entry + assert "channel" not in entry + assert "payload_type" not in entry + assert "payload_bytes" not in entry + assert "results" not in entry + assert envelope.message_id not in caplog.text + + +def test_mesh_metrics_requires_audit_scope(client, monkeypatch): + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "Forbidden — invalid or missing admin key")) + + response = client.get("/api/mesh/metrics") + + assert response.status_code == 403 + assert response.json()["detail"] == "Forbidden — invalid or missing admin key" + + +def test_mesh_metrics_allows_audit_scope(client, monkeypatch): + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + + response = client.get("/api/mesh/metrics") + + assert response.status_code == 200 + assert "counters" in response.json() + + +def test_dm_send_rejects_unsealed_shared_private_dm(monkeypatch): + from services import wormhole_supervisor + + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_strong") + + response = asyncio.run( + main.dm_send( + _json_request( + "/api/mesh/dm/send", + { + "sender_id": "alice", + "recipient_id": "bob", + "delivery_class": "shared", + "recipient_token": "shared-mailbox-token", + "ciphertext": "ciphertext", + "msg_id": "dm-shared-1", + "timestamp": int(time.time()), + "public_key": "cHVi", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 1, + "protocol_version": "infonet/2", + }, + ) + ) + ) + + assert response == {"ok": False, "detail": "sealed sender required for shared private DMs"} + + +def test_dm_key_package_error_detail_is_sanitized(monkeypatch): + from services.mesh import mesh_dm_mls + + def _raise(*_args, **_kwargs): + raise RuntimeError("sensitive backend detail") + + monkeypatch.setattr(mesh_dm_mls, "_identity_handle_for_alias", _raise) + + response = mesh_dm_mls.export_dm_key_package_for_alias("alpha") + + assert response == {"ok": False, "detail": "dm_mls_key_package_failed"} + + +def test_gate_compose_error_detail_is_sanitized(monkeypatch): + from services.mesh import mesh_gate_mls + from services import wormhole_supervisor + + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + monkeypatch.setattr(mesh_gate_mls, "_active_gate_persona", lambda *_args, **_kwargs: {"persona_id": "p1"}) + monkeypatch.setattr(mesh_gate_mls, "_sync_binding", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("sensitive gate detail"))) + + response = mesh_gate_mls.compose_encrypted_gate_message("finance", "hello") + + assert response == {"ok": False, "detail": "gate_mls_compose_failed"} + + +def test_dm_alias_blob_sign_error_detail_is_sanitized(monkeypatch): + from services.mesh import mesh_wormhole_persona + + monkeypatch.setattr(mesh_wormhole_persona, "bootstrap_wormhole_persona_state", lambda: None) + monkeypatch.setattr( + mesh_wormhole_persona, + "read_wormhole_persona_state", + lambda: { + "dm_identity": { + "private_key": "not-base64", + "public_key": "pub", + "public_key_algo": "Ed25519", + } + }, + ) + + response = mesh_wormhole_persona.sign_dm_alias_blob("alpha", b"payload") + + assert response == {"ok": False, "detail": "dm_alias_blob_sign_failed"} + + +def test_public_mesh_reputation_is_summary_only(client, monkeypatch): + from services.mesh import mesh_reputation as mesh_reputation_mod + + class _FakeLedger: + def __init__(self): + self.calls = [] + + def get_reputation_log(self, node_id, detailed=False): + self.calls.append((node_id, detailed)) + payload = { + "node_id": node_id, + "overall": 7, + "upvotes": 3, + "downvotes": 1, + } + if detailed: + payload.update( + { + "gates": {"finance": 5}, + "recent_votes": [{"voter": "blind-voter"}], + "node_age_days": 14.0, + "is_agent": True, + } + ) + return payload + + fake_ledger = _FakeLedger() + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (False, "no")) + monkeypatch.setattr(main, "_is_debug_test_request", lambda *_args, **_kwargs: False) + + response = client.get("/api/mesh/reputation?node_id=!alpha") + + assert response.status_code == 200 + assert response.json() == { + "node_id": "!alpha", + "overall": 7, + "upvotes": 3, + "downvotes": 1, + } + assert fake_ledger.calls == [("!alpha", False)] + + +def test_audit_mesh_reputation_keeps_detailed_breakdown(client, monkeypatch): + from services.mesh import mesh_reputation as mesh_reputation_mod + + class _FakeLedger: + def __init__(self): + self.calls = [] + + def get_reputation_log(self, node_id, detailed=False): + self.calls.append((node_id, detailed)) + payload = { + "node_id": node_id, + "overall": 9, + "upvotes": 4, + "downvotes": 1, + } + if detailed: + payload.update( + { + "gates": {"finance": 6}, + "recent_votes": [{"voter": "blind-voter"}], + "node_age_days": 21.0, + "is_agent": False, + } + ) + return payload + + fake_ledger = _FakeLedger() + monkeypatch.setattr(mesh_reputation_mod, "reputation_ledger", fake_ledger, raising=False) + monkeypatch.setattr(main, "_check_scoped_auth", lambda *_args, **_kwargs: (True, "ok")) + + response = client.get("/api/mesh/reputation?node_id=!bravo") + + assert response.status_code == 200 + assert response.json()["gates"] == {"finance": 6} + assert response.json()["recent_votes"] == [{"voter": "blind-voter"}] + assert response.json()["node_age_days"] == 21.0 + assert response.json()["is_agent"] is False + assert fake_ledger.calls == [("!bravo", True)] + + +def test_dm_mls_logs_only_hashed_aliases_on_failure(caplog, monkeypatch): + from services.mesh import mesh_dm_mls + + def _raise(*_args, **_kwargs): + raise RuntimeError("sensitive backend detail") + + monkeypatch.setattr(mesh_dm_mls, "_identity_handle_for_alias", _raise) + + with caplog.at_level(logging.ERROR, logger="services.mesh.mesh_dm_mls"): + response = mesh_dm_mls.export_dm_key_package_for_alias("alpha-alias") + + assert response == {"ok": False, "detail": "dm_mls_key_package_failed"} + assert "alpha-alias" not in caplog.text + assert "alias#" in caplog.text + + +def test_gate_mls_logs_only_hashed_gate_ids_on_failure(caplog, monkeypatch): + from services import wormhole_supervisor + from services.mesh import mesh_gate_mls + + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + monkeypatch.setattr(mesh_gate_mls, "_active_gate_persona", lambda *_args, **_kwargs: {"persona_id": "p1"}) + monkeypatch.setattr( + mesh_gate_mls, + "_sync_binding", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("gate failure")), + ) + + with caplog.at_level(logging.ERROR, logger="services.mesh.mesh_gate_mls"): + response = mesh_gate_mls.compose_encrypted_gate_message("finance-ops", "hello") + + assert response == {"ok": False, "detail": "gate_mls_compose_failed"} + assert "finance-ops" not in caplog.text + assert "gate#" in caplog.text + + +def test_gate_persona_sign_logs_hashed_ids_on_failure(caplog, monkeypatch): + from services.mesh import mesh_wormhole_persona + + monkeypatch.setattr(mesh_wormhole_persona, "bootstrap_wormhole_persona_state", lambda: None) + monkeypatch.setattr( + mesh_wormhole_persona, + "read_wormhole_persona_state", + lambda: { + "gate_personas": { + "finance": [ + { + "persona_id": "persona-raw", + "private_key": "not-base64", + } + ] + } + }, + ) + + with caplog.at_level(logging.ERROR, logger="services.mesh.mesh_wormhole_persona"): + response = mesh_wormhole_persona.sign_gate_persona_blob("finance", "persona-raw", b"payload") + + assert response == {"ok": False, "detail": "persona_blob_sign_failed"} + assert "persona-raw" not in caplog.text + assert "gate#" in caplog.text + assert "persona#" in caplog.text + + +def test_reputation_logs_hash_node_and_gate_identifiers(tmp_path, monkeypatch, caplog): + from services.mesh import mesh_reputation + + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "LEDGER_FILE", tmp_path / "reputation_ledger.json") + monkeypatch.setattr(mesh_reputation, "GATES_FILE", tmp_path / "gates.json") + + ledger = mesh_reputation.ReputationLedger() + + with caplog.at_level(logging.INFO, logger="services.mesh_reputation"): + ledger.register_node("!alpha", "pub-a", "Ed25519") + ledger.register_node("!bravo", "pub-b", "Ed25519") + ok, _detail = ledger.cast_vote("!alpha", "!bravo", 1, "finance") + + assert ok is True + assert "!alpha" not in caplog.text + assert "!bravo" not in caplog.text + assert "finance" not in caplog.text + assert "node#" in caplog.text + assert "gate#" in caplog.text diff --git a/backend/tests/mesh/test_mesh_protocol_hygiene.py b/backend/tests/mesh/test_mesh_protocol_hygiene.py new file mode 100644 index 00000000..25681c9f --- /dev/null +++ b/backend/tests/mesh/test_mesh_protocol_hygiene.py @@ -0,0 +1,85 @@ +import asyncio + + +def test_x3dh_hkdf_uses_nonzero_ff_salt(): + from services.mesh.mesh_wormhole_prekey import _hkdf + + derived = _hkdf(b"input-material", "SB-TEST") + assert derived + assert derived != _hkdf(b"input-material", "SB-TEST-ALT") + + +def test_ratchet_padding_extends_large_payloads(): + from services.mesh.mesh_wormhole_ratchet import _build_padded_payload, PAD_MAGIC, PAD_STEP + + plaintext = "x" * 5000 + padded = _build_padded_payload(plaintext) + + assert padded[:4].decode("utf-8") == PAD_MAGIC + assert len(padded) > len(plaintext.encode("utf-8")) + assert len(padded) % PAD_STEP == 0 + + +def test_dead_drop_epoch_shortens_in_high_privacy(monkeypatch): + from services.mesh import mesh_wormhole_dead_drop + + monkeypatch.setattr( + mesh_wormhole_dead_drop, + "read_wormhole_settings", + lambda: {"privacy_profile": "high"}, + ) + assert mesh_wormhole_dead_drop.mailbox_epoch_seconds() == 2 * 60 * 60 + + +def test_relay_jitter_only_applies_in_high_privacy(monkeypatch): + import main + + sleeps: list[float] = [] + + async def fake_sleep(delay: float): + sleeps.append(delay) + + monkeypatch.setattr(main, "_high_privacy_profile_enabled", lambda: True) + monkeypatch.setattr(main.asyncio, "sleep", fake_sleep) + + asyncio.run(main._maybe_apply_dm_relay_jitter()) + + assert len(sleeps) == 1 + assert 0.05 <= sleeps[0] <= 0.5 + + +def test_high_privacy_refuses_private_tier_clearnet_fallback(monkeypatch): + from services.mesh.mesh_router import MeshEnvelope, MeshRouter, Priority, TransportResult + + router = MeshRouter() + internet_attempts: list[str] = [] + + monkeypatch.setattr( + "services.mesh.mesh_router._high_privacy_profile_blocks_clearnet_fallback", + lambda: True, + ) + monkeypatch.setattr(router.tor_arti, "can_reach", lambda _envelope: False) + monkeypatch.setattr( + router.internet, + "send", + lambda *_args, **_kwargs: ( + internet_attempts.append("internet"), + TransportResult(True, "internet", "sent"), + )[1], + ) + + results = router.route( + MeshEnvelope( + sender_id="!sb_sender", + destination="!sb_dest", + payload="ciphertext", + trust_tier="private_transitional", + priority=Priority.NORMAL, + ), + {}, + ) + + assert internet_attempts == [] + assert len(results) == 1 + assert results[0].transport == "policy" + assert "clearnet fallback refused" in results[0].detail diff --git a/backend/tests/mesh/test_mesh_public_meshtastic_boundary.py b/backend/tests/mesh/test_mesh_public_meshtastic_boundary.py new file mode 100644 index 00000000..70e87914 --- /dev/null +++ b/backend/tests/mesh/test_mesh_public_meshtastic_boundary.py @@ -0,0 +1,296 @@ +import asyncio +import time +from collections import deque +from types import SimpleNamespace + + +class _DummyBreaker: + def check_and_record(self, _priority): + return True, "ok" + + +class _FakeMeshtasticTransport: + NAME = "meshtastic" + + def __init__(self, can_reach: bool = True, send_ok: bool = True): + self._can_reach = can_reach + self._send_ok = send_ok + self.sent = [] + + def can_reach(self, _envelope): + return self._can_reach + + def send(self, envelope, _credentials): + from services.mesh.mesh_router import TransportResult + + self.sent.append(envelope) + return TransportResult(self._send_ok, self.NAME, "sent") + + +class _FakeMeshRouter: + def __init__(self, meshtastic): + self.meshtastic = meshtastic + self.breakers = {"meshtastic": _DummyBreaker()} + self.route_called = False + + def route(self, _envelope, _credentials): + self.route_called = True + return [] + + +def _valid_body(**overrides): + body = { + "destination": "!a0cc7a80", + "message": "hello mesh", + "sender_id": "!sb_sender", + "node_id": "!sb_sender", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 1, + "protocol_version": "1", + "channel": "LongFast", + "priority": "normal", + "ephemeral": False, + "transport_lock": "meshtastic", + "credentials": {"mesh_region": "US"}, + } + body.update(overrides) + return body + + +def test_meshtastic_transport_lock_stays_on_public_direct_path(monkeypatch): + import main + from services.mesh import mesh_router as mesh_router_mod + from services.sigint_bridge import sigint_grid + from httpx import ASGITransport, AsyncClient + + fake_meshtastic = _FakeMeshtasticTransport(can_reach=True, send_ok=True) + fake_router = _FakeMeshRouter(fake_meshtastic) + fake_bridge = SimpleNamespace(messages=deque(maxlen=10)) + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_check_throttle", lambda *_: (True, "ok")) + monkeypatch.setattr(mesh_router_mod, "mesh_router", fake_router) + monkeypatch.setattr(sigint_grid, "mesh", fake_bridge) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post("/api/mesh/send", json=_valid_body()) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is True + assert result["routed_via"] == "meshtastic" + assert "public node-targeted path" in result["route_reason"] + assert fake_router.route_called is False + assert len(fake_meshtastic.sent) == 1 + assert fake_meshtastic.sent[0].destination == "!a0cc7a80" + assert fake_bridge.messages[0]["from"] == "!0779e8b8" + + +def test_meshtastic_transport_lock_does_not_fallback_when_unreachable(monkeypatch): + import main + from services.mesh import mesh_router as mesh_router_mod + from httpx import ASGITransport, AsyncClient + + fake_meshtastic = _FakeMeshtasticTransport(can_reach=False, send_ok=False) + fake_router = _FakeMeshRouter(fake_meshtastic) + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_check_throttle", lambda *_: (True, "ok")) + monkeypatch.setattr(mesh_router_mod, "mesh_router", fake_router) + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post("/api/mesh/send", json=_valid_body(message="x" * 10)) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is False + assert result["routed_via"] == "" + assert fake_router.route_called is False + assert fake_meshtastic.sent == [] + assert result["results"] == [ + { + "ok": False, + "transport": "meshtastic", + "detail": "Message exceeds Meshtastic payload limit", + } + ] + + +def test_meshtastic_transport_lock_allows_two_messages_per_minute(monkeypatch): + import main + + node_id = "!sb_meshrate" + now = time.time() + main._node_throttle[node_id] = { + "last_send": now - 31, + "daily_count": 0, + "daily_reset": now, + "first_seen": now, + } + + ok_first, _reason_first = main._check_throttle(node_id, "normal", "meshtastic") + ok_second, reason_second = main._check_throttle(node_id, "normal", "meshtastic") + + assert ok_first is True + assert ok_second is False + assert "1 message per 30s" in reason_second + + +def test_private_trust_tier_skips_public_transports(): + from services.mesh.mesh_router import MeshEnvelope, MeshRouter, Priority, TransportResult + + class _FakeTransport: + def __init__(self, name): + self.NAME = name + self.sent = [] + + def can_reach(self, _envelope): + return True + + def send(self, envelope, _credentials): + self.sent.append(envelope) + return TransportResult(True, self.NAME, "sent") + + router = MeshRouter() + router.aprs = _FakeTransport("aprs") + router.meshtastic = _FakeTransport("meshtastic") + router.internet = _FakeTransport("internet") + router.transports = [router.aprs, router.meshtastic, router.internet] + + envelope = MeshEnvelope( + sender_id="!sb_sender", + destination="broadcast", + priority=Priority.NORMAL, + payload="private payload", + trust_tier="private_strong", + ) + + results = router.route(envelope, {}) + + assert [r.transport for r in results] == ["policy"] + assert router.aprs.sent == [] + assert router.meshtastic.sent == [] + assert len(router.internet.sent) == 0 + + +def test_private_route_recognizes_tor_arti_and_falls_back_to_internet(): + from services.mesh.mesh_router import MeshEnvelope, MeshRouter, Priority, TransportResult + + class _FakeTransport: + def __init__(self, name, ok=True): + self.NAME = name + self.ok = ok + self.sent = [] + + def can_reach(self, _envelope): + return True + + def send(self, envelope, _credentials): + self.sent.append(envelope) + return TransportResult(self.ok, self.NAME, "sent" if self.ok else "stub") + + router = MeshRouter() + router.aprs = _FakeTransport("aprs") + router.meshtastic = _FakeTransport("meshtastic") + router.tor_arti = _FakeTransport("tor_arti", ok=False) + router.internet = _FakeTransport("internet", ok=True) + router.transports = [router.aprs, router.meshtastic, router.tor_arti, router.internet] + + envelope = MeshEnvelope( + sender_id="!sb_sender", + destination="broadcast", + priority=Priority.NORMAL, + payload="private payload", + trust_tier="private_strong", + ) + + results = router.route(envelope, {}) + + assert [r.transport for r in results] == ["tor_arti", "policy"] + assert router.aprs.sent == [] + assert router.meshtastic.sent == [] + assert len(router.tor_arti.sent) == 1 + assert len(router.internet.sent) == 0 + + +def test_private_tier_blocks_meshtastic_transport_lock(monkeypatch): + """C-2 fix: transport_lock=meshtastic must be rejected when trust_tier is private.""" + import main + from services.mesh import mesh_router as mesh_router_mod + from services import wormhole_supervisor + from httpx import ASGITransport, AsyncClient + + fake_meshtastic = _FakeMeshtasticTransport(can_reach=True, send_ok=True) + fake_router = _FakeMeshRouter(fake_meshtastic) + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_check_throttle", lambda *_: (True, "ok")) + monkeypatch.setattr(mesh_router_mod, "mesh_router", fake_router) + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post("/api/mesh/send", json=_valid_body()) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is False + assert "Private-tier content cannot be sent over Meshtastic" in result["results"][0]["detail"] + assert fake_meshtastic.sent == [] + assert fake_router.route_called is False + + +def test_envelope_trust_tier_set_from_wormhole_state(monkeypatch): + """C-1 fix: MeshEnvelope.trust_tier must reflect actual Wormhole transport tier.""" + import main + from services.mesh import mesh_router as mesh_router_mod + from services import wormhole_supervisor + from services.sigint_bridge import sigint_grid + from httpx import ASGITransport, AsyncClient + + captured_envelopes = [] + + class _CapturingRouter: + def __init__(self): + self.meshtastic = _FakeMeshtasticTransport(can_reach=True, send_ok=True) + self.breakers = {"meshtastic": _DummyBreaker()} + + def route(self, envelope, _credentials): + from services.mesh.mesh_router import TransportResult + + captured_envelopes.append(envelope) + return [TransportResult(True, "internet", "sent")] + + fake_router = _CapturingRouter() + fake_bridge = SimpleNamespace(messages=deque(maxlen=10)) + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_preflight_signed_event_integrity", lambda **_: (True, "ok")) + monkeypatch.setattr(main, "_check_throttle", lambda *_: (True, "ok")) + monkeypatch.setattr(mesh_router_mod, "mesh_router", fake_router) + monkeypatch.setattr(sigint_grid, "mesh", fake_bridge) + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_transitional") + + body = _valid_body() + del body["transport_lock"] # no lock — use normal routing + + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + response = await ac.post("/api/mesh/send", json=body) + return response.json() + + result = asyncio.run(_run()) + + assert result["ok"] is True + assert len(captured_envelopes) == 1 + assert captured_envelopes[0].trust_tier == "private_transitional" diff --git a/backend/tests/mesh/test_mesh_reputation_link.py b/backend/tests/mesh/test_mesh_reputation_link.py new file mode 100644 index 00000000..f435be74 --- /dev/null +++ b/backend/tests/mesh/test_mesh_reputation_link.py @@ -0,0 +1,158 @@ +import json +import time + +from services.mesh import mesh_reputation, mesh_secure_storage + + +def test_identity_link_merges_reputation(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "LEDGER_FILE", tmp_path / "rep.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + + ledger = mesh_reputation.ReputationLedger() + + now = time.time() + ledger.votes = [ + { + "voter_id": "!sb_v1", + "target_id": "!sb_old", + "vote": 1, + "gate": "", + "timestamp": now, + "weight": 1.0, + "agent_verify": False, + }, + { + "voter_id": "!sb_v2", + "target_id": "!sb_new", + "vote": 1, + "gate": "", + "timestamp": now, + "weight": 1.0, + "agent_verify": False, + }, + ] + ledger._scores_dirty = True + + ok, _ = ledger.link_identities("!sb_old", "!sb_new") + assert ok is True + + rep = ledger.get_reputation("!sb_new") + assert rep["overall"] == 2 + assert "linked_from" not in rep + assert ledger.aliases["!sb_new"] == "!sb_old" + + +def test_reputation_ledger_is_encrypted_at_rest(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "LEDGER_FILE", tmp_path / "reputation_ledger.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + + ledger = mesh_reputation.ReputationLedger() + ledger.register_node("!sb_voter") + ledger.register_node("!sb_target") + + ok, _reason = ledger.cast_vote("!sb_voter", "!sb_target", 1) + assert ok is True + + ledger._flush() + domain_path = tmp_path / mesh_reputation.LEDGER_DOMAIN / mesh_reputation.LEDGER_FILE.name + raw = domain_path.read_text(encoding="utf-8") + + assert '"kind": "sb_secure_json"' in raw + assert domain_path.exists() + assert not mesh_reputation.LEDGER_FILE.exists() + assert "!sb_voter" not in raw + assert "!sb_target" not in raw + + +def test_reputation_votes_are_blinded_inside_encrypted_ledger(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "LEDGER_FILE", tmp_path / "reputation_ledger.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + + ledger = mesh_reputation.ReputationLedger() + ledger.register_node("!sb_voter") + ledger.register_node("!sb_target") + + ok, _reason = ledger.cast_vote("!sb_voter", "!sb_target", 1) + assert ok is True + + ledger._flush() + stored = mesh_secure_storage.read_domain_json( + mesh_reputation.LEDGER_DOMAIN, + mesh_reputation.LEDGER_FILE.name, + lambda: {}, + ) + vote = stored["votes"][0] + + assert "voter_id" not in vote + assert vote["blinded_voter_id"] + + +def test_reputation_duplicate_same_direction_vote_is_rejected(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "LEDGER_FILE", tmp_path / "reputation_ledger.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + + ledger = mesh_reputation.ReputationLedger() + ledger.register_node("!sb_voter") + ledger.register_node("!sb_target") + + ok, reason = ledger.cast_vote("!sb_voter", "!sb_target", 1, "infonet") + assert ok is True + assert "Voted up" in reason + assert len(ledger.votes) == 1 + + ok, reason = ledger.cast_vote("!sb_voter", "!sb_target", 1, "infonet") + assert ok is False + assert reason == "Vote already set to up on !sb_target in gate 'infonet'" + assert len(ledger.votes) == 1 + + +def test_reputation_vote_direction_can_change_without_creating_duplicates(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "LEDGER_FILE", tmp_path / "reputation_ledger.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + + ledger = mesh_reputation.ReputationLedger() + ledger.register_node("!sb_voter") + ledger.register_node("!sb_target") + + ok, _reason = ledger.cast_vote("!sb_voter", "!sb_target", 1, "infonet") + assert ok is True + assert len(ledger.votes) == 1 + + ok, reason = ledger.cast_vote("!sb_voter", "!sb_target", -1, "infonet") + assert ok is True + assert "Voted down" in reason + assert len(ledger.votes) == 1 + assert ledger.votes[0]["vote"] == -1 + + +def test_gate_catalog_is_domain_encrypted_with_legacy_migration(tmp_path, monkeypatch): + monkeypatch.setattr(mesh_reputation, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_reputation, "GATES_FILE", tmp_path / "gates.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + + legacy = {"ops": {"display_name": "Ops", "fixed": False}} + mesh_reputation.GATES_FILE.write_text(json.dumps(legacy), encoding="utf-8") + + manager = mesh_reputation.GateManager(mesh_reputation.ReputationLedger()) + domain_path = tmp_path / mesh_reputation.GATES_DOMAIN / mesh_reputation.GATES_FILE.name + stored = mesh_secure_storage.read_domain_json( + mesh_reputation.GATES_DOMAIN, + mesh_reputation.GATES_FILE.name, + lambda: {}, + ) + + assert domain_path.exists() + assert not mesh_reputation.GATES_FILE.exists() + assert stored["ops"]["display_name"] == "Ops" + assert manager.gates["ops"]["display_name"] == "Ops" diff --git a/backend/tests/mesh/test_mesh_rns_concurrency.py b/backend/tests/mesh/test_mesh_rns_concurrency.py new file mode 100644 index 00000000..19928ffc --- /dev/null +++ b/backend/tests/mesh/test_mesh_rns_concurrency.py @@ -0,0 +1,421 @@ +import base64 +import json +import random +import threading +import time +import uuid +from types import SimpleNamespace + +from services.mesh.mesh_hashchain import infonet +from services.mesh.mesh_rns import RNSBridge + + +def _xor_parity(data: list[bytes]) -> bytes: + parity = data[0] + for shard in data[1:]: + parity = bytes(a ^ b for a, b in zip(parity, shard)) + return parity + + +def _make_shard_body( + shard_id: str, + index: int, + total: int, + data_shards: int, + parity_shards: int, + size: int, + length: int, + parity: bool, + fec: str, + blob: bytes, +) -> dict: + return { + "shard_id": shard_id, + "index": index, + "total": total, + "data_shards": data_shards, + "parity_shards": parity_shards, + "size": size, + "length": length, + "parity": parity, + "fec": fec, + "data": base64.b64encode(blob).decode("ascii"), + } + + +def test_rns_quorum_thread_safety(monkeypatch) -> None: + bridge = RNSBridge() + sync_id = "sync-test" + head_hash = infonet.head_hash or "head" + with bridge._sync_lock: + bridge._pending_sync[sync_id] = { + "created": time.time(), + "expected": set(), + "quorum": 2, + "responses": {}, + "responders": set(), + } + + ingested: list[list[dict]] = [] + + def _fake_ingest(events: list[dict]) -> None: + ingested.append(events) + + monkeypatch.setattr(bridge, "_ingest_ordered", _fake_ingest) + + threads = [] + for idx in range(4): + meta = {"sync_id": sync_id, "head_hash": head_hash, "reply_to": f"peer-{idx}"} + t = threading.Thread( + target=bridge._ingest_with_quorum, args=([{"event_id": f"e{idx}"}], meta) + ) + threads.append(t) + t.start() + for t in threads: + t.join() + + assert sync_id not in bridge._pending_sync + assert ingested + + +def test_rns_shard_reassembly_thread_safety(monkeypatch) -> None: + bridge = RNSBridge() + payload = b"mesh-concurrency" * 256 + data_shards = 4 + data, length = bridge._split_payload(payload, data_shards) + size = len(data[0]) + parity = _xor_parity(data) + shard_id = uuid.uuid4().hex + total = data_shards + 1 + + bodies = [ + _make_shard_body( + shard_id, + idx, + total, + data_shards, + 1, + size, + length, + False, + "xor", + shard, + ) + for idx, shard in enumerate(data) + ] + bodies.append( + _make_shard_body( + shard_id, + data_shards, + total, + data_shards, + 1, + size, + length, + True, + "xor", + parity, + ) + ) + random.shuffle(bodies) + + assembled: list[bytes] = [] + + def _fake_on_packet(data: bytes, packet=None) -> None: + assembled.append(data) + + monkeypatch.setattr(bridge, "_on_packet", _fake_on_packet) + + threads = [threading.Thread(target=bridge._handle_infonet_shard, args=(body,)) for body in bodies] + for t in threads: + t.start() + for t in threads: + t.join() + + assert assembled + assert assembled[-1] == payload + + +def test_rns_shard_reassembly_with_loss_and_delay(monkeypatch) -> None: + bridge = RNSBridge() + payload = b"mesh-loss-delay" * 256 + data_shards = 5 + data, length = bridge._split_payload(payload, data_shards) + size = len(data[0]) + parity = _xor_parity(data) + shard_id = uuid.uuid4().hex + total = data_shards + 1 + + bodies = [ + _make_shard_body( + shard_id, + idx, + total, + data_shards, + 1, + size, + length, + False, + "xor", + shard, + ) + for idx, shard in enumerate(data) + ] + bodies.append( + _make_shard_body( + shard_id, + data_shards, + total, + data_shards, + 1, + size, + length, + True, + "xor", + parity, + ) + ) + + rng = random.Random(1337) + drop_index = rng.randrange(data_shards) + bodies = [b for b in bodies if not (not b["parity"] and b["index"] == drop_index)] + rng.shuffle(bodies) + + assembled: list[bytes] = [] + + def _fake_on_packet(data: bytes, packet=None) -> None: + assembled.append(data) + + monkeypatch.setattr(bridge, "_on_packet", _fake_on_packet) + + def _deliver(body: dict) -> None: + time.sleep(rng.uniform(0.0, 0.03)) + bridge._handle_infonet_shard(body) + + threads = [threading.Thread(target=_deliver, args=(body,)) for body in bodies] + for t in threads: + t.start() + for t in threads: + t.join() + + assert assembled + assert assembled[-1] == payload + + +def test_rns_publish_gate_event_freezes_current_v1_signer_bundle(monkeypatch) -> None: + from services import config as config_mod + from services.mesh import mesh_rns as mesh_rns_mod + + bridge = RNSBridge() + sent: list[tuple[bytes, str | None]] = [] + settings = SimpleNamespace( + MESH_PEER_PUSH_SECRET="peer-secret", + MESH_RNS_MAX_PAYLOAD=8192, + MESH_RNS_DANDELION_DELAY_MS=0, + ) + + monkeypatch.setattr(config_mod, "get_settings", lambda: settings) + monkeypatch.setattr(mesh_rns_mod, "get_settings", lambda: settings) + monkeypatch.setattr(bridge, "enabled", lambda: True) + monkeypatch.setattr(bridge, "_maybe_rotate_session", lambda: None) + monkeypatch.setattr(bridge, "_seen", lambda _message_id: False) + monkeypatch.setattr(bridge, "_make_message_id", lambda prefix: f"{prefix}-wire-id") + monkeypatch.setattr(bridge, "_dandelion_hops", lambda: 3) + monkeypatch.setattr(bridge, "_pick_stem_peer", lambda: None) + monkeypatch.setattr( + bridge, + "_send_diffuse", + lambda payload, exclude=None: sent.append((payload, exclude)), + ) + + bridge.publish_gate_event( + "finance", + { + "event_type": "gate_message", + "timestamp": 1710000000, + "event_id": "gate-evt-1", + "node_id": "!gate-persona-1", + "sequence": 19, + "signature": "deadbeef", + "public_key": "pubkey-1", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "payload": { + "gate": "finance", + "ciphertext": "abc123", + "format": "mls1", + "nonce": "nonce-7", + "sender_ref": "sender-ref-7", + "epoch": 4, + }, + }, + ) + + assert len(sent) == 1 + message, exclude = sent[0] + decoded = json.loads(message.decode("utf-8")) + event = decoded["body"]["event"] + + assert exclude is None + assert decoded["type"] == "gate_event" + assert decoded["meta"] == { + "message_id": "gate-wire-id", + "dandelion": {"phase": "stem", "hops": 0, "max_hops": 3}, + } + assert set(event.keys()) == { + "event_type", + "timestamp", + "payload", + "event_id", + "node_id", + "sequence", + "signature", + "public_key", + "public_key_algo", + "protocol_version", + } + assert event["event_id"] == "gate-evt-1" + assert event["node_id"] == "!gate-persona-1" + assert event["sequence"] == 19 + assert event["signature"] == "deadbeef" + assert event["public_key"] == "pubkey-1" + assert event["public_key_algo"] == "Ed25519" + assert event["protocol_version"] == "infonet/2" + assert set(event["payload"].keys()) == {"ciphertext", "format", "gate_ref", "nonce", "sender_ref", "epoch"} + assert event["payload"]["ciphertext"] == "abc123" + assert event["payload"]["format"] == "mls1" + assert event["payload"]["nonce"] == "nonce-7" + assert event["payload"]["sender_ref"] == "sender-ref-7" + assert event["payload"]["epoch"] == 4 + assert event["payload"]["gate_ref"] + assert "gate" not in event["payload"] + + +def test_rns_inbound_gate_event_resolves_gate_ref_before_local_ingest(monkeypatch) -> None: + from services import config as config_mod + from services.mesh import mesh_hashchain as mesh_hashchain_mod, mesh_rns as mesh_rns_mod + + bridge = RNSBridge() + ingested: list[tuple[str, list[dict]]] = [] + settings = SimpleNamespace(MESH_RNS_DANDELION_HOPS=3) + + monkeypatch.setattr(config_mod, "get_settings", lambda: settings) + monkeypatch.setattr(mesh_rns_mod, "get_settings", lambda: settings) + monkeypatch.setattr(bridge, "_seen", lambda _message_id: False) + monkeypatch.setattr(mesh_hashchain_mod, "resolve_gate_wire_ref", lambda gate_ref, event: "finance") + monkeypatch.setattr( + mesh_hashchain_mod.gate_store, + "ingest_peer_events", + lambda gate_id, events: ingested.append((gate_id, events)) or {"accepted": 1, "duplicates": 0, "rejected": 0}, + ) + + packet = mesh_rns_mod.RNSMessage( + msg_type="gate_event", + body={ + "event": { + "event_type": "gate_message", + "timestamp": 1710000000, + "event_id": "gate-evt-inbound", + "node_id": "!gate-persona-1", + "sequence": 9, + "signature": "deadbeef", + "public_key": "pubkey-1", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "payload": { + "ciphertext": "abc123", + "format": "mls1", + "nonce": "nonce-7", + "sender_ref": "sender-ref-7", + "epoch": 4, + "gate_ref": "opaque-ref-1", + }, + } + }, + meta={"message_id": "gate-inbound-1", "dandelion": {"phase": "diffuse"}}, + ).encode() + + bridge._on_packet(packet) + + assert len(ingested) == 1 + gate_id, events = ingested[0] + assert gate_id == "finance" + assert len(events) == 1 + event = events[0] + assert event["event_id"] == "gate-evt-inbound" + assert event["node_id"] == "!gate-persona-1" + assert event["sequence"] == 9 + assert event["signature"] == "deadbeef" + assert event["public_key"] == "pubkey-1" + assert event["public_key_algo"] == "Ed25519" + assert event["protocol_version"] == "infonet/2" + assert event["payload"]["gate"] == "finance" + assert event["payload"]["gate_ref"] == "opaque-ref-1" + assert event["payload"]["ciphertext"] == "abc123" + assert event["payload"]["nonce"] == "nonce-7" + assert event["payload"]["sender_ref"] == "sender-ref-7" + assert event["payload"]["epoch"] == 4 + + +def test_rns_inbound_gate_event_blind_forwards_when_gate_cannot_be_resolved(monkeypatch) -> None: + from services import config as config_mod + from services.mesh import mesh_hashchain as mesh_hashchain_mod, mesh_rns as mesh_rns_mod + + bridge = RNSBridge() + forwarded: list[tuple[str, dict]] = [] + ingested: list[tuple[str, list[dict]]] = [] + settings = SimpleNamespace(MESH_RNS_DANDELION_HOPS=3) + + monkeypatch.setattr(config_mod, "get_settings", lambda: settings) + monkeypatch.setattr(mesh_rns_mod, "get_settings", lambda: settings) + monkeypatch.setattr(bridge, "_seen", lambda _message_id: False) + monkeypatch.setattr(bridge, "_pick_stem_peer", lambda: "peer-stem") + monkeypatch.setattr( + bridge, + "_send_to_peer", + lambda peer, payload: forwarded.append((peer, json.loads(payload.decode("utf-8")))), + ) + monkeypatch.setattr(mesh_hashchain_mod, "resolve_gate_wire_ref", lambda gate_ref, event: "") + monkeypatch.setattr( + mesh_hashchain_mod.gate_store, + "ingest_peer_events", + lambda gate_id, events: ingested.append((gate_id, events)) or {"accepted": 1, "duplicates": 0, "rejected": 0}, + ) + + original_event = { + "event_type": "gate_message", + "timestamp": 1710000000, + "event_id": "gate-evt-blind", + "node_id": "!gate-persona-1", + "sequence": 9, + "signature": "deadbeef", + "public_key": "pubkey-1", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "payload": { + "ciphertext": "abc123", + "format": "mls1", + "nonce": "nonce-7", + "sender_ref": "sender-ref-7", + "epoch": 4, + "gate_ref": "opaque-ref-1", + }, + } + packet = mesh_rns_mod.RNSMessage( + msg_type="gate_event", + body={"event": original_event}, + meta={"message_id": "gate-inbound-2", "dandelion": {"phase": "stem", "hops": 0, "max_hops": 2}}, + ).encode() + + bridge._on_packet(packet) + + assert ingested == [] + assert len(forwarded) == 1 + peer, forwarded_msg = forwarded[0] + assert peer == "peer-stem" + assert forwarded_msg["type"] == "gate_event" + assert forwarded_msg["meta"] == { + "message_id": "gate-inbound-2", + "dandelion": {"phase": "stem", "hops": 1, "max_hops": 2}, + } + assert forwarded_msg["body"]["event"] == original_event diff --git a/backend/tests/mesh/test_mesh_rns_private_dm.py b/backend/tests/mesh/test_mesh_rns_private_dm.py new file mode 100644 index 00000000..c11b1fc5 --- /dev/null +++ b/backend/tests/mesh/test_mesh_rns_private_dm.py @@ -0,0 +1,909 @@ +import asyncio +import base64 +import hashlib +import hmac +import time + +from httpx import ASGITransport, AsyncClient + +import main +from services.config import get_settings +from services.mesh.mesh_crypto import derive_node_id +from services.mesh import mesh_dm_relay, mesh_hashchain, mesh_rns + + +def _fresh_relay(tmp_path, monkeypatch): + from services import wormhole_supervisor + + monkeypatch.setattr(mesh_dm_relay, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_dm_relay, "RELAY_FILE", tmp_path / "dm_relay.json") + monkeypatch.setattr( + wormhole_supervisor, + "get_wormhole_state", + lambda: {"configured": True, "ready": True, "arti_ready": True, "rns_ready": True}, + ) + get_settings.cache_clear() + relay = mesh_dm_relay.DMRelay() + monkeypatch.setattr(mesh_dm_relay, "dm_relay", relay) + return relay + + +def _post(path: str, payload: dict): + async def _run(): + async with AsyncClient(transport=ASGITransport(app=main.app), base_url="http://test") as ac: + return await ac.post(path, json=payload) + + return asyncio.run(_run()) + + +class _FakeInfonet: + def __init__(self): + self.appended = [] + self.sequences = {} + + def append(self, **kwargs): + self.appended.append(kwargs) + + def validate_and_set_sequence(self, node_id, sequence): + last = self.sequences.get(node_id, 0) + if sequence <= last: + return False, f"Replay detected: sequence {sequence} <= last {last}" + self.sequences[node_id] = sequence + return True, "" + + +class _DirectRNS: + def __init__(self, send_result=True, direct_messages=None, direct_ids=None): + self.send_result = send_result + self.sent = [] + self.direct_messages = list(direct_messages or []) + self.direct_ids_value = set(direct_ids or []) + + def send_private_dm(self, *, mailbox_key, envelope): + self.sent.append({"mailbox_key": mailbox_key, "envelope": envelope}) + return self.send_result + + def collect_private_dm(self, mailbox_keys): + return list(self.direct_messages) + + def private_dm_ids(self, mailbox_keys): + return set(self.direct_ids_value) + + def count_private_dm(self, mailbox_keys): + return len(self.direct_ids_value) + + +TEST_PUBLIC_KEY = base64.b64encode(b"0" * 32).decode("ascii") +TEST_SENDER_ID = derive_node_id(TEST_PUBLIC_KEY) +REQUEST_CLAIMS = [{"type": "requests", "token": "request-claim-token"}] +NOW_TS = lambda: int(time.time()) + + +def test_secure_dm_send_prefers_reticulum(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + direct_rns = _DirectRNS(send_result=True) + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: True) + monkeypatch.setattr(main, "_rns_private_dm_ready", lambda: True) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "msg_id": "msg-reticulum-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 7, + "protocol_version": "infonet/2", + }, + ) + + body = response.json() + assert response.status_code == 200 + assert body["ok"] is True + assert body["transport"] == "reticulum" + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 0 + assert len(direct_rns.sent) == 1 + assert direct_rns.sent[0]["envelope"]["msg_id"] == "msg-reticulum-1" + assert len(infonet.appended) == 0 + + +def test_secure_dm_send_falls_back_to_relay(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + direct_rns = _DirectRNS(send_result=False) + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: True) + monkeypatch.setattr(main, "_rns_private_dm_ready", lambda: True) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "msg_id": "msg-relay-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 8, + "protocol_version": "infonet/2", + }, + ) + + body = response.json() + assert response.status_code == 200 + assert body["ok"] is True + assert body["transport"] == "relay" + assert "relay fallback" in body["detail"].lower() + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 1 + assert len(infonet.appended) == 0 + + +def test_request_sender_seal_reduces_relay_sender_handle_on_fallback(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + direct_rns = _DirectRNS(send_result=False) + relay_salt = "0123456789abcdef0123456789abcdef" + expected_sender = "sealed:" + hmac.new( + bytes.fromhex(relay_salt), TEST_SENDER_ID.encode("utf-8"), hashlib.sha256 + ).hexdigest()[:16] + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: True) + monkeypatch.setattr(main, "_rns_private_dm_ready", lambda: True) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "sender_seal": "v3:test-seal", + "relay_salt": relay_salt, + "msg_id": "msg-relay-sealed-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 18, + "protocol_version": "infonet/2", + }, + ) + + body = response.json() + assert response.status_code == 200 + assert body["ok"] is True + assert body["transport"] == "relay" + messages = relay.collect_claims("!sb_recipient1234", REQUEST_CLAIMS) + assert [msg["msg_id"] for msg in messages] == ["msg-relay-sealed-1"] + assert messages[0]["sender_id"] == expected_sender + assert messages[0]["sender_id"] != TEST_SENDER_ID + assert messages[0]["sender_seal"] == "v3:test-seal" + + +def test_request_sender_seal_reduces_direct_rns_sender_handle(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + direct_rns = _DirectRNS(send_result=True) + relay_salt = "fedcba9876543210fedcba9876543210" + expected_sender = "sealed:" + hmac.new( + bytes.fromhex(relay_salt), TEST_SENDER_ID.encode("utf-8"), hashlib.sha256 + ).hexdigest()[:16] + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: True) + monkeypatch.setattr(main, "_rns_private_dm_ready", lambda: True) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "sender_seal": "v3:test-seal", + "relay_salt": relay_salt, + "msg_id": "msg-direct-sealed-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 19, + "protocol_version": "infonet/2", + }, + ) + + body = response.json() + assert response.status_code == 200 + assert body["ok"] is True + assert body["transport"] == "reticulum" + assert len(direct_rns.sent) == 1 + assert direct_rns.sent[0]["envelope"]["sender_id"] == expected_sender + assert direct_rns.sent[0]["envelope"]["sender_id"] != TEST_SENDER_ID + assert direct_rns.sent[0]["envelope"]["sender_seal"] == "v3:test-seal" + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 0 + + +def test_request_sender_block_prevents_direct_rns_delivery(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + direct_rns = _DirectRNS(send_result=True) + relay.block("!sb_recipient1234", TEST_SENDER_ID) + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: True) + monkeypatch.setattr(main, "_rns_private_dm_ready", lambda: True) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "sender_seal": "v3:test-seal", + "relay_salt": "00112233445566778899aabbccddeeff", + "msg_id": "msg-direct-blocked-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 20, + "protocol_version": "infonet/2", + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": False, "detail": "Recipient is not accepting your messages"} + assert len(direct_rns.sent) == 0 + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 0 + + +def test_request_sender_seal_respects_raw_sender_block_on_relay_send_path(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + relay.block("!sb_recipient1234", TEST_SENDER_ID) + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "sender_seal": "v3:test-seal", + "relay_salt": "00112233445566778899aabbccddeeff", + "msg_id": "msg-blocked-sealed-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 20, + "protocol_version": "infonet/2", + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": False, "detail": "Recipient is not accepting your messages"} + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 0 + + +def test_secure_dm_send_rejects_replayed_msg_id_nonce(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + + payload = { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "msg_id": "msg-replay-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 14, + "protocol_version": "infonet/2", + } + + first = _post("/api/mesh/dm/send", payload) + second = _post("/api/mesh/dm/send", payload) + + assert first.status_code == 200 + assert first.json()["ok"] is True + assert second.status_code == 200 + assert second.json() == {"ok": False, "detail": "nonce replay detected"} + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 1 + + +def test_secure_dm_send_rejects_replayed_sequence_with_new_nonce(tmp_path, monkeypatch): + _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + + first = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "msg_id": "msg-seq-1", + "nonce": "nonce-seq-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 15, + "protocol_version": "infonet/2", + }, + ) + second = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext-again", + "msg_id": "msg-seq-2", + "nonce": "nonce-seq-2", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 15, + "protocol_version": "infonet/2", + }, + ) + + assert first.status_code == 200 + assert first.json()["ok"] is True + assert second.status_code == 200 + assert second.json() == {"ok": False, "detail": "Replay detected: sequence 15 <= last 15"} + + +def test_secure_dm_send_does_not_consume_nonce_before_signature_verification(tmp_path, monkeypatch): + _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + consumed = {"count": 0} + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (False, "Invalid signature")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr( + mesh_dm_relay.dm_relay, + "consume_nonce", + lambda *_args, **_kwargs: consumed.__setitem__("count", consumed["count"] + 1) or (True, "ok"), + ) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "msg_id": "msg-invalid-sig", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 16, + "protocol_version": "infonet/2", + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": False, "detail": "Invalid signature"} + assert consumed["count"] == 0 + + +def test_anonymous_mode_dm_send_stays_off_reticulum(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + infonet = _FakeInfonet() + direct_rns = _DirectRNS(send_result=True) + + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: True) + monkeypatch.setattr(main, "_rns_private_dm_ready", lambda: True) + monkeypatch.setattr(main, "_anonymous_dm_hidden_transport_enforced", lambda: True) + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + response = _post( + "/api/mesh/dm/send", + { + "sender_id": TEST_SENDER_ID, + "recipient_id": "!sb_recipient1234", + "delivery_class": "request", + "ciphertext": "ciphertext", + "msg_id": "msg-anon-relay-1", + "timestamp": NOW_TS(), + "public_key": TEST_PUBLIC_KEY, + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 9, + "protocol_version": "infonet/2", + }, + ) + + body = response.json() + assert response.status_code == 200 + assert body["ok"] is True + assert body["transport"] == "relay" + assert "off direct transport" in body["detail"].lower() + assert relay.count_claims("!sb_recipient1234", REQUEST_CLAIMS) == 1 + assert len(direct_rns.sent) == 0 + assert len(infonet.appended) == 0 + + +def test_secure_dm_poll_and_count_merge_relay_and_reticulum(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-dup", + msg_id="dup", + delivery_class="request", + ) + relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-only", + msg_id="relay-only", + delivery_class="request", + ) + + direct_rns = _DirectRNS( + direct_messages=[ + { + "sender_id": "sealed:1234", + "ciphertext": "cipher-direct-dup", + "timestamp": 100.0, + "msg_id": "dup", + "delivery_class": "request", + "sender_seal": "", + "transport": "reticulum", + }, + { + "sender_id": "sealed:1234", + "ciphertext": "cipher-direct-only", + "timestamp": 101.0, + "msg_id": "direct-only", + "delivery_class": "request", + "sender_seal": "", + "transport": "reticulum", + }, + ], + direct_ids={"dup", "direct-only"}, + ) + infonet = _FakeInfonet() + + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: (True, "", {"mailbox_claims": REQUEST_CLAIMS}), + ) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + poll_response = _post( + "/api/mesh/dm/poll", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-poll", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 10, + "protocol_version": "infonet/2", + }, + ) + poll_body = poll_response.json() + assert poll_response.status_code == 200 + assert poll_body["ok"] is True + assert poll_body["count"] == 3 + assert {msg["msg_id"] for msg in poll_body["messages"]} == {"dup", "relay-only", "direct-only"} + dup_message = next(msg for msg in poll_body["messages"] if msg["msg_id"] == "dup") + assert dup_message["sender_id"] == "alice" + assert dup_message["ciphertext"] == "cipher-relay-dup" + + count_response = _post( + "/api/mesh/dm/count", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-count", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 11, + "protocol_version": "infonet/2", + }, + ) + count_body = count_response.json() + assert count_response.status_code == 200 + assert count_body["ok"] is True + assert count_body["count"] == 2 + + +def test_secure_dm_poll_marks_reduced_v3_request_recovery_fields(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + relay.deposit( + sender_id="sealed:relayv3", + raw_sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-v3", + msg_id="relay-v3", + delivery_class="request", + sender_seal="v3:relay-seal", + ) + relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher-legacy", + msg_id="legacy-raw", + delivery_class="request", + ) + + direct_rns = _DirectRNS( + direct_messages=[ + { + "sender_id": "sealed:directv3", + "ciphertext": "cipher-direct-v3", + "timestamp": 101.0, + "msg_id": "direct-v3", + "delivery_class": "request", + "sender_seal": "v3:direct-seal", + "transport": "reticulum", + } + ], + direct_ids={"direct-v3"}, + ) + infonet = _FakeInfonet() + + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: (True, "", {"mailbox_claims": REQUEST_CLAIMS}), + ) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + poll_response = _post( + "/api/mesh/dm/poll", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-poll-markers", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 12, + "protocol_version": "infonet/2", + }, + ) + poll_body = poll_response.json() + + assert poll_response.status_code == 200 + assert poll_body["ok"] is True + assert poll_body["count"] == 3 + + by_id = {msg["msg_id"]: msg for msg in poll_body["messages"]} + + assert by_id["relay-v3"]["request_contract_version"] == "request-v2-reduced-v3" + assert by_id["relay-v3"]["sender_recovery_required"] is True + assert by_id["relay-v3"]["sender_recovery_state"] == "pending" + + assert by_id["direct-v3"]["request_contract_version"] == "request-v2-reduced-v3" + assert by_id["direct-v3"]["sender_recovery_required"] is True + assert by_id["direct-v3"]["sender_recovery_state"] == "pending" + + assert "request_contract_version" not in by_id["legacy-raw"] + assert "sender_recovery_required" not in by_id["legacy-raw"] + assert "sender_recovery_state" not in by_id["legacy-raw"] + + +def test_secure_dm_poll_prefers_canonical_v2_duplicate_over_legacy_raw(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-raw", + msg_id="dup-v2-over-raw", + delivery_class="request", + ) + + direct_rns = _DirectRNS( + direct_messages=[ + { + "sender_id": "sealed:directv3", + "ciphertext": "cipher-direct-v3", + "timestamp": 101.0, + "msg_id": "dup-v2-over-raw", + "delivery_class": "request", + "sender_seal": "v3:direct-seal", + "transport": "reticulum", + } + ], + direct_ids={"dup-v2-over-raw"}, + ) + infonet = _FakeInfonet() + + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: (True, "", {"mailbox_claims": REQUEST_CLAIMS}), + ) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + poll_response = _post( + "/api/mesh/dm/poll", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-poll-v2-over-raw", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 13, + "protocol_version": "infonet/2", + }, + ) + poll_body = poll_response.json() + + assert poll_response.status_code == 200 + assert poll_body["ok"] is True + assert poll_body["count"] == 1 + message = poll_body["messages"][0] + assert message["msg_id"] == "dup-v2-over-raw" + assert message["sender_id"] == "sealed:directv3" + assert message["ciphertext"] == "cipher-direct-v3" + assert message["transport"] == "reticulum" + assert message["request_contract_version"] == "request-v2-reduced-v3" + assert message["sender_recovery_required"] is True + assert message["sender_recovery_state"] == "pending" + + +def test_secure_dm_poll_prefers_legacy_raw_duplicate_over_legacy_sealed(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + relay.deposit( + sender_id="sealed:relaylegacy", + raw_sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-sealed", + msg_id="dup-raw-over-sealed", + delivery_class="request", + sender_seal="v2:legacy-seal", + ) + + direct_rns = _DirectRNS( + direct_messages=[ + { + "sender_id": "alice", + "ciphertext": "cipher-direct-raw", + "timestamp": 101.0, + "msg_id": "dup-raw-over-sealed", + "delivery_class": "request", + "sender_seal": "", + "transport": "reticulum", + } + ], + direct_ids={"dup-raw-over-sealed"}, + ) + infonet = _FakeInfonet() + + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: (True, "", {"mailbox_claims": REQUEST_CLAIMS}), + ) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + poll_response = _post( + "/api/mesh/dm/poll", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-poll-raw-over-sealed", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 14, + "protocol_version": "infonet/2", + }, + ) + poll_body = poll_response.json() + + assert poll_response.status_code == 200 + assert poll_body["ok"] is True + assert poll_body["count"] == 1 + message = poll_body["messages"][0] + assert message["msg_id"] == "dup-raw-over-sealed" + assert message["sender_id"] == "alice" + assert message["ciphertext"] == "cipher-direct-raw" + assert message["transport"] == "reticulum" + assert "request_contract_version" not in message + assert "sender_recovery_required" not in message + assert "sender_recovery_state" not in message + + +def test_secure_dm_poll_keeps_relay_copy_for_same_contract_v2_duplicate(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + relay.deposit( + sender_id="sealed:sharedv3", + raw_sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-v3-dup", + msg_id="dup-v2-tie", + delivery_class="request", + sender_seal="v3:relay-seal", + ) + + direct_rns = _DirectRNS( + direct_messages=[ + { + "sender_id": "sealed:sharedv3", + "ciphertext": "cipher-direct-v3-dup", + "timestamp": 101.0, + "msg_id": "dup-v2-tie", + "delivery_class": "request", + "sender_seal": "v3:relay-seal", + "transport": "reticulum", + } + ], + direct_ids={"dup-v2-tie"}, + ) + infonet = _FakeInfonet() + + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: (True, "", {"mailbox_claims": REQUEST_CLAIMS}), + ) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + poll_response = _post( + "/api/mesh/dm/poll", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-poll-v2-tie", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 15, + "protocol_version": "infonet/2", + }, + ) + poll_body = poll_response.json() + + assert poll_response.status_code == 200 + assert poll_body["ok"] is True + assert poll_body["count"] == 1 + message = poll_body["messages"][0] + assert message["msg_id"] == "dup-v2-tie" + assert message["sender_id"] == "sealed:sharedv3" + assert message["ciphertext"] == "cipher-relay-v3-dup" + assert "transport" not in message + assert message["request_contract_version"] == "request-v2-reduced-v3" + assert message["sender_recovery_required"] is True + assert message["sender_recovery_state"] == "pending" + + +def test_anonymous_mode_poll_and_count_ignore_reticulum(tmp_path, monkeypatch): + relay = _fresh_relay(tmp_path, monkeypatch) + relay.deposit( + sender_id="alice", + recipient_id="bob", + ciphertext="cipher-relay-only", + msg_id="relay-only", + delivery_class="request", + ) + + direct_rns = _DirectRNS( + direct_messages=[ + { + "sender_id": "sealed:1234", + "ciphertext": "cipher-direct-only", + "timestamp": 101.0, + "msg_id": "direct-only", + "delivery_class": "request", + "sender_seal": "", + "transport": "reticulum", + }, + ], + direct_ids={"direct-only"}, + ) + infonet = _FakeInfonet() + + monkeypatch.setattr( + main, + "_verify_dm_mailbox_request", + lambda **_kwargs: (True, "", {"mailbox_claims": REQUEST_CLAIMS}), + ) + monkeypatch.setattr(main, "_anonymous_dm_hidden_transport_enforced", lambda: True) + monkeypatch.setattr(mesh_hashchain, "infonet", infonet) + monkeypatch.setattr(mesh_rns, "rns_bridge", direct_rns) + + poll_response = _post( + "/api/mesh/dm/poll", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-poll-anon", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 12, + "protocol_version": "infonet/2", + }, + ) + poll_body = poll_response.json() + assert poll_response.status_code == 200 + assert poll_body["ok"] is True + assert poll_body["count"] == 1 + assert {msg["msg_id"] for msg in poll_body["messages"]} == {"relay-only"} + + count_response = _post( + "/api/mesh/dm/count", + { + "agent_id": "bob", + "mailbox_claims": REQUEST_CLAIMS, + "timestamp": NOW_TS(), + "nonce": "nonce-count-anon", + "public_key": "pub", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 13, + "protocol_version": "infonet/2", + }, + ) + count_body = count_response.json() + assert count_response.status_code == 200 + assert count_body["ok"] is True + assert count_body["count"] == 0 diff --git a/backend/tests/mesh/test_mesh_secure_storage.py b/backend/tests/mesh/test_mesh_secure_storage.py new file mode 100644 index 00000000..5279c821 --- /dev/null +++ b/backend/tests/mesh/test_mesh_secure_storage.py @@ -0,0 +1,225 @@ +import json +import os +import subprocess +import sys +from types import SimpleNamespace + + +def _reset_secure_storage_state(mesh_secure_storage) -> None: + mesh_secure_storage._MASTER_KEY_CACHE = None + mesh_secure_storage._DOMAIN_KEY_CACHE.clear() + + +def test_secure_storage_encrypts_and_reads_json(tmp_path, monkeypatch): + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + path = tmp_path / "secret.json" + mesh_secure_storage.write_secure_json(path, {"alpha": 1, "bravo": "two"}) + + raw = json.loads(path.read_text(encoding="utf-8")) + assert raw["kind"] == "sb_secure_json" + assert "alpha" not in path.read_text(encoding="utf-8") + + data = mesh_secure_storage.read_secure_json(path, lambda: {}) + assert data == {"alpha": 1, "bravo": "two"} + + +def test_secure_storage_migrates_plaintext_json(tmp_path, monkeypatch): + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + path = tmp_path / "legacy.json" + path.write_text(json.dumps({"legacy": True}), encoding="utf-8") + + data = mesh_secure_storage.read_secure_json(path, lambda: {}) + assert data == {"legacy": True} + + migrated = json.loads(path.read_text(encoding="utf-8")) + assert migrated["kind"] == "sb_secure_json" + + +def test_secure_storage_fails_closed_on_decrypt_error(tmp_path, monkeypatch): + import pytest + + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + path = tmp_path / "corrupt.json" + mesh_secure_storage.write_secure_json(path, {"secret": "value"}) + payload = json.loads(path.read_text(encoding="utf-8")) + payload["ciphertext"] = payload["ciphertext"][:-4] + "AAAA" + path.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(mesh_secure_storage.SecureStorageError): + mesh_secure_storage.read_secure_json(path, lambda: {}) + + +def test_secure_storage_round_trips_across_process_boundary(tmp_path, monkeypatch): + if os.name != "nt": + return + + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + path = tmp_path / "cross-process.json" + mesh_secure_storage.write_secure_json(path, {"alpha": 7, "bravo": "cross-process"}) + backend_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + + script = f""" +import json +from pathlib import Path +from services.mesh import mesh_secure_storage +mesh_secure_storage.DATA_DIR = Path(r"{tmp_path}") +mesh_secure_storage.MASTER_KEY_FILE = Path(r"{tmp_path / 'wormhole_secure_store.key'}") +print(json.dumps(mesh_secure_storage.read_secure_json(r"{path}", lambda: {{}}))) +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=backend_root, + capture_output=True, + text=True, + env={**os.environ.copy(), "PYTHONPATH": backend_root}, + check=True, + ) + + assert json.loads(result.stdout.strip()) == {"alpha": 7, "bravo": "cross-process"} + + +def test_domain_storage_isolation_keeps_gate_and_dm_data_separate(tmp_path, monkeypatch): + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + mesh_secure_storage.write_domain_json("gate_persona", "gate.json", {"gate": "alpha"}) + mesh_secure_storage.write_domain_json("dm_alias", "dm.json", {"alias": "bravo"}) + + gate_data = mesh_secure_storage.read_domain_json("gate_persona", "gate.json", lambda: {}) + dm_data = mesh_secure_storage.read_domain_json("dm_alias", "dm.json", lambda: {}) + + assert gate_data == {"gate": "alpha"} + assert dm_data == {"alias": "bravo"} + assert (tmp_path / "gate_persona" / "gate.json").exists() + assert (tmp_path / "dm_alias" / "dm.json").exists() + + +def test_domain_storage_uses_independent_domain_key_files(tmp_path, monkeypatch): + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + mesh_secure_storage.write_domain_json("gate_persona", "gate.json", {"gate": "alpha"}) + mesh_secure_storage.write_domain_json("dm_alias", "dm.json", {"alias": "bravo"}) + + gate_key = tmp_path / "_domain_keys" / "gate_persona.key" + dm_key = tmp_path / "_domain_keys" / "dm_alias.key" + + assert gate_key.exists() + assert dm_key.exists() + assert gate_key.read_text(encoding="utf-8") != dm_key.read_text(encoding="utf-8") + assert not mesh_secure_storage.MASTER_KEY_FILE.exists() + + +def test_domain_storage_migrates_legacy_master_derived_ciphertext(tmp_path, monkeypatch): + import pytest + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + domain = "gate_persona" + filename = "legacy.json" + payload = {"legacy": True} + file_path = tmp_path / domain / filename + file_path.parent.mkdir(parents=True, exist_ok=True) + + nonce = os.urandom(12) + ciphertext = AESGCM(mesh_secure_storage._derive_legacy_domain_key(domain)).encrypt( + nonce, + mesh_secure_storage._stable_json(payload), + mesh_secure_storage._domain_aad(domain, filename), + ) + envelope = mesh_secure_storage._secure_envelope(file_path, nonce, ciphertext) + file_path.write_text(json.dumps(envelope), encoding="utf-8") + _reset_secure_storage_state(mesh_secure_storage) + + data = mesh_secure_storage.read_domain_json(domain, filename, lambda: {}) + + assert data == payload + assert (tmp_path / "_domain_keys" / f"{domain}.key").exists() + + migrated = json.loads(file_path.read_text(encoding="utf-8")) + with pytest.raises(Exception): + AESGCM(mesh_secure_storage._derive_legacy_domain_key(domain)).decrypt( + mesh_secure_storage._unb64(migrated["nonce"]), + mesh_secure_storage._unb64(migrated["ciphertext"]), + mesh_secure_storage._domain_aad(domain, filename), + ) + + +def test_domain_storage_rejects_path_traversal(tmp_path, monkeypatch): + import pytest + + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + _reset_secure_storage_state(mesh_secure_storage) + + with pytest.raises(mesh_secure_storage.SecureStorageError): + mesh_secure_storage._domain_file_path("../../etc", "passwd") + + +def test_raw_fallback_requires_explicit_opt_in_not_debug(monkeypatch): + from services import config as config_mod + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "_is_windows", lambda: False) + monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False) + monkeypatch.setattr( + config_mod, + "get_settings", + lambda: SimpleNamespace( + MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK=False, + MESH_DEBUG_MODE=True, + ), + ) + + assert mesh_secure_storage._raw_fallback_allowed() is False + + +def test_raw_fallback_allows_explicit_opt_in(monkeypatch): + from services import config as config_mod + from services.mesh import mesh_secure_storage + + monkeypatch.setattr(mesh_secure_storage, "_is_windows", lambda: False) + monkeypatch.delenv("PYTEST_CURRENT_TEST", raising=False) + monkeypatch.setattr( + config_mod, + "get_settings", + lambda: SimpleNamespace( + MESH_ALLOW_RAW_SECURE_STORAGE_FALLBACK=True, + MESH_DEBUG_MODE=False, + ), + ) + + assert mesh_secure_storage._raw_fallback_allowed() is True diff --git a/backend/tests/mesh/test_mesh_sensitive_no_store.py b/backend/tests/mesh/test_mesh_sensitive_no_store.py new file mode 100644 index 00000000..f8b1424b --- /dev/null +++ b/backend/tests/mesh/test_mesh_sensitive_no_store.py @@ -0,0 +1,73 @@ +import asyncio + + +class TestSensitiveBackendNoStore: + def test_mesh_status_sets_privacy_security_headers(self, client): + r = client.get("/api/mesh/infonet/status") + assert r.status_code == 200 + assert "default-src 'self'" in (r.headers.get("content-security-policy") or "") + assert (r.headers.get("x-frame-options") or "").upper() == "DENY" + assert (r.headers.get("x-content-type-options") or "").lower() == "nosniff" + assert (r.headers.get("referrer-policy") or "").lower() == "no-referrer" + + def test_wormhole_status_is_no_store(self, client): + r = client.get("/api/wormhole/status") + assert r.status_code == 200 + assert "no-store" in (r.headers.get("cache-control") or "").lower() + + def test_settings_privacy_profile_is_no_store(self, client): + r = client.get("/api/settings/privacy-profile") + assert r.status_code == 200 + assert "no-store" in (r.headers.get("cache-control") or "").lower() + + def test_settings_wormhole_is_no_store(self, client): + r = client.get("/api/settings/wormhole") + assert r.status_code == 200 + assert "no-store" in (r.headers.get("cache-control") or "").lower() + + def test_settings_wormhole_status_is_no_store(self, client): + r = client.get("/api/settings/wormhole-status") + assert r.status_code == 200 + assert "no-store" in (r.headers.get("cache-control") or "").lower() + + def test_dm_pubkey_is_no_store_even_on_failure(self, client): + r = client.get("/api/mesh/dm/pubkey?agent_id=missing") + assert r.status_code == 200 + body = r.json() + assert body["ok"] is False + assert "no-store" in (r.headers.get("cache-control") or "").lower() + + def test_anonymous_mode_blocked_dm_send_is_no_store(self, client, monkeypatch): + import main + from services import wormhole_settings, wormhole_status + + monkeypatch.setattr( + wormhole_settings, + "read_wormhole_settings", + lambda: { + "enabled": True, + "privacy_profile": "default", + "transport": "direct", + "anonymous_mode": True, + }, + ) + monkeypatch.setattr( + wormhole_status, + "read_wormhole_status", + lambda: { + "running": True, + "ready": True, + "transport_active": "direct", + }, + ) + + async def _post(): + from httpx import ASGITransport, AsyncClient + + transport = ASGITransport(app=main.app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + return await ac.post("/api/mesh/dm/send", json={}) + + response = asyncio.run(_post()) + assert response.status_code == 428 + assert "no-store" in (response.headers.get("cache-control") or "").lower() diff --git a/backend/tests/mesh/test_mesh_wormhole_endpoint_boundary.py b/backend/tests/mesh/test_mesh_wormhole_endpoint_boundary.py new file mode 100644 index 00000000..9349dee7 --- /dev/null +++ b/backend/tests/mesh/test_mesh_wormhole_endpoint_boundary.py @@ -0,0 +1,63 @@ +def test_wormhole_identity_allows_local_operator_without_admin_key(client, monkeypatch): + import main + + monkeypatch.setattr(main, "_current_admin_key", lambda: "test-key") + monkeypatch.setattr(main, "_allow_insecure_admin", lambda: False) + monkeypatch.setattr( + main, + "get_transport_identity", + lambda: { + "node_id": "transport-node", + "public_key": "pub", + "public_key_algo": "Ed25519", + }, + ) + + allowed = client.get("/api/wormhole/identity") + assert allowed.status_code == 200 + assert allowed.json()["node_id"] == "transport-node" + + +def test_wormhole_gate_identity_allows_local_operator_without_admin_key(client, monkeypatch): + import main + + monkeypatch.setattr(main, "_current_admin_key", lambda: "test-key") + monkeypatch.setattr(main, "_allow_insecure_admin", lambda: False) + monkeypatch.setattr( + main, + "get_active_gate_identity", + lambda gate_id: { + "ok": True, + "gate_id": gate_id, + "identity": {"node_id": "gate-node", "scope": "gate_session"}, + }, + ) + + allowed = client.get("/api/wormhole/gate/journalists/identity") + assert allowed.status_code == 200 + body = allowed.json() + assert body["gate_id"] == "journalists" + assert body["identity"]["node_id"] == "gate-node" + + +def test_wormhole_gate_personas_allows_local_operator_without_admin_key(client, monkeypatch): + import main + + monkeypatch.setattr(main, "_current_admin_key", lambda: "test-key") + monkeypatch.setattr(main, "_allow_insecure_admin", lambda: False) + monkeypatch.setattr( + main, + "list_gate_personas", + lambda gate_id: { + "ok": True, + "gate_id": gate_id, + "active_persona_id": "", + "personas": [{"node_id": "persona-node", "scope": "gate_persona"}], + }, + ) + + allowed = client.get("/api/wormhole/gate/journalists/personas") + assert allowed.status_code == 200 + body = allowed.json() + assert body["gate_id"] == "journalists" + assert body["personas"][0]["node_id"] == "persona-node" diff --git a/backend/tests/mesh/test_mesh_wormhole_hardening.py b/backend/tests/mesh/test_mesh_wormhole_hardening.py new file mode 100644 index 00000000..41de93f7 --- /dev/null +++ b/backend/tests/mesh/test_mesh_wormhole_hardening.py @@ -0,0 +1,480 @@ +import base64 +import asyncio +import json +import time + +import pytest +from cryptography.hazmat.primitives.asymmetric import x25519 +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat +from starlette.requests import Request + + +def _fresh_mesh_state(tmp_path, monkeypatch): + from services.mesh import ( + mesh_dm_relay, + mesh_secure_storage, + mesh_wormhole_identity, + mesh_wormhole_persona, + ) + + monkeypatch.setattr(mesh_dm_relay, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_dm_relay, "RELAY_FILE", tmp_path / "dm_relay.json") + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + relay = mesh_dm_relay.DMRelay() + monkeypatch.setattr(mesh_dm_relay, "dm_relay", relay) + return relay, mesh_wormhole_identity + + +def _json_request(path: str, body: dict) -> Request: + payload = json.dumps(body).encode("utf-8") + sent = {"value": False} + + async def receive(): + if sent["value"]: + return {"type": "http.request", "body": b"", "more_body": False} + sent["value"] = True + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "headers": [(b"content-type", b"application/json")], + "client": ("test", 12345), + "method": "POST", + "path": path, + }, + receive, + ) + + +def test_sender_token_can_resolve_recipient_without_clear_recipient_id(tmp_path, monkeypatch): + _relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh.mesh_wormhole_sender_token import ( + consume_wormhole_dm_sender_token, + issue_wormhole_dm_sender_token, + ) + + issued = issue_wormhole_dm_sender_token( + recipient_id="peer123", + delivery_class="shared", + recipient_token="tok123", + ) + assert issued["ok"] + + consumed = consume_wormhole_dm_sender_token( + sender_token=issued["sender_token"], + recipient_id="", + delivery_class="shared", + recipient_token="tok123", + ) + assert consumed["ok"] + assert consumed["recipient_id"] == "peer123" + assert consumed["sender_token_hash"] + + +def test_signed_prekey_rotation_preserves_old_bootstrap_decrypt(tmp_path, monkeypatch): + _relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh.mesh_wormhole_prekey import ( + SIGNED_PREKEY_ROTATE_AFTER_S, + bootstrap_decrypt_from_sender, + bootstrap_encrypt_for_peer, + register_wormhole_prekey_bundle, + ) + + reg1 = register_wormhole_prekey_bundle(force_signed_prekey=True) + assert reg1["ok"] + agent_id = reg1["agent_id"] + + old_envelope = bootstrap_encrypt_for_peer(agent_id, "ACCESS_REQUEST:X25519:testpub|geo=1,2") + assert old_envelope["ok"] + + data = identity_mod.read_wormhole_identity() + data["signed_prekey_generated_at"] = int(time.time()) - SIGNED_PREKEY_ROTATE_AFTER_S - 10 + identity_mod._write_identity(data) + + reg2 = register_wormhole_prekey_bundle() + assert reg2["ok"] + assert reg2["bundle"]["signed_prekey_id"] != reg1["bundle"]["signed_prekey_id"] + + refreshed = identity_mod.read_wormhole_identity() + history = list(refreshed.get("signed_prekey_history") or []) + assert any(int(item.get("signed_prekey_id", 0) or 0) == reg1["bundle"]["signed_prekey_id"] for item in history) + + dec = bootstrap_decrypt_from_sender(agent_id, old_envelope["result"]) + assert dec["ok"] + assert dec["result"] == "ACCESS_REQUEST:X25519:testpub|geo=1,2" + + +def test_prekey_bundle_fetch_rejects_stale_or_tampered_bundle(tmp_path, monkeypatch): + relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh import mesh_wormhole_prekey as prekey_mod + + registered = prekey_mod.register_wormhole_prekey_bundle(force_signed_prekey=True) + assert registered["ok"] is True + agent_id = registered["agent_id"] + + fresh = prekey_mod.fetch_dm_prekey_bundle(agent_id) + assert fresh["ok"] is True + assert int(fresh["signed_at"]) > 0 + assert fresh["bundle_signature"] + + stored = relay.get_prekey_bundle(agent_id) + stale_bundle = dict(stored.get("bundle") or {}) + stale_bundle["signed_at"] = int(time.time()) - prekey_mod._max_prekey_bundle_age_s() - 10 + stale_bundle = prekey_mod._attach_bundle_signature(stale_bundle, signed_at=stale_bundle["signed_at"]) + relay._prekey_bundles[agent_id]["bundle"] = stale_bundle + + stale = prekey_mod.fetch_dm_prekey_bundle(agent_id) + assert stale == {"ok": False, "detail": "Prekey bundle is stale"} + + tampered_bundle = dict(stale_bundle) + tampered_bundle["signed_at"] = int(time.time()) + tampered_bundle = prekey_mod._attach_bundle_signature(tampered_bundle, signed_at=tampered_bundle["signed_at"]) + tampered_bundle["bundle_signature"] = "00" * 64 + relay._prekey_bundles[agent_id]["bundle"] = tampered_bundle + + tampered = prekey_mod.fetch_dm_prekey_bundle(agent_id) + assert tampered == {"ok": False, "detail": "Prekey bundle signature invalid"} + + +def test_prekey_bundle_fetch_rejects_future_dated_bundle(tmp_path, monkeypatch): + relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh import mesh_wormhole_prekey as prekey_mod + + registered = prekey_mod.register_wormhole_prekey_bundle(force_signed_prekey=True) + assert registered["ok"] is True + agent_id = registered["agent_id"] + + stored = relay.get_prekey_bundle(agent_id) + future_bundle = dict(stored.get("bundle") or {}) + future_bundle["signed_at"] = int(time.time()) + 301 + future_bundle = prekey_mod._attach_bundle_signature(future_bundle, signed_at=future_bundle["signed_at"]) + relay._prekey_bundles[agent_id]["bundle"] = future_bundle + + future = prekey_mod.fetch_dm_prekey_bundle(agent_id) + assert future == {"ok": False, "detail": "Prekey bundle signed_at is in the future"} + + +def test_remote_prekey_identity_is_pinned_and_detects_mismatch(tmp_path, monkeypatch): + _relay, _identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + from services.mesh import mesh_wormhole_contacts + + monkeypatch.setattr(mesh_wormhole_contacts, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_contacts, "CONTACTS_FILE", tmp_path / "wormhole_dm_contacts.json") + + pinned = mesh_wormhole_contacts.observe_remote_prekey_identity( + "peer-alpha", + fingerprint="aa" * 32, + sequence=3, + signed_at=111, + ) + same = mesh_wormhole_contacts.observe_remote_prekey_identity( + "peer-alpha", + fingerprint="aa" * 32, + sequence=4, + signed_at=222, + ) + changed = mesh_wormhole_contacts.observe_remote_prekey_identity( + "peer-alpha", + fingerprint="bb" * 32, + sequence=5, + signed_at=333, + ) + + assert pinned["trust_changed"] is False + assert same["trust_changed"] is False + assert changed["trust_changed"] is True + stored = mesh_wormhole_contacts.list_wormhole_dm_contacts()["peer-alpha"] + assert stored["remotePrekeyFingerprint"] == "aa" * 32 + assert stored["remotePrekeyObservedFingerprint"] == "bb" * 32 + assert stored["remotePrekeyMismatch"] is True + + +def test_compose_wormhole_dm_rejects_remote_prekey_identity_change(tmp_path, monkeypatch): + _relay, _identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + import main + from services.mesh import mesh_wormhole_contacts + + monkeypatch.setattr(mesh_wormhole_contacts, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_contacts, "CONTACTS_FILE", tmp_path / "wormhole_dm_contacts.json") + monkeypatch.setattr(main, "has_mls_dm_session", lambda *_args, **_kwargs: {"ok": True, "exists": False}) + monkeypatch.setattr(main, "initiate_mls_dm_session", lambda *_args, **_kwargs: {"ok": True, "welcome": "welcome"}) + monkeypatch.setattr(main, "encrypt_mls_dm", lambda *_args, **_kwargs: {"ok": True, "ciphertext": "ct", "nonce": "n"}) + + initial = { + "ok": True, + "agent_id": "peer-alpha", + "mls_key_package": "ZmFrZQ==", + "identity_dh_pub_key": "peer-dh-pub", + "public_key": "peer-signing-pub", + "public_key_algo": "Ed25519", + "protocol_version": "infonet/2", + "sequence": 2, + "signed_at": int(time.time()), + "trust_fingerprint": "11" * 32, + } + changed = { + **initial, + "sequence": 3, + "signed_at": int(time.time()) + 1, + "trust_fingerprint": "22" * 32, + } + + first = main.compose_wormhole_dm( + peer_id="peer-alpha", + peer_dh_pub="peer-dh-pub", + plaintext="hello", + remote_prekey_bundle=initial, + ) + second = main.compose_wormhole_dm( + peer_id="peer-alpha", + peer_dh_pub="peer-dh-pub", + plaintext="hello again", + remote_prekey_bundle=changed, + ) + + assert first["ok"] is True + assert second == { + "ok": False, + "peer_id": "peer-alpha", + "detail": "remote prekey identity changed; verification required", + "trust_changed": True, + } + + +def test_prekey_bundle_registration_rejects_invalid_bundle(tmp_path, monkeypatch): + relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity = identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh import mesh_wormhole_prekey as prekey_mod + + bundle = prekey_mod.ensure_wormhole_prekeys(force_signed_prekey=True) + bundle = prekey_mod._attach_bundle_signature(bundle, signed_at=int(time.time()) + 301) + + ok, reason, meta = relay.register_prekey_bundle( + identity["node_id"], + bundle, + "sig", + identity["public_key"], + identity["public_key_algo"], + "infonet/2", + 1, + ) + + assert ok is False + assert reason == "Prekey bundle signed_at is in the future" + assert meta is None + assert relay.get_prekey_bundle(identity["node_id"]) is None + + +def test_dm_mailbox_token_derivation_and_shared_sender_token_routing(tmp_path, monkeypatch): + relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity = identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh.mesh_wormhole_identity import derive_dm_mailbox_token + from services.mesh.mesh_wormhole_sender_token import issue_wormhole_dm_sender_token + import main + from services import wormhole_supervisor + from services.mesh import mesh_dm_relay, mesh_hashchain + + mailbox_token = derive_dm_mailbox_token(identity["node_id"]) + assert mailbox_token + + issued = issue_wormhole_dm_sender_token( + recipient_id="peer123", + delivery_class="shared", + recipient_token=mailbox_token, + ) + assert issued["ok"] is True + + monkeypatch.setattr(main, "_verify_signed_event", lambda **_kwargs: (True, "")) + monkeypatch.setattr(main, "_secure_dm_enabled", lambda: False) + monkeypatch.setattr(wormhole_supervisor, "get_transport_tier", lambda: "private_strong") + monkeypatch.setattr(mesh_hashchain.infonet, "validate_and_set_sequence", lambda *_args, **_kwargs: (True, "")) + monkeypatch.setattr(mesh_dm_relay, "dm_relay", relay) + + response = asyncio.run( + main.dm_send( + _json_request( + "/api/mesh/dm/send", + { + "sender_token": issued["sender_token"], + "recipient_id": "", + "delivery_class": "shared", + "recipient_token": mailbox_token, + "ciphertext": "cipher-shared", + "sender_seal": "v3:test-seal", + "msg_id": "shared-msg-1", + "timestamp": int(time.time()), + "public_key": "", + "public_key_algo": "Ed25519", + "signature": "sig", + "sequence": 1, + "protocol_version": "infonet/2", + }, + ) + ) + ) + + assert response["ok"] is True + hashed_mailbox = relay._hashed_mailbox_token(mailbox_token) + assert list(relay._mailboxes.keys()) == [hashed_mailbox] + assert relay._mailboxes[hashed_mailbox][0].sender_id.startswith("sender_token:") + assert relay._mailboxes[hashed_mailbox][0].sender_id != identity["node_id"] + delivered = relay.collect_claims(identity["node_id"], [{"type": "shared", "token": mailbox_token}]) + assert [msg["msg_id"] for msg in delivered] == ["shared-msg-1"] + + +def test_open_sender_seal_verifies_in_wormhole(tmp_path, monkeypatch): + _relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity = identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh.mesh_wormhole_seal import build_sender_seal, open_sender_seal + + msg_id = "dm_test_1" + timestamp = 1234567890 + built = build_sender_seal( + recipient_id=identity["node_id"], + recipient_dh_pub=identity["dh_pub_key"], + msg_id=msg_id, + timestamp=timestamp, + ) + assert built["ok"] + assert str(built["sender_seal"]).startswith("v3:") + + opened = open_sender_seal( + sender_seal=built["sender_seal"], + candidate_dh_pub=identity["dh_pub_key"], + recipient_id=identity["node_id"], + expected_msg_id=msg_id, + ) + assert opened["ok"] + assert opened["sender_id"] == identity["node_id"] + assert opened["seal_verified"] is True + + +def test_open_sender_seal_still_accepts_legacy_format(tmp_path, monkeypatch): + _relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity = identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh.mesh_wormhole_identity import sign_wormhole_message + from services.mesh.mesh_wormhole_seal import open_sender_seal + + sender_priv = x25519.X25519PrivateKey.generate() + sender_pub = sender_priv.public_key() + recipient_pub = x25519.X25519PublicKey.from_public_bytes(base64.b64decode(identity["dh_pub_key"])) + shared = sender_priv.exchange(recipient_pub) + + msg_id = "dm_test_legacy" + timestamp = 1234567890 + signed = sign_wormhole_message(f"seal|{msg_id}|{timestamp}|{identity['node_id']}") + seal_payload = { + "sender_id": signed["node_id"], + "public_key": signed["public_key"], + "public_key_algo": signed["public_key_algo"], + "msg_id": msg_id, + "timestamp": timestamp, + "signature": signed["signature"], + } + iv = b"\x00" * 12 + ciphertext = AESGCM(shared).encrypt(iv, json.dumps(seal_payload).encode("utf-8"), None) + sender_seal = base64.b64encode(iv + ciphertext).decode("ascii") + candidate_dh_pub = base64.b64encode( + sender_pub.public_bytes(Encoding.Raw, PublicFormat.Raw) + ).decode("ascii") + + opened = open_sender_seal( + sender_seal=sender_seal, + candidate_dh_pub=candidate_dh_pub, + recipient_id=identity["node_id"], + expected_msg_id=msg_id, + ) + assert opened["ok"] + assert opened["sender_id"] == identity["node_id"] + assert opened["seal_verified"] is True + + +def test_legacy_sender_seal_rejected_in_hardened_mode(tmp_path, monkeypatch): + _relay, identity_mod = _fresh_mesh_state(tmp_path, monkeypatch) + identity = identity_mod.bootstrap_wormhole_identity(force=True) + + from services.mesh.mesh_wormhole_identity import sign_wormhole_message + from services.mesh import mesh_wormhole_seal + + monkeypatch.setattr( + mesh_wormhole_seal, + "read_wormhole_settings", + lambda: {"enabled": True, "anonymous_mode": True}, + ) + + sender_priv = x25519.X25519PrivateKey.generate() + sender_pub = sender_priv.public_key() + recipient_pub = x25519.X25519PublicKey.from_public_bytes(base64.b64decode(identity["dh_pub_key"])) + shared = sender_priv.exchange(recipient_pub) + + msg_id = "dm_test_legacy_hardened" + timestamp = 1234567890 + signed = sign_wormhole_message(f"seal|{msg_id}|{timestamp}|{identity['node_id']}") + seal_payload = { + "sender_id": signed["node_id"], + "public_key": signed["public_key"], + "public_key_algo": signed["public_key_algo"], + "msg_id": msg_id, + "timestamp": timestamp, + "signature": signed["signature"], + } + iv = b"\x00" * 12 + ciphertext = AESGCM(shared).encrypt(iv, json.dumps(seal_payload).encode("utf-8"), None) + sender_seal = base64.b64encode(iv + ciphertext).decode("ascii") + candidate_dh_pub = base64.b64encode( + sender_pub.public_bytes(Encoding.Raw, PublicFormat.Raw) + ).decode("ascii") + + opened = mesh_wormhole_seal.open_sender_seal( + sender_seal=sender_seal, + candidate_dh_pub=candidate_dh_pub, + recipient_id=identity["node_id"], + expected_msg_id=msg_id, + ) + assert opened["ok"] is False + assert "Legacy sender seals" in opened["detail"] + + +def test_require_admin_no_longer_trusts_loopback_without_override(monkeypatch): + from fastapi import HTTPException + from starlette.requests import Request + import main + + monkeypatch.setattr(main, "_current_admin_key", lambda: "") + monkeypatch.setattr(main, "_allow_insecure_admin", lambda: False) + + request = Request( + { + "type": "http", + "headers": [], + "client": ("127.0.0.1", 12345), + "method": "GET", + "path": "/api/wormhole/status", + } + ) + + with pytest.raises(HTTPException) as exc: + main.require_admin(request) + assert exc.value.status_code == 403 diff --git a/backend/tests/mesh/test_mesh_wormhole_persona.py b/backend/tests/mesh/test_mesh_wormhole_persona.py new file mode 100644 index 00000000..4aac9283 --- /dev/null +++ b/backend/tests/mesh/test_mesh_wormhole_persona.py @@ -0,0 +1,307 @@ +import asyncio + +from starlette.requests import Request + + +def _fresh_persona_state(tmp_path, monkeypatch): + from services.mesh import mesh_secure_storage, mesh_wormhole_identity, mesh_wormhole_persona + + monkeypatch.setattr(mesh_secure_storage, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_secure_storage, "MASTER_KEY_FILE", tmp_path / "wormhole_secure_store.key") + monkeypatch.setattr(mesh_wormhole_persona, "DATA_DIR", tmp_path) + monkeypatch.setattr(mesh_wormhole_persona, "PERSONA_FILE", tmp_path / "wormhole_persona.json") + monkeypatch.setattr( + mesh_wormhole_persona, + "LEGACY_DM_IDENTITY_FILE", + tmp_path / "wormhole_identity.json", + ) + return mesh_wormhole_persona, mesh_wormhole_identity + + +def _request(path: str) -> Request: + return Request( + { + "type": "http", + "headers": [], + "client": ("test", 12345), + "method": "POST", + "path": path, + } + ) + + +def test_transport_identity_is_separate_from_dm_identity(tmp_path, monkeypatch): + persona_mod, identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + dm_identity = identity_mod.bootstrap_wormhole_identity(force=True) + persona_state = persona_mod.bootstrap_wormhole_persona_state(force=True) + transport_identity = persona_state["transport_identity"] + + assert dm_identity["node_id"] + assert transport_identity["node_id"] + assert dm_identity["node_id"] != transport_identity["node_id"] + assert dm_identity["public_key"] != transport_identity["public_key"] + + +def test_gate_anonymous_session_differs_from_transport_identity(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + transport_identity = persona_mod.get_transport_identity() + gate_identity = persona_mod.enter_gate_anonymously("journalists", rotate=True) + + assert gate_identity["ok"] is True + assert gate_identity["identity"]["scope"] == "gate_session" + assert gate_identity["identity"]["gate_id"] == "journalists" + assert gate_identity["identity"]["node_id"] != transport_identity["node_id"] + + +def test_gate_identities_are_separate_from_root_identity(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + state = persona_mod.read_wormhole_persona_state() + root_identity = state["root_identity"] + gate_session = persona_mod.enter_gate_anonymously("journalists", rotate=True)["identity"] + gate_persona = persona_mod.create_gate_persona("journalists", label="source-a")["identity"] + + assert root_identity["node_id"] + assert gate_session["node_id"] != root_identity["node_id"] + assert gate_session["public_key"] != root_identity["public_key"] + assert gate_persona["node_id"] != root_identity["node_id"] + assert gate_persona["public_key"] != root_identity["public_key"] + + +def test_gate_persona_activation_is_gate_local(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona("sources", label="source-a") + second = persona_mod.create_gate_persona("leaks", label="source-a") + + assert first["ok"] is True + assert second["ok"] is True + assert first["identity"]["gate_id"] == "sources" + assert second["identity"]["gate_id"] == "leaks" + assert first["identity"]["node_id"] != second["identity"]["node_id"] + + active_sources = persona_mod.get_active_gate_identity("sources") + active_leaks = persona_mod.get_active_gate_identity("leaks") + assert active_sources["identity"]["persona_id"] == first["identity"]["persona_id"] + assert active_leaks["identity"]["persona_id"] == second["identity"]["persona_id"] + + +def test_gate_persona_duplicate_labels_get_unique_suffixes(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.create_gate_persona("sources", label="source-a") + second = persona_mod.create_gate_persona("sources", label="source-a") + third = persona_mod.create_gate_persona("sources", label="Source-A") + + assert first["ok"] is True + assert second["ok"] is True + assert third["ok"] is True + assert first["identity"]["label"] == "source-a" + assert second["identity"]["label"] == "source-a-2" + assert third["identity"]["label"] == "Source-A-3" + + +def test_sign_public_event_uses_transport_identity(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + transport_identity = persona_mod.get_transport_identity() + signed = persona_mod.sign_public_wormhole_event( + event_type="message", + payload={ + "message": "hello", + "destination": "broadcast", + "channel": "LongFast", + "priority": "normal", + "ephemeral": False, + }, + ) + + assert signed["identity_scope"] == "transport" + assert signed["node_id"] == transport_identity["node_id"] + assert signed["public_key"] == transport_identity["public_key"] + + +def test_sign_gate_event_uses_gate_session_identity(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + transport_identity = persona_mod.get_transport_identity() + signed = persona_mod.sign_gate_wormhole_event( + gate_id="journalists", + event_type="gate_message", + payload={ + "gate": "journalists", + "epoch": 1, + "ciphertext": "opaque-source-drop", + "nonce": "nonce-j1", + "sender_ref": "gate-session-j1", + }, + ) + gate_identity = persona_mod.get_active_gate_identity("journalists") + + assert signed["identity_scope"] == "gate_session" + assert signed["gate_id"] == "journalists" + assert signed["node_id"] == gate_identity["identity"]["node_id"] + assert signed["node_id"] != transport_identity["node_id"] + + +def test_leave_gate_forces_new_anonymous_session_on_reentry(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.enter_gate_anonymously("sources", rotate=True) + persona_mod.leave_gate("sources") + second = persona_mod.enter_gate_anonymously("sources", rotate=False) + + assert first["identity"]["node_id"] + assert second["identity"]["node_id"] + assert first["identity"]["node_id"] != second["identity"]["node_id"] + + +def test_gate_session_rotation_uses_jitter_window_before_auto_swap(tmp_path, monkeypatch): + from services.config import get_settings + + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + monkeypatch.setenv("MESH_GATE_SESSION_ROTATE_MSGS", "1") + monkeypatch.setenv("MESH_GATE_SESSION_ROTATE_JITTER_S", "120") + get_settings.cache_clear() + try: + now = {"value": 1_000.0} + monkeypatch.setattr(persona_mod.time, "time", lambda: now["value"]) + monkeypatch.setattr(persona_mod.random, "uniform", lambda *_args, **_kwargs: 45.0) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + first = persona_mod.enter_gate_anonymously("sources", rotate=True) + persona_mod.sign_gate_wormhole_event( + gate_id="sources", + event_type="gate_message", + payload={ + "gate": "sources", + "epoch": 1, + "ciphertext": "opaque", + "nonce": "nonce-1", + "sender_ref": "sender-ref-1", + "format": "mls1", + }, + ) + + same = persona_mod.enter_gate_anonymously("sources", rotate=False) + scheduled = persona_mod.read_wormhole_persona_state()["gate_sessions"]["sources"]["_rotate_after"] + + assert same["identity"]["node_id"] == first["identity"]["node_id"] + assert scheduled == 1_045.0 + + now["value"] = 1_046.0 + rotated = persona_mod.enter_gate_anonymously("sources", rotate=False) + + assert rotated["identity"]["node_id"] != first["identity"]["node_id"] + finally: + get_settings.cache_clear() + + +def test_gate_enter_leave_do_not_emit_public_breadcrumbs(tmp_path, monkeypatch): + import main + from services.mesh import mesh_hashchain + + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + append_called = {"count": 0} + + def fake_append(**kwargs): + append_called["count"] += 1 + return {"event_id": "unexpected"} + + monkeypatch.setattr(mesh_hashchain.infonet, "append", fake_append) + + body = main.WormholeGateRequest(gate_id="sources", rotate=True) + entered = asyncio.run(main.api_wormhole_gate_enter(_request("/api/wormhole/gate/enter"), body)) + left = asyncio.run( + main.api_wormhole_gate_leave( + _request("/api/wormhole/gate/leave"), + main.WormholeGateRequest(gate_id="sources"), + ) + ) + + assert entered["ok"] is True + assert left["ok"] is True + assert append_called["count"] == 0 + + +def test_clear_active_persona_reverts_gate_to_anonymous_session(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.enter_gate_anonymously("evidence", rotate=True) + created = persona_mod.create_gate_persona("evidence", label="reporter") + cleared = persona_mod.clear_active_gate_persona("evidence") + active = persona_mod.get_active_gate_identity("evidence") + + assert created["identity"]["scope"] == "gate_persona" + assert cleared["ok"] is True + assert cleared["identity"]["scope"] == "gate_session" + assert active["source"] == "anonymous" + + +def test_sign_gate_event_uses_active_persona_when_selected(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.enter_gate_anonymously("ops", rotate=True) + created = persona_mod.create_gate_persona("ops", label="scribe") + signed = persona_mod.sign_gate_wormhole_event( + gate_id="ops", + event_type="gate_message", + payload={ + "gate": "ops", + "epoch": 1, + "ciphertext": "opaque-persona-post", + "nonce": "nonce-o1", + "sender_ref": "persona-ops-1", + }, + ) + + assert created["identity"]["persona_id"] + assert signed["identity_scope"] == "gate_persona" + assert signed["node_id"] == created["identity"]["node_id"] + + +def test_enter_gate_anonymously_clears_existing_active_persona(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + created = persona_mod.create_gate_persona("ops", label="scribe") + entered = persona_mod.enter_gate_anonymously("ops", rotate=True) + active = persona_mod.get_active_gate_identity("ops") + + assert created["identity"]["scope"] == "gate_persona" + assert entered["identity"]["scope"] == "gate_session" + assert active["source"] == "anonymous" + assert active["identity"]["node_id"] == entered["identity"]["node_id"] + assert active["identity"]["node_id"] != created["identity"]["node_id"] + + +def test_sign_gate_event_rejects_cross_gate_payload_mismatch(tmp_path, monkeypatch): + persona_mod, _identity_mod = _fresh_persona_state(tmp_path, monkeypatch) + + persona_mod.bootstrap_wormhole_persona_state(force=True) + persona_mod.enter_gate_anonymously("ops", rotate=True) + signed = persona_mod.sign_gate_wormhole_event( + gate_id="ops", + event_type="gate_message", + payload={ + "gate": "finance", + "epoch": 1, + "ciphertext": "opaque-cross-gate-post", + "nonce": "nonce-cross-1", + "sender_ref": "persona-finance-1", + }, + ) + + assert signed["ok"] is False + assert signed["detail"] == "gate payload mismatch" diff --git a/backend/tests/mesh/test_node_settings.py b/backend/tests/mesh/test_node_settings.py new file mode 100644 index 00000000..13acfb50 --- /dev/null +++ b/backend/tests/mesh/test_node_settings.py @@ -0,0 +1,15 @@ +def test_node_settings_roundtrip(tmp_path, monkeypatch): + from services import node_settings + + settings_path = tmp_path / "node.json" + monkeypatch.setattr(node_settings, "NODE_FILE", settings_path) + monkeypatch.setattr(node_settings, "_cache", None) + monkeypatch.setattr(node_settings, "_cache_ts", 0.0) + + initial = node_settings.read_node_settings() + updated = node_settings.write_node_settings(enabled=True) + reread = node_settings.read_node_settings() + + assert initial["enabled"] is False + assert updated["enabled"] is True + assert reread["enabled"] is True diff --git a/backend/tests/mesh/test_privacy_core_cross_node.py b/backend/tests/mesh/test_privacy_core_cross_node.py new file mode 100644 index 00000000..caf05164 --- /dev/null +++ b/backend/tests/mesh/test_privacy_core_cross_node.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import base64 +import shutil +from pathlib import Path + +import pytest + +from services.privacy_core_client import ( + PrivacyCoreError, + PrivacyCoreClient, + PrivacyCoreUnavailable, + candidate_library_paths, +) + + +def _built_library_path() -> Path: + for candidate in candidate_library_paths(): + if candidate.exists(): + return candidate + raise PrivacyCoreUnavailable("privacy-core shared library not found") + + +def _isolated_client(tmp_path: Path, name: str) -> PrivacyCoreClient: + source = _built_library_path() + target = tmp_path / f"{name}{source.suffix}" + shutil.copy2(source, target) + return PrivacyCoreClient.load(target) + + +# NOTE: This test runs both clients in the same process. It validates key-package +# serialization/deserialization correctness but does not prove cross-process isolation. +# True cross-process testing deferred — see BUILD_TRACKER S3-F4 note. +def test_cross_client_key_package_serialization_round_trip(tmp_path): + try: + client_a = _isolated_client(tmp_path, "privacy_core_node_a") + client_b = _isolated_client(tmp_path, "privacy_core_node_b") + except PrivacyCoreUnavailable: + pytest.skip("privacy-core shared library not found") + + assert client_a.reset_all_state() is True + assert client_b.reset_all_state() is True + + alice = client_a.create_identity() + group = client_a.create_group(alice) + throwaway = client_b.create_identity() + bob = client_b.create_identity() + + exported = client_b.export_key_package(bob) + transported = base64.b64decode(base64.b64encode(exported)) + imported = client_a.import_key_package(transported) + commit = client_a.add_member(group, imported) + + assert client_a.commit_message_bytes(commit) + assert client_a.commit_welcome_message_bytes(commit) + + assert client_a.release_commit(commit) is True + assert client_a.release_key_package(imported) is True + assert client_a.release_group(group) is True + assert client_a.release_identity(alice) is True + assert client_b.release_identity(throwaway) is True + assert client_b.release_identity(bob) is True + + +def test_import_key_package_rejects_oversized_payload(tmp_path): + try: + client = _isolated_client(tmp_path, "privacy_core_oversized") + except PrivacyCoreUnavailable: + pytest.skip("privacy-core shared library not found") + + assert client.reset_all_state() is True + + with pytest.raises(PrivacyCoreError, match="maximum size"): + client.import_key_package(b"x" * 65_537) diff --git a/backend/tests/mesh/test_privacy_core_ffi_invariants.py b/backend/tests/mesh/test_privacy_core_ffi_invariants.py new file mode 100644 index 00000000..0ff687dc --- /dev/null +++ b/backend/tests/mesh/test_privacy_core_ffi_invariants.py @@ -0,0 +1,335 @@ +"""Sprint 1 security-review invariant tests for privacy-core FFI. + +These tests exercise the FFI boundary, handle lifecycle, buffer ownership, +and MLS correctness of the Rust privacy-core as accessed through the Python +ctypes bridge. + +Test IDs map to the S1 Security Review findings: + + S1-T1 Use-after-release: freed handle must produce error, no ciphertext + S1-T2 Double-release: second release returns False, no crash + S1-T3 Public-bundle key-material: exported JSON contains no private key + S1-T4 MLS round-trip: encrypt → decrypt produces original plaintext + S1-T5 Removed member cannot decrypt post-removal messages + +Requires a compiled privacy-core shared library. If unavailable, tests are +skipped rather than failed. +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from pathlib import Path + +# Ensure the backend package is importable. +_backend = Path(__file__).resolve().parents[2] +if str(_backend) not in sys.path: + sys.path.insert(0, str(_backend)) + +from services.privacy_core_client import ( + PrivacyCoreClient, + PrivacyCoreError, + PrivacyCoreUnavailable, +) + +_client: PrivacyCoreClient | None = None + + +def _get_client() -> PrivacyCoreClient: + """Lazy-load the privacy-core library; skip if unavailable.""" + global _client # noqa: PLW0603 + if _client is not None: + return _client + try: + _client = PrivacyCoreClient.load() + except PrivacyCoreUnavailable: + raise unittest.SkipTest( + "privacy-core shared library not found — skipping FFI invariant tests" + ) + return _client + + +class TestUseAfterRelease(unittest.TestCase): + """S1-T1: Operations on a released handle must fail cleanly.""" + + def test_encrypt_after_release_group(self) -> None: + """Releasing a group handle then encrypting with it must raise, not encrypt.""" + client = _get_client() + + identity = client.create_identity() + group = client.create_group(identity) + + # Sanity: encryption works before release. + ciphertext = client.encrypt_group_message(group, b"pre-release") + self.assertIsInstance(ciphertext, bytes) + self.assertGreater(len(ciphertext), 0) + + # Release the group handle. + released = client.release_group(group) + self.assertTrue(released) + + # Post-release: must raise, must NOT return ciphertext. + with self.assertRaises(PrivacyCoreError) as ctx: + client.encrypt_group_message(group, b"post-release-secret") + self.assertIn("unknown group handle", str(ctx.exception).lower()) + + # Cleanup. + client.release_identity(identity) + + def test_decrypt_after_release_group(self) -> None: + """Decrypting with a freed group handle must fail.""" + client = _get_client() + + identity = client.create_identity() + group = client.create_group(identity) + ciphertext = client.encrypt_group_message(group, b"test") + client.release_group(group) + + with self.assertRaises(PrivacyCoreError): + client.decrypt_group_message(group, ciphertext) + + client.release_identity(identity) + + +class TestDoubleRelease(unittest.TestCase): + """S1-T2: Double-releasing a handle must not crash and must return False.""" + + def test_double_release_identity(self) -> None: + client = _get_client() + identity = client.create_identity() + first = client.release_identity(identity) + second = client.release_identity(identity) + self.assertTrue(first) + self.assertFalse(second) + + def test_double_release_group(self) -> None: + client = _get_client() + identity = client.create_identity() + group = client.create_group(identity) + first = client.release_group(group) + second = client.release_group(group) + self.assertTrue(first) + self.assertFalse(second) + client.release_identity(identity) + + def test_double_release_commit(self) -> None: + client = _get_client() + + alice = client.create_identity() + bob = client.create_identity() + group = client.create_group(alice) + kp_bytes = client.export_key_package(bob) + kp_handle = client.import_key_package(kp_bytes) + commit = client.add_member(group, kp_handle) + + first = client.release_commit(commit) + second = client.release_commit(commit) + self.assertTrue(first) + self.assertFalse(second) + + # Cleanup. + client.release_group(group) + client.release_key_package(kp_handle) + client.release_identity(alice) + client.release_identity(bob) + + +class TestPublicBundleNoPrivateKey(unittest.TestCase): + """S1-T3: Exported public bundle must not contain private key material.""" + + def test_bundle_contains_only_public_fields(self) -> None: + client = _get_client() + identity = client.create_identity() + + bundle_bytes = client.export_public_bundle(identity) + bundle = json.loads(bundle_bytes) + + # Expected fields only. + allowed_keys = {"label", "cipher_suite", "signing_public_key", "credential"} + self.assertEqual(set(bundle.keys()), allowed_keys) + + # The signing_public_key field must be present and non-empty (it's the PUBLIC key). + self.assertIsInstance(bundle["signing_public_key"], list) + self.assertGreater(len(bundle["signing_public_key"]), 0) + + # Verify no field name suggests private material. + for key in bundle: + self.assertNotIn("private", key.lower(), f"Field '{key}' suggests private material") + self.assertNotIn("secret", key.lower(), f"Field '{key}' suggests secret material") + + client.release_identity(identity) + + def test_bundle_for_unknown_identity_fails(self) -> None: + client = _get_client() + with self.assertRaises(PrivacyCoreError): + client.export_public_bundle(0xDEAD) + + +class TestMLSRoundTrip(unittest.TestCase): + """S1-T4: Encrypt → decrypt must produce original plaintext.""" + + def test_two_member_encrypt_decrypt(self) -> None: + client = _get_client() + + alice_id = client.create_identity() + bob_id = client.create_identity() + + # Alice creates a group. + alice_group = client.create_group(alice_id) + + # Bob exports a key package; Alice imports it and adds Bob. + kp_bytes = client.export_key_package(bob_id) + kp_handle = client.import_key_package(kp_bytes) + commit = client.add_member(alice_group, kp_handle) + + # Get Bob's joined group handle. + bob_group = client.commit_joined_group_handle(commit) + + # Alice encrypts; Bob decrypts. + plaintext = b"hello from alice" + ciphertext = client.encrypt_group_message(alice_group, plaintext) + self.assertNotEqual(ciphertext, plaintext) + + decrypted = client.decrypt_group_message(bob_group, ciphertext) + self.assertEqual(decrypted, plaintext) + + # Bob encrypts; Alice decrypts. + plaintext2 = b"hello from bob" + ciphertext2 = client.encrypt_group_message(bob_group, plaintext2) + decrypted2 = client.decrypt_group_message(alice_group, ciphertext2) + self.assertEqual(decrypted2, plaintext2) + + # Cleanup. + client.release_commit(commit) + client.release_key_package(kp_handle) + client.release_group(alice_group) + client.release_group(bob_group) + client.release_identity(alice_id) + client.release_identity(bob_id) + + def test_old_epoch_ciphertext_fails_after_membership_change(self) -> None: + client = _get_client() + + alice_id = client.create_identity() + bob_id = client.create_identity() + charlie_id = client.create_identity() + + alice_group = client.create_group(alice_id) + bob_kp = client.export_key_package(bob_id) + bob_kp_handle = client.import_key_package(bob_kp) + commit1 = client.add_member(alice_group, bob_kp_handle) + bob_group = client.commit_joined_group_handle(commit1) + + old_epoch_ct = client.encrypt_group_message(alice_group, b"epoch one") + self.assertEqual(client.decrypt_group_message(bob_group, old_epoch_ct), b"epoch one") + + charlie_kp = client.export_key_package(charlie_id) + charlie_kp_handle = client.import_key_package(charlie_kp) + commit2 = client.add_member(alice_group, charlie_kp_handle) + charlie_group = client.commit_joined_group_handle(commit2) + + with self.assertRaises(PrivacyCoreError): + client.decrypt_group_message(alice_group, old_epoch_ct) + with self.assertRaises(PrivacyCoreError): + client.decrypt_group_message(bob_group, old_epoch_ct) + + new_epoch_ct = client.encrypt_group_message(alice_group, b"epoch two") + self.assertEqual(client.decrypt_group_message(charlie_group, new_epoch_ct), b"epoch two") + + for handle in (commit1, commit2): + client.release_commit(handle) + for handle in (bob_kp_handle, charlie_kp_handle): + client.release_key_package(handle) + for handle in (alice_group, bob_group, charlie_group): + client.release_group(handle) + for handle in (alice_id, bob_id, charlie_id): + client.release_identity(handle) + + +class TestRemovedMemberCannotDecrypt(unittest.TestCase): + """S1-T5: A removed member must fail to decrypt post-removal messages.""" + + def test_removed_member_decryption_fails(self) -> None: + client = _get_client() + + alice_id = client.create_identity() + bob_id = client.create_identity() + charlie_id = client.create_identity() + + # Alice creates group, adds Bob. + alice_group = client.create_group(alice_id) + bob_kp = client.export_key_package(bob_id) + bob_kp_h = client.import_key_package(bob_kp) + commit1 = client.add_member(alice_group, bob_kp_h) + bob_group = client.commit_joined_group_handle(commit1) + + # Alice adds Charlie. + charlie_kp = client.export_key_package(charlie_id) + charlie_kp_h = client.import_key_package(charlie_kp) + commit2 = client.add_member(alice_group, charlie_kp_h) + charlie_group = client.commit_joined_group_handle(commit2) + + # Verify all three can communicate. + ct = client.encrypt_group_message(alice_group, b"all three") + self.assertEqual(client.decrypt_group_message(bob_group, ct), b"all three") + + # Alice removes Bob (member_ref=1, since Alice is 0, Bob is 1). + # Note: member indices depend on insertion order in mls-rs. + # We try member_ref=1 for Bob. If it fails we try 2. + try: + remove_commit = client.remove_member(alice_group, 1) + except PrivacyCoreError: + remove_commit = client.remove_member(alice_group, 2) + + # Post-removal: Alice encrypts a new message. + post_removal_ct = client.encrypt_group_message(alice_group, b"bob is gone") + + # Charlie should still be able to decrypt. + post_removal_plain = client.decrypt_group_message(charlie_group, post_removal_ct) + self.assertEqual(post_removal_plain, b"bob is gone") + + # Bob's group handle should have been removed by the remove_member + # operation's family cleanup. Attempting to decrypt should fail. + with self.assertRaises(PrivacyCoreError): + client.decrypt_group_message(bob_group, post_removal_ct) + + # Cleanup. + for c in (commit1, commit2, remove_commit): + client.release_commit(c) + for kp in (bob_kp_h, charlie_kp_h): + client.release_key_package(kp) + for g in (alice_group, charlie_group): + client.release_group(g) + for i in (alice_id, bob_id, charlie_id): + client.release_identity(i) + + +class TestPrivacyCoreLimits(unittest.TestCase): + """S7-T5: privacy-core must enforce handle limits and report stats.""" + + def test_identity_limit_and_handle_stats(self) -> None: + client = _get_client() + client.reset_all_state() + stats_before = client.handle_stats() + self.assertEqual(stats_before["identities"], 0) + self.assertEqual(stats_before["max_identities"], 1024) + + handles = [] + for _ in range(stats_before["max_identities"]): + handles.append(client.create_identity()) + + stats_full = client.handle_stats() + self.assertEqual(stats_full["identities"], stats_full["max_identities"]) + + with self.assertRaises(PrivacyCoreError): + client.create_identity() + + for handle in handles: + client.release_identity(handle) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/mesh/test_wormhole_supervisor_hardening.py b/backend/tests/mesh/test_wormhole_supervisor_hardening.py new file mode 100644 index 00000000..f3998967 --- /dev/null +++ b/backend/tests/mesh/test_wormhole_supervisor_hardening.py @@ -0,0 +1,43 @@ +import os +from types import SimpleNamespace +from unittest.mock import patch + + +def test_wormhole_subprocess_env_whitelists_runtime_and_mesh_vars(): + from services import wormhole_supervisor + + settings = { + "transport": "tor", + "socks_proxy": "127.0.0.1:9050", + "socks_dns": True, + } + config_snapshot = SimpleNamespace(MESH_RNS_ENABLED=False) + + with patch.dict( + os.environ, + { + "PATH": "C:\\Python;C:\\Windows\\System32", + "SYSTEMROOT": "C:\\Windows", + "PYTHONPATH": "F:\\Codebase\\Oracle\\live-risk-dashboard\\backend", + "ADMIN_KEY": "admin-secret", + "MESH_PEER_PUSH_SECRET": "peer-secret-value", + "UNRELATED_SECRET": "should-not-leak", + }, + clear=True, + ): + env = wormhole_supervisor._wormhole_subprocess_env( + settings, + settings_obj=config_snapshot, + ) + + assert env["PATH"] == "C:\\Python;C:\\Windows\\System32" + assert env["SYSTEMROOT"] == "C:\\Windows" + assert env["PYTHONPATH"] == "F:\\Codebase\\Oracle\\live-risk-dashboard\\backend" + assert env["ADMIN_KEY"] == "admin-secret" + assert env["MESH_PEER_PUSH_SECRET"] == "peer-secret-value" + assert env["MESH_ONLY"] == "true" + assert env["MESH_RNS_ENABLED"] == "false" + assert env["WORMHOLE_TRANSPORT"] == "tor" + assert env["WORMHOLE_SOCKS_PROXY"] == "127.0.0.1:9050" + assert env["WORMHOLE_SOCKS_DNS"] == "true" + assert "UNRELATED_SECRET" not in env diff --git a/backend/tests/test_api_smoke.py b/backend/tests/test_api_smoke.py index 1ec510c0..6a029226 100644 --- a/backend/tests/test_api_smoke.py +++ b/backend/tests/test_api_smoke.py @@ -1,4 +1,7 @@ """Smoke tests for all API endpoints — verifies routes exist and return valid responses.""" + +import asyncio + import pytest @@ -51,6 +54,92 @@ def test_slow_has_expected_keys(self, client): for key in ("news", "stocks", "weather", "earthquakes"): assert key in data, f"Missing key: {key}" + def test_fast_returns_full_world_payload_and_filters_disabled_sigint_sources(self, client, monkeypatch): + from services.fetchers import _store + + ships = [{"lat": float(i % 80), "lng": float((i % 360) - 180), "id": i} for i in range(2000)] + sigint = ( + [{"source": "aprs", "lat": 1.0, "lng": 1.0, "id": f"a-{i}"} for i in range(50)] + + [{"source": "meshtastic", "lat": 2.0, "lng": 2.0, "id": f"m-{i}"} for i in range(50)] + + [{"source": "meshtastic", "from_api": True, "lat": 3.0, "lng": 3.0, "id": f"mm-{i}"} for i in range(50)] + + [{"source": "js8call", "lat": 4.0, "lng": 4.0, "id": f"j-{i}"} for i in range(50)] + ) + + monkeypatch.setitem(_store.latest_data, "ships", ships) + monkeypatch.setitem(_store.latest_data, "sigint", sigint) + monkeypatch.setitem(_store.active_layers, "sigint_aprs", False) + monkeypatch.setitem(_store.active_layers, "sigint_meshtastic", True) + + r = client.get("/api/live-data/fast") + + assert r.status_code == 200 + data = r.json() + assert len(data["ships"]) == len(ships) + assert all(item["source"] != "aprs" for item in data["sigint"]) + assert data["sigint_totals"]["aprs"] == 0 + assert data["sigint_totals"]["meshtastic"] == 100 + assert data["sigint_totals"]["meshtastic_map"] == 50 + assert data["sigint_totals"]["js8call"] == 50 + + def test_slow_omits_disabled_power_plants_and_returns_full_world_datacenters(self, client, monkeypatch): + from services.fetchers import _store + + datacenters = [{"lat": float(i % 80), "lng": float((i % 360) - 180), "id": i} for i in range(2000)] + power_plants = [{"lat": float(i % 80), "lng": float((i % 360) - 180), "id": i} for i in range(4000)] + + monkeypatch.setitem(_store.latest_data, "datacenters", datacenters) + monkeypatch.setitem(_store.latest_data, "power_plants", power_plants) + monkeypatch.setitem(_store.active_layers, "datacenters", True) + monkeypatch.setitem(_store.active_layers, "power_plants", False) + + r = client.get("/api/live-data/slow") + + assert r.status_code == 200 + data = r.json() + assert len(data["datacenters"]) == len(datacenters) + assert data["power_plants"] == [] + + def test_slow_handles_geojson_incidents_without_crashing(self, client, monkeypatch): + from services.fetchers import _store + + gdelt = [ + { + "type": "Feature", + "properties": {"name": "Incident A"}, + "geometry": {"type": "Point", "coordinates": [10.0, 20.0]}, + } + ] + + monkeypatch.setitem(_store.latest_data, "gdelt", gdelt) + monkeypatch.setitem(_store.active_layers, "global_incidents", True) + + r = client.get("/api/live-data/slow") + + assert r.status_code == 200 + data = r.json() + assert data["gdelt"] == gdelt + + def test_enabling_viirs_layer_queues_immediate_refresh(self, monkeypatch): + import main + from httpx import ASGITransport, AsyncClient + from services.fetchers import _store + + queued = {"called": False} + + monkeypatch.setitem(_store.active_layers, "viirs_nightlights", False) + monkeypatch.setattr(main, "_queue_viirs_change_refresh", lambda: queued.__setitem__("called", True)) + + async def _exercise(): + transport = ASGITransport(app=main.app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + return await ac.post("/api/layers", json={"layers": {"viirs_nightlights": True}}) + + response = asyncio.run(_exercise()) + + assert response.status_code == 200 + assert response.json()["status"] == "ok" + assert queued["called"] is True + class TestDebugEndpoint: def test_debug_latest_returns_list(self, client): @@ -74,6 +163,20 @@ def test_get_news_feeds(self, client): assert isinstance(data, list) +class TestAdminProtection: + def test_refresh_requires_admin_key(self, client, monkeypatch): + import main + + monkeypatch.setattr(main, "_ADMIN_KEY", "test-key") + monkeypatch.setattr(main, "_ALLOW_INSECURE_ADMIN", False) + + r = client.get("/api/refresh") + assert r.status_code == 403 + + r_ok = client.get("/api/refresh", headers={"X-Admin-Key": "test-key"}) + assert r_ok.status_code in (200, 202) + + class TestRadioEndpoints: def test_radio_top_returns_200(self, client): r = client.get("/api/radio/top") diff --git a/backend/tests/test_fetch_health.py b/backend/tests/test_fetch_health.py new file mode 100644 index 00000000..31c45164 --- /dev/null +++ b/backend/tests/test_fetch_health.py @@ -0,0 +1,15 @@ +from services.fetch_health import record_success, record_failure, get_health_snapshot + + +def test_record_success_and_failure(): + record_success("unit_test_source", duration_s=0.1, count=3) + record_failure("unit_test_source", error=Exception("boom"), duration_s=0.2) + + snap = get_health_snapshot() + assert "unit_test_source" in snap + entry = snap["unit_test_source"] + assert entry["ok_count"] >= 1 + assert entry["error_count"] >= 1 + assert entry["last_ok"] is not None + assert entry["last_error"] is not None + assert entry["last_duration_ms"] is not None diff --git a/backend/tests/test_fetchers_geo.py b/backend/tests/test_fetchers_geo.py new file mode 100644 index 00000000..10deccec --- /dev/null +++ b/backend/tests/test_fetchers_geo.py @@ -0,0 +1,18 @@ +import json +from services.fetchers import geo + + +def test_find_nearest_airport_from_fixture(): + with open("tests/fixtures/airports.json", "r", encoding="utf-8") as f: + airports = json.load(f) + + geo.cached_airports = airports + + # Near Denver + result = geo.find_nearest_airport(39.74, -104.99, max_distance_nm=200) + assert result is not None + assert result["iata"] == "DEN" + + # Far away (middle of the ocean) + result_far = geo.find_nearest_airport(0.0, -140.0, max_distance_nm=50) + assert result_far is None diff --git a/backend/tests/test_gdelt_updater_hardening.py b/backend/tests/test_gdelt_updater_hardening.py new file mode 100644 index 00000000..887176de --- /dev/null +++ b/backend/tests/test_gdelt_updater_hardening.py @@ -0,0 +1,100 @@ +import zipfile +from unittest.mock import patch + +import pytest + +from services import geopolitics, updater + + +@pytest.fixture(autouse=True) +def _clear_gdelt_caches(): + geopolitics._article_title_cache.clear() + geopolitics._article_url_safety_cache.clear() + yield + geopolitics._article_title_cache.clear() + geopolitics._article_url_safety_cache.clear() + + +class TestGdeltArticleUrlSafety: + def test_safe_public_article_url_allows_public_dns(self, monkeypatch): + monkeypatch.setattr( + geopolitics.socket, + "getaddrinfo", + lambda *args, **kwargs: [(0, 0, 0, "", ("93.184.216.34", 443))], + ) + + allowed, reason = geopolitics._is_safe_public_article_url("https://example.com/story") + + assert allowed is True + assert reason == "" + + def test_safe_public_article_url_blocks_private_dns(self, monkeypatch): + monkeypatch.setattr( + geopolitics.socket, + "getaddrinfo", + lambda *args, **kwargs: [(0, 0, 0, "", ("10.0.0.7", 443))], + ) + + allowed, reason = geopolitics._is_safe_public_article_url("https://example.com/story") + + assert allowed is False + assert reason == "private_dns" + + def test_fetch_article_title_refuses_private_ip_without_request(self): + with patch("services.geopolitics.requests.get") as mock_get: + title = geopolitics._fetch_article_title("http://127.0.0.1/story") + + assert title is None + mock_get.assert_not_called() + + +class TestUpdaterHardening: + def test_validate_update_url_rejects_untrusted_host(self): + with pytest.raises(RuntimeError, match="untrusted release host"): + updater._validate_update_url("https://evil.example.com/update.zip") + + def test_extract_and_copy_rejects_zip_path_traversal(self, tmp_path): + zip_path = tmp_path / "bad.zip" + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("../escape.txt", "nope") + + with pytest.raises(RuntimeError, match="path traversal entry"): + updater._extract_and_copy(str(zip_path), str(tmp_path / "project"), str(tmp_path / "work")) + + def test_perform_update_returns_manual_url_on_failure(self, monkeypatch, tmp_path): + def _boom(_temp_dir): + raise RuntimeError("update exploded") + + monkeypatch.setattr(updater, "_download_release", _boom) + + result = updater.perform_update(str(tmp_path)) + + assert result["status"] == "error" + assert result["manual_url"] == updater.GITHUB_RELEASES_PAGE_URL + assert "update exploded" in result["message"] + + def test_perform_update_surfaces_release_metadata(self, monkeypatch, tmp_path): + release_url = "https://github.com/BigBodyCobain/Shadowbroker/releases/tag/v1.2.3" + download_url = ( + "https://github.com/BigBodyCobain/Shadowbroker/releases/download/v1.2.3/update.zip" + ) + backup_path = tmp_path / "backup.zip" + + monkeypatch.setattr( + updater, + "_download_release", + lambda _temp_dir: ("dummy.zip", "v1.2.3", download_url, release_url), + ) + monkeypatch.setattr(updater, "_validate_zip_hash", lambda _zip_path: None) + monkeypatch.setattr(updater, "_backup_current", lambda *_args: str(backup_path)) + monkeypatch.setattr(updater, "_extract_and_copy", lambda *_args: 42) + + result = updater.perform_update(str(tmp_path)) + + assert result["status"] == "ok" + assert result["version"] == "v1.2.3" + assert result["files_updated"] == 42 + assert result["backup_path"] == str(backup_path) + assert result["manual_url"] == release_url + assert result["release_url"] == release_url + assert result["download_url"] == download_url diff --git a/backend/tests/test_geocode_api.py b/backend/tests/test_geocode_api.py new file mode 100644 index 00000000..7781f13c --- /dev/null +++ b/backend/tests/test_geocode_api.py @@ -0,0 +1,20 @@ +from unittest.mock import patch + + +def test_geocode_search_proxy(client): + with patch("main.search_geocode") as mock_search: + mock_search.return_value = [{"label": "Denver, CO, USA", "lat": 39.7392, "lng": -104.9903}] + r = client.get("/api/geocode/search?q=denver&limit=1") + assert r.status_code == 200 + data = r.json() + assert data["count"] == 1 + assert data["results"][0]["label"] == "Denver, CO, USA" + + +def test_geocode_reverse_proxy(client): + with patch("main.reverse_geocode") as mock_reverse: + mock_reverse.return_value = {"label": "Boulder, CO, USA"} + r = client.get("/api/geocode/reverse?lat=40.01499&lng=-105.27055") + assert r.status_code == 200 + data = r.json() + assert data["label"] == "Boulder, CO, USA" diff --git a/backend/tests/test_meshtastic_topics.py b/backend/tests/test_meshtastic_topics.py new file mode 100644 index 00000000..63da369d --- /dev/null +++ b/backend/tests/test_meshtastic_topics.py @@ -0,0 +1,37 @@ +from services.mesh.meshtastic_topics import ( + build_subscription_topics, + normalize_root, + parse_topic_metadata, +) + + +def test_normalize_root_accepts_custom_subroots(): + assert normalize_root("msh/US/rob/snd/#") == "US/rob/snd" + assert normalize_root(" PL ") == "PL" + + +def test_build_subscription_topics_keeps_defaults_and_extras(): + topics = build_subscription_topics(extra_roots="PL,US/rob/snd", extra_topics="msh/+/2/json/#") + assert "msh/US/#" in topics + assert "msh/PL/#" in topics + assert "msh/US/rob/snd/#" in topics + assert "msh/+/2/json/#" in topics + + +def test_parse_topic_metadata_preserves_root_and_channel(): + meta = parse_topic_metadata("msh/US/rob/snd/2/e/LongFast/!abcd1234") + assert meta == { + "region": "US", + "root": "US/rob/snd", + "channel": "LongFast", + "mode": "e", + "version": "2", + } + + +def test_parse_topic_metadata_handles_json_topics(): + meta = parse_topic_metadata("msh/PL/2/json/PKI/!cafefeed") + assert meta["region"] == "PL" + assert meta["root"] == "PL" + assert meta["channel"] == "PKI" + assert meta["mode"] == "json" diff --git a/backend/tests/test_military_bases.py b/backend/tests/test_military_bases.py index ba457b4a..3721e886 100644 --- a/backend/tests/test_military_bases.py +++ b/backend/tests/test_military_bases.py @@ -40,12 +40,34 @@ def test_branch_values_are_known(self): for entry in raw: assert entry["branch"] in known_branches, f"{entry['name']} has unknown branch: {entry['branch']}" - def test_adversary_bases_present(self): + def test_multi_country_bases_present(self): raw = json.loads(BASES_PATH.read_text(encoding="utf-8")) countries = {entry["country"] for entry in raw} - for expected in ("China", "Russia", "North Korea", "Taiwan"): + for expected in ( + "China", "Russia", "North Korea", "Taiwan", "Japan", "Guam", + "Israel", "France", "Germany", "India", "Pakistan", + "United States", "United Kingdom", "Iran", "Italy", + "South Korea", "Australia", "Philippines", "Greece", + "Netherlands", "Spain", "Poland", + ): assert expected in countries, f"Missing bases for {expected}" + def test_nuclear_sites_present(self): + raw = json.loads(BASES_PATH.read_text(encoding="utf-8")) + nuclear = [e for e in raw if e["branch"] == "nuclear"] + countries_with_nuclear = {e["country"] for e in nuclear} + for expected in ("China", "Russia", "North Korea", "Iran", "Israel", + "India", "Pakistan", "United Kingdom", "France"): + assert expected in countries_with_nuclear, f"Missing nuclear sites for {expected}" + + def test_missile_sites_present(self): + raw = json.loads(BASES_PATH.read_text(encoding="utf-8")) + missiles = [e for e in raw if e["branch"] == "missile"] + countries_with_missiles = {e["country"] for e in missiles} + for expected in ("China", "Russia", "North Korea", "Iran", "Israel", + "India", "Pakistan", "Taiwan", "South Korea", "Poland"): + assert expected in countries_with_missiles, f"Missing missile sites for {expected}" + def test_no_duplicate_names(self): raw = json.loads(BASES_PATH.read_text(encoding="utf-8")) names = [entry["name"] for entry in raw] diff --git a/backend/tests/test_network_utils.py b/backend/tests/test_network_utils.py index 8a2ab190..b6b6cdbd 100644 --- a/backend/tests/test_network_utils.py +++ b/backend/tests/test_network_utils.py @@ -1,8 +1,15 @@ """Tests for network_utils — fetch_with_curl, circuit breaker, domain fail cache.""" + import time import pytest from unittest.mock import patch, MagicMock -from services.network_utils import fetch_with_curl, _circuit_breaker, _domain_fail_cache, _cb_lock, _DummyResponse +from services.network_utils import ( + fetch_with_curl, + _circuit_breaker, + _domain_fail_cache, + _cb_lock, + _DummyResponse, +) class TestDummyResponse: @@ -81,7 +88,7 @@ def test_domain_fail_cache_skips_to_curl(self): mock_result = MagicMock() mock_result.returncode = 0 mock_result.stdout = '{"data": true}\n200' - mock_result.stderr = '' + mock_result.stderr = "" with patch("subprocess.run", return_value=mock_result) as mock_run: result = fetch_with_curl("https://skip-to-curl.com/api") @@ -138,8 +145,9 @@ def test_post_with_json_data(self): with patch("services.network_utils._session") as mock_session: mock_session.post.return_value = mock_resp - result = fetch_with_curl("https://api.example.com/create", - method="POST", json_data={"name": "test"}) + result = fetch_with_curl( + "https://api.example.com/create", method="POST", json_data={"name": "test"} + ) assert result.status_code == 200 mock_session.post.assert_called_once() @@ -151,8 +159,9 @@ def test_custom_headers_merged(self): with patch("services.network_utils._session") as mock_session: mock_session.get.return_value = mock_resp - fetch_with_curl("https://api.example.com/data", - headers={"Authorization": "Bearer token123"}) + fetch_with_curl( + "https://api.example.com/data", headers={"Authorization": "Bearer token123"} + ) call_args = mock_session.get.call_args headers = call_args.kwargs.get("headers", {}) assert "Authorization" in headers diff --git a/backend/tests/test_release_helper.py b/backend/tests/test_release_helper.py new file mode 100644 index 00000000..a638c8b0 --- /dev/null +++ b/backend/tests/test_release_helper.py @@ -0,0 +1,48 @@ +import json +import importlib.util +from pathlib import Path + +import pytest + + +_HELPER_PATH = Path(__file__).resolve().parents[1] / "scripts" / "release_helper.py" +_SPEC = importlib.util.spec_from_file_location("release_helper", _HELPER_PATH) +assert _SPEC and _SPEC.loader +release_helper = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(release_helper) + + +def test_normalize_version_accepts_plain_and_prefixed(): + assert release_helper._normalize_version("0.9.6") == "0.9.6" + assert release_helper._normalize_version("v0.9.6") == "0.9.6" + + +def test_normalize_version_rejects_non_semver_triplet(): + with pytest.raises(ValueError, match="X.Y.Z"): + release_helper._normalize_version("0.9") + + +def test_expected_release_names(): + assert release_helper.expected_tag("0.9.6") == "v0.9.6" + assert release_helper.expected_asset("0.9.6") == "ShadowBroker_v0.9.6.zip" + + +def test_set_version_updates_package_json(monkeypatch, tmp_path): + package_json = tmp_path / "package.json" + package_json.write_text(json.dumps({"name": "frontend", "version": "0.9.5"}) + "\n", encoding="utf-8") + monkeypatch.setattr(release_helper, "PACKAGE_JSON", package_json) + + version = release_helper.set_version("0.9.6") + + assert version == "0.9.6" + data = json.loads(package_json.read_text(encoding="utf-8")) + assert data["version"] == "0.9.6" + + +def test_sha256_file(tmp_path): + payload = tmp_path / "payload.zip" + payload.write_bytes(b"shadowbroker") + + digest = release_helper.sha256_file(payload) + + assert digest == "153f774fe47e71734bf608e20fd59d9ee0ad522811dc9a121bbfd3dbd79a4229" diff --git a/backend/tests/test_schemas.py b/backend/tests/test_schemas.py index 8ff71783..59b4021c 100644 --- a/backend/tests/test_schemas.py +++ b/backend/tests/test_schemas.py @@ -1,4 +1,5 @@ """Tests for Pydantic response schemas.""" + import pytest from pydantic import ValidationError from services.schemas import HealthResponse, RefreshResponse, AisFeedResponse, RouteResponse @@ -11,7 +12,7 @@ def test_valid_health_response(self): "last_updated": "2024-01-01T00:00:00", "sources": {"flights": 150, "ships": 42}, "freshness": {"flights": "2024-01-01T00:00:00", "ships": "2024-01-01T00:00:00"}, - "uptime_seconds": 3600 + "uptime_seconds": 3600, } resp = HealthResponse(**data) assert resp.status == "ok" @@ -19,12 +20,7 @@ def test_valid_health_response(self): assert resp.uptime_seconds == 3600 def test_health_response_optional_last_updated(self): - data = { - "status": "ok", - "sources": {}, - "freshness": {}, - "uptime_seconds": 0 - } + data = {"status": "ok", "sources": {}, "freshness": {}, "uptime_seconds": 0} resp = HealthResponse(**data) assert resp.last_updated is None @@ -59,7 +55,7 @@ def test_valid_route(self): orig_loc=[40.6413, -73.7781], dest_loc=[51.4700, -0.4543], origin_name="JFK", - dest_name="LHR" + dest_name="LHR", ) assert resp.origin_name == "JFK" assert len(resp.orig_loc) == 2 diff --git a/backend/tests/test_sigint_cctv_accuracy.py b/backend/tests/test_sigint_cctv_accuracy.py new file mode 100644 index 00000000..12ad8cc5 --- /dev/null +++ b/backend/tests/test_sigint_cctv_accuracy.py @@ -0,0 +1,557 @@ +from types import SimpleNamespace +import pytest + + +def test_build_sigint_snapshot_merges_map_nodes_without_duplicate_meshtastic(monkeypatch): + from services.fetchers import sigint as sigint_fetcher + from services.fetchers._store import _data_lock, latest_data + from services import sigint_bridge as sigint_bridge_mod + + fake_grid = SimpleNamespace( + get_all_signals=lambda: [ + { + "callsign": "!live1", + "source": "meshtastic", + "timestamp": "2026-03-22T18:00:00+00:00", + "region": "US", + }, + { + "callsign": "K1ABC", + "source": "aprs", + "timestamp": "2026-03-22T17:59:00+00:00", + }, + ], + get_mesh_channel_stats=lambda api_nodes=None: {"total_api": len(api_nodes or [])}, + ) + monkeypatch.setattr(sigint_bridge_mod, "sigint_grid", fake_grid) + with _data_lock: + latest_data["meshtastic_map_nodes"] = [ + { + "callsign": "!live1", + "source": "meshtastic", + "timestamp": "2026-03-22T17:40:00+00:00", + "from_api": True, + }, + { + "callsign": "!map2", + "source": "meshtastic", + "timestamp": "2026-03-22T17:58:00+00:00", + "from_api": True, + }, + ] + + signals, channel_stats, totals = sigint_fetcher.build_sigint_snapshot() + + assert [sig["callsign"] for sig in signals] == ["!live1", "K1ABC", "!map2"] + assert channel_stats == {"total_api": 2} + assert totals == { + "total": 3, + "meshtastic": 2, + "meshtastic_live": 1, + "meshtastic_map": 1, + "aprs": 1, + "js8call": 0, + } + + +def test_rewrite_cctv_hls_playlist_proxies_relative_segments_and_keys(): + import main + + playlist = """#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-KEY:METHOD=AES-128,URI="keys/key.bin" +#EXTINF:5.0, +segment-001.ts +""" + + rewritten = main._rewrite_cctv_hls_playlist( + "https://navigator-c2c.dot.ga.gov/live/cam/index.m3u8", + playlist, + ) + + assert '/api/cctv/media?url=https%3A%2F%2Fnavigator-c2c.dot.ga.gov%2Flive%2Fcam%2Fkeys%2Fkey.bin' in rewritten + assert '/api/cctv/media?url=https%3A%2F%2Fnavigator-c2c.dot.ga.gov%2Flive%2Fcam%2Fsegment-001.ts' in rewritten + + +def test_cctv_proxy_allows_known_state_dot_media_hosts(): + import main + + allowed_hosts = { + "wzmedia.dot.ca.gov", + "511ga.org", + "cctv.travelmidwest.com", + "micamerasimages.net", + "tripcheck.com", + } + + for host in allowed_hosts: + assert main._cctv_host_allowed(host) + + +def test_fetch_satnogs_keeps_last_good_snapshot_on_error(monkeypatch): + from services.fetchers import infrastructure + from services.fetchers._store import _data_lock, latest_data + from services.fetchers import _store as store_mod + from services import satnogs_fetcher + + with _data_lock: + latest_data["satnogs_stations"] = [{"id": "station-1"}] + latest_data["satnogs_observations"] = [{"id": "obs-1"}] + + monkeypatch.setattr(store_mod, "is_any_active", lambda *args: True) + monkeypatch.setattr(satnogs_fetcher, "fetch_satnogs_stations", lambda: (_ for _ in ()).throw(ValueError("boom"))) + monkeypatch.setattr(satnogs_fetcher, "fetch_satnogs_observations", lambda: []) + + infrastructure.fetch_satnogs() + + with _data_lock: + assert latest_data["satnogs_stations"] == [{"id": "station-1"}] + assert latest_data["satnogs_observations"] == [{"id": "obs-1"}] + + +def test_fetch_tinygs_keeps_last_good_snapshot_on_error(monkeypatch): + from services.fetchers import infrastructure + from services.fetchers._store import _data_lock, latest_data + from services.fetchers import _store as store_mod + from services import tinygs_fetcher + + with _data_lock: + latest_data["tinygs_satellites"] = [{"norad_id": 12345}] + + monkeypatch.setattr(store_mod, "is_any_active", lambda *args: True) + monkeypatch.setattr(tinygs_fetcher, "fetch_tinygs_satellites", lambda: (_ for _ in ()).throw(ValueError("boom"))) + + infrastructure.fetch_tinygs() + + with _data_lock: + assert latest_data["tinygs_satellites"] == [{"norad_id": 12345}] + + +def test_caltrans_ingestor_prefers_static_image_when_stream_url_is_not_browser_safe(monkeypatch): + from services import cctv_pipeline + + class _Response: + status_code = 200 + + def json(self): + return { + "data": [ + { + "cctv": { + "location": { + "latitude": "34.123", + "longitude": "-118.456", + "locationName": "I-5 @ Main", + "route": "I-5", + }, + "inService": "true", + "imageData": { + "streamingVideoURL": "viewer?id=123", + "static": {"currentImageURL": "/images/cam123.jpg"}, + }, + "index": 123, + } + } + ] + } + + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", lambda *args, **kwargs: _Response()) + + cameras = cctv_pipeline.CaltransIngestor().fetch_data() + + assert cameras[0]["media_url"] == "https://cwwp2.dot.ca.gov/images/cam123.jpg" + assert cameras[0]["media_type"] == "image" + + +def test_georgia_ingestor_uses_public_511ga_feed_and_paginates(monkeypatch): + from services import cctv_pipeline + + class _Response: + def __init__(self, payload): + self.status_code = 200 + self._payload = payload + + def json(self): + return self._payload + + responses = [ + { + "recordsTotal": 2, + "data": [ + { + "id": 14968, + "location": "ALPH-0050: Rucker Rd at Charlotte Dr (Alpharetta)", + "latLng": { + "geography": { + "wellKnownText": "POINT (-84.33039 34.076298)", + } + }, + "images": [ + { + "id": 22378, + "imageUrl": "/map/Cctv/22378", + "blocked": False, + } + ], + } + ], + }, + { + "recordsTotal": 2, + "data": [ + { + "id": 14969, + "location": "BARR-0034: SR 211 at Pinot Nior Dr (Barrow)", + "latLng": { + "geography": { + "wellKnownText": "POINT (-83.81524 34.10526)", + } + }, + "images": [ + { + "id": 22379, + "imageUrl": "/map/Cctv/22379", + "blocked": False, + } + ], + } + ], + }, + ] + calls = [] + + def _fake_fetch(url, method="GET", json_data=None, timeout=15, headers=None): + calls.append( + { + "url": url, + "method": method, + "json_data": json_data, + "headers": headers, + } + ) + return _Response(responses.pop(0)) + + monkeypatch.setattr(cctv_pipeline.GeorgiaDOTIngestor, "PAGE_SIZE", 1) + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", _fake_fetch) + + cameras = cctv_pipeline.GeorgiaDOTIngestor().fetch_data() + + assert [cam["id"] for cam in cameras] == ["GDOT-14968", "GDOT-14969"] + assert cameras[0]["media_url"] == "https://511ga.org/map/Cctv/22378" + assert cameras[0]["media_type"] == "image" + assert cameras[0]["lat"] == pytest.approx(34.076298) + assert cameras[0]["lon"] == pytest.approx(-84.33039) + assert len(calls) == 2 + assert all(call["url"] == "https://511ga.org/List/GetData/Cameras" for call in calls) + assert all(call["method"] == "POST" for call in calls) + assert calls[0]["json_data"] == {"draw": 1, "start": 0, "length": 1} + assert calls[1]["json_data"] == {"draw": 2, "start": 1, "length": 1} + assert calls[0]["headers"]["Referer"] == "https://511ga.org/cctv" + + +def test_michigan_ingestor_absolutizes_relative_image_urls(monkeypatch): + from services import cctv_pipeline + + class _Response: + status_code = 200 + + def json(self): + return [ + { + "county": "id=42&lat=42.3314&lon=-83.0458", + "image": '', + "route": "I-94", + "location": "Downtown", + } + ] + + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", lambda *args, **kwargs: _Response()) + + cameras = cctv_pipeline.MichiganDOTIngestor().fetch_data() + + assert cameras[0]["media_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/image/42.jpg" + assert cameras[0]["media_type"] == "image" + + +def test_austin_ingestor_prefers_source_screenshot_address_and_filters_disabled(monkeypatch): + from services import cctv_pipeline + + class _Response: + def raise_for_status(self): + return None + + def json(self): + return [ + { + "camera_id": "316", + "camera_status": "TURNED_ON", + "location_name": "Austin Camera 316", + "screenshot_address": "https://cctv.austinmobility.io/image/316.jpg", + "location": {"coordinates": [-97.74, 30.24]}, + }, + { + "camera_id": "317", + "camera_status": "TURNED_OFF", + "location_name": "Austin Camera 317", + "screenshot_address": "https://cctv.austinmobility.io/image/317.jpg", + "location": {"coordinates": [-97.75, 30.25]}, + }, + ] + + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", lambda *args, **kwargs: _Response()) + + cameras = cctv_pipeline.AustinTXIngestor().fetch_data() + + assert len(cameras) == 1 + assert cameras[0]["id"] == "ATX-316" + assert cameras[0]["media_url"] == "https://cctv.austinmobility.io/image/316.jpg" + assert cameras[0]["media_type"] == "image" + + +def test_cctv_proxy_profiles_are_source_specific(): + import main + + tfl = main._cctv_proxy_profile_for_url("https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/00001.mp4") + austin = main._cctv_proxy_profile_for_url("https://cctv.austinmobility.io/image/316.jpg") + georgia = main._cctv_proxy_profile_for_url("https://511ga.org/map/Cctv/22378") + spain = main._cctv_proxy_profile_for_url("https://infocar.dgt.es/etraffic/data/camaras/1050.jpg") + + assert tfl.name == "tfl-jamcam" + assert tfl.headers["Accept"].startswith("video/mp4") + assert austin.name == "austin-mobility" + assert austin.headers["Origin"] == "https://data.mobility.austin.gov" + assert georgia.name == "gdot-511ga-image" + assert georgia.timeout == (5.0, 12.0) + assert georgia.headers["Referer"] == "https://511ga.org/cctv" + assert spain.name == "dgt-spain" + assert spain.headers["Referer"] == "https://infocar.dgt.es/" + + +def test_cctv_proxy_preserves_upstream_http_status(monkeypatch): + import main + + class _Response: + status_code = 404 + headers = {} + + def close(self): + return None + + monkeypatch.setattr("requests.get", lambda *args, **kwargs: _Response()) + + request = SimpleNamespace(headers={}) + profile = main._cctv_proxy_profile_for_url("https://infocar.dgt.es/etraffic/data/camaras/1050.jpg") + + with pytest.raises(main.HTTPException) as exc: + main._fetch_cctv_upstream_response(request, "https://infocar.dgt.es/etraffic/data/camaras/1050.jpg", profile) + + assert exc.value.status_code == 404 + assert exc.value.detail == "Upstream returned 404" + + +def test_colorado_ingestor_prefers_preview_image_with_hls_fallback(monkeypatch): + from services import cctv_pipeline + + class _Response: + status_code = 200 + + def json(self): + return [ + { + "id": 1, + "public": True, + "active": True, + "name": "I-70 EB", + "location": {"latitude": 39.7, "longitude": -105.2, "routeId": "I-70"}, + "cameraOwner": {"name": "Colorado DOT"}, + "views": [ + { + "url": "https://publicstreamer2.cotrip.org/rtplive/test/playlist.m3u8", + "videoPreviewUrl": "https://cocam.carsprogram.org/Snapshots/test.flv.png", + } + ], + }, + { + "id": 2, + "public": True, + "active": True, + "name": "US-285 NB", + "location": {"latitude": 39.6, "longitude": -105.1, "routeId": "US-285"}, + "cameraOwner": {"name": "Colorado DOT"}, + "views": [ + { + "url": "", + "videoPreviewUrl": "https://cocam.carsprogram.org/Snapshots/test2.flv.png", + } + ], + }, + ] + + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", lambda *args, **kwargs: _Response()) + + cameras = cctv_pipeline.ColoradoDOTIngestor().fetch_data() + + assert cameras[0]["media_url"] == "https://cocam.carsprogram.org/Snapshots/test.flv.png" + assert cameras[0]["media_type"] == "image" + assert cameras[1]["media_url"] == "https://cocam.carsprogram.org/Snapshots/test2.flv.png" + assert cameras[1]["media_type"] == "image" + + +def test_caltrans_ingestor_prefers_static_image_over_flaky_hls(monkeypatch): + from services import cctv_pipeline + + class _Response: + status_code = 200 + + def json(self): + return { + "data": [ + { + "cctv": { + "index": "1", + "inService": "true", + "location": { + "latitude": "37.82539", + "longitude": "-122.27291", + "locationName": "TV102 -- I-580 : West of SR-24", + }, + "imageData": { + "streamingVideoURL": "https://wzmedia.dot.ca.gov/D4/W580_JWO_24_IC.stream/playlist.m3u8", + "static": { + "currentImageURL": "https://cwwp2.dot.ca.gov/data/d4/cctv/image/tv102i580westofsr24/tv102i580westofsr24.jpg" + }, + }, + } + } + ] + } + + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", lambda *args, **kwargs: _Response()) + monkeypatch.setattr(cctv_pipeline.CaltransIngestor, "DISTRICTS", [4]) + + cameras = cctv_pipeline.CaltransIngestor().fetch_data() + + assert len(cameras) == 1 + assert cameras[0]["media_type"] == "image" + assert cameras[0]["media_url"].endswith("tv102i580westofsr24.jpg") + + +def test_dgt_ingestor_skips_dead_seed_urls(monkeypatch): + from services import cctv_pipeline + + monkeypatch.setattr( + cctv_pipeline, + "_media_url_reachable", + lambda url, **kwargs: url.endswith("/1001.jpg"), + ) + + cameras = cctv_pipeline.DGTNationalIngestor().fetch_data() + + assert len(cameras) == 1 + assert cameras[0]["id"] == "DGT-1001" + + +def test_base_ingestor_prunes_stale_rows_for_successful_source_refresh(tmp_path, monkeypatch): + import sqlite3 + from services import cctv_pipeline + + db_path = tmp_path / "cctv.db" + monkeypatch.setattr(cctv_pipeline, "DB_PATH", db_path) + cctv_pipeline.init_db() + + rows = [ + { + "id": "DGT-1001", + "source_agency": "DGT Spain", + "lat": 40.4, + "lon": -3.7, + "direction_facing": "A-6 Madrid", + "media_url": "https://infocar.dgt.es/etraffic/data/camaras/1001.jpg", + "media_type": "image", + "refresh_rate_seconds": 300, + }, + { + "id": "DGT-1002", + "source_agency": "DGT Spain", + "lat": 40.45, + "lon": -3.68, + "direction_facing": "A-2 Madrid", + "media_url": "https://infocar.dgt.es/etraffic/data/camaras/1002.jpg", + "media_type": "image", + "refresh_rate_seconds": 300, + }, + ] + + class _Ingestor(cctv_pipeline.BaseCCTVIngestor): + def fetch_data(self): + return list(rows) + + _Ingestor().ingest() + rows.pop() + _Ingestor().ingest() + + conn = sqlite3.connect(db_path) + try: + stored_ids = [row[0] for row in conn.execute("select id from cameras order by id")] + finally: + conn.close() + + assert stored_ids == ["DGT-1001"] + + +def test_osm_ingestor_keeps_only_direct_media_urls(monkeypatch): + from services import cctv_pipeline + + class _Response: + status_code = 200 + + def json(self): + return { + "elements": [ + { + "id": 101, + "lat": 39.7392, + "lon": -104.9903, + "tags": { + "camera:type": "traffic_monitoring", + "camera:url": "https://example.gov/cam101/playlist.m3u8", + "camera:direction": "270", + "operator": "Colorado DOT", + }, + }, + { + "id": 102, + "lat": 39.7400, + "lon": -104.9910, + "tags": { + "camera:type": "traffic_monitoring", + "website": "https://example.gov/traffic/cameras", + }, + }, + ] + } + + monkeypatch.setattr(cctv_pipeline, "fetch_with_curl", lambda *args, **kwargs: _Response()) + + cameras = cctv_pipeline.OSMTrafficCameraIngestor().fetch_data() + + assert len(cameras) == 1 + assert cameras[0]["id"] == "OSM-101" + assert cameras[0]["media_type"] == "hls" + assert cameras[0]["direction_facing"] == "270" + + +def test_cctv_proxy_allows_colorado_media_hosts(): + import main + + assert main._cctv_host_allowed("publicstreamer2.cotrip.org") + assert main._cctv_host_allowed("cocam.carsprogram.org") + + +def test_data_fetcher_cctv_scheduler_includes_colorado_and_osm(): + from pathlib import Path + + source = Path("backend/services/data_fetcher.py").read_text(encoding="utf-8") + + assert "ColoradoDOTIngestor()" in source + assert "OSMTrafficCameraIngestor()" in source diff --git a/backend/tests/test_store.py b/backend/tests/test_store.py index 63429856..eee16767 100644 --- a/backend/tests/test_store.py +++ b/backend/tests/test_store.py @@ -1,4 +1,5 @@ """Tests for the shared in-memory data store.""" + import threading import time import pytest @@ -10,19 +11,46 @@ class TestLatestDataStructure: def test_has_all_required_keys(self): expected_keys = { - "last_updated", "news", "stocks", "oil", "flights", "ships", - "military_flights", "tracked_flights", "cctv", "weather", - "earthquakes", "uavs", "frontlines", "gdelt", "liveuamap", - "kiwisdr", "space_weather", "internet_outages", "firms_fires", - "datacenters" + "last_updated", + "news", + "stocks", + "oil", + "flights", + "ships", + "military_flights", + "tracked_flights", + "cctv", + "weather", + "earthquakes", + "uavs", + "frontlines", + "gdelt", + "liveuamap", + "kiwisdr", + "space_weather", + "internet_outages", + "firms_fires", + "datacenters", } assert expected_keys.issubset(set(latest_data.keys())) def test_list_keys_default_to_empty_list(self): - list_keys = ["news", "flights", "ships", "military_flights", - "tracked_flights", "cctv", "earthquakes", "uavs", - "gdelt", "liveuamap", "kiwisdr", "internet_outages", - "firms_fires", "datacenters"] + list_keys = [ + "news", + "flights", + "ships", + "military_flights", + "tracked_flights", + "cctv", + "earthquakes", + "uavs", + "gdelt", + "liveuamap", + "kiwisdr", + "internet_outages", + "firms_fires", + "datacenters", + ] for key in list_keys: assert isinstance(latest_data[key], list), f"{key} should default to list" diff --git a/backend/tests/test_trains_fetcher.py b/backend/tests/test_trains_fetcher.py new file mode 100644 index 00000000..8f03793c --- /dev/null +++ b/backend/tests/test_trains_fetcher.py @@ -0,0 +1,150 @@ +from services.fetchers import trains as train_fetcher +from services.fetchers._store import _data_lock, latest_data + + +def setup_function(): + train_fetcher._TRAIN_TRACK_CACHE.clear() + with _data_lock: + latest_data["trains"] = [] + + +def test_merge_nonredundant_trains_prefers_higher_priority_source_and_backfills_fields(): + lower_fidelity = train_fetcher._normalize_train( + source="amtrak", + raw_id="AMTK-8", + number="8", + lat=40.0000, + lng=-75.0000, + route="New York -> Chicago", + status="On Time", + operator="Shared Rail", + country="US", + observed_at="2026-03-22T18:00:00Z", + ) + higher_fidelity = train_fetcher._normalize_train( + source="digitraffic", + raw_id="FIN-8", + number="8", + lat=40.0080, + lng=-75.0020, + speed_kmh=128.4, + status="Active", + operator="Shared Rail", + country="US", + observed_at="2026-03-22T18:00:05Z", + ) + + merged = train_fetcher._merge_nonredundant_trains([lower_fidelity], [higher_fidelity]) + + assert len(merged) == 1 + train = merged[0] + assert train["source"] == "digitraffic" + assert train["speed_kmh"] == 128.4 + assert train["route"] == "New York -> Chicago" + assert train["source_label"] == "Digitraffic Finland" + + +def test_motion_estimates_infer_speed_and_heading_from_previous_position(): + first = train_fetcher._normalize_train( + source="amtrak", + raw_id="AMTK-14", + number="14", + lat=40.0000, + lng=-75.0000, + observed_at=1_000, + ) + assert first is not None + second = train_fetcher._normalize_train( + source="amtrak", + raw_id="AMTK-14", + number="14", + lat=40.0000, + lng=-74.9900, + observed_at=1_060, + ) + + assert second is not None + assert second["speed_kmh"] is not None + assert 40.0 <= second["speed_kmh"] <= 80.0 + assert second["heading"] is not None + assert 80.0 <= second["heading"] <= 100.0 + + +def test_fetch_trains_merges_sources_into_store(monkeypatch): + def _batch_one(): + return [ + train_fetcher._normalize_train( + source="amtrak", + raw_id="AMTK-22", + number="22", + lat=41.0000, + lng=-87.0000, + route="Chicago -> St. Louis", + operator="Shared Rail", + country="US", + observed_at="2026-03-22T19:00:00Z", + ) + ] + + def _batch_two(): + return [ + train_fetcher._normalize_train( + source="digitraffic", + raw_id="FIN-22", + number="22", + lat=41.0040, + lng=-87.0010, + speed_kmh=96.0, + operator="Shared Rail", + country="US", + observed_at="2026-03-22T19:00:05Z", + ) + ] + + monkeypatch.setattr( + train_fetcher, + "_TRAIN_FETCHERS", + (("amtrak", _batch_one), ("digitraffic", _batch_two)), + ) + + train_fetcher.fetch_trains() + + with _data_lock: + trains = list(latest_data["trains"]) + + assert len(trains) == 1 + assert trains[0]["source"] == "digitraffic" + assert trains[0]["route"] == "Chicago -> St. Louis" + assert trains[0]["speed_kmh"] == 96.0 + + +def test_fetch_trains_preserves_last_good_snapshot_when_refresh_fails(monkeypatch): + with _data_lock: + latest_data["trains"] = [ + { + "id": "AMTK-1", + "name": "Sunset Limited", + "number": "1", + "source": "amtrak", + "lat": 32.33, + "lng": -109.76, + "speed_kmh": None, + "heading": None, + "status": "On Time", + "route": "New Orleans -> Los Angeles", + } + ] + + monkeypatch.setattr( + train_fetcher, + "_TRAIN_FETCHERS", + (("amtrak", lambda: []), ("digitraffic", lambda: [])), + ) + + train_fetcher.fetch_trains() + + with _data_lock: + trains = list(latest_data["trains"]) + + assert len(trains) == 1 + assert trains[0]["id"] == "AMTK-1" diff --git a/backend/wormhole_server.py b/backend/wormhole_server.py new file mode 100644 index 00000000..15251040 --- /dev/null +++ b/backend/wormhole_server.py @@ -0,0 +1,121 @@ +"""Wormhole local agent — Reticulum-enabled mesh-only API server.""" + +from __future__ import annotations + +import os +import socket +import sys +import threading +import time +from pathlib import Path + +import uvicorn + +from services.wormhole_settings import read_wormhole_settings +from services.wormhole_status import write_wormhole_status + + +def _env_flag(name: str, default: str = "") -> bool: + value = os.environ.get(name, default).strip().lower() + return value in ("1", "true", "yes") + + +os.environ.setdefault("MESH_ONLY", "true") +os.environ.setdefault("MESH_RNS_ENABLED", "true") + +HOST = os.environ.get("WORMHOLE_HOST", "127.0.0.1") +PORT = int(os.environ.get("WORMHOLE_PORT", "8787")) +RELOAD = _env_flag("WORMHOLE_RELOAD") + +settings = read_wormhole_settings() +TRANSPORT = os.environ.get("WORMHOLE_TRANSPORT", "") or settings.get("transport", "direct") +SOCKS_PROXY = os.environ.get("WORMHOLE_SOCKS_PROXY", "") or settings.get("socks_proxy", "") +SOCKS_DNS = _env_flag("WORMHOLE_SOCKS_DNS", "true" if settings.get("socks_dns") else "false") +TRANSPORT_ACTIVE = "direct" +PROXY_ACTIVE = "" + +if TRANSPORT.lower() in ("tor", "i2p", "mixnet") and SOCKS_PROXY: + try: + import socks # type: ignore + + host, port_str = SOCKS_PROXY.split(":") + socks.set_default_proxy(socks.SOCKS5, host, int(port_str), rdns=SOCKS_DNS) + socket.socket = socks.socksocket # type: ignore + TRANSPORT_ACTIVE = TRANSPORT.lower() + PROXY_ACTIVE = SOCKS_PROXY + os.environ["WORMHOLE_TRANSPORT_ACTIVE"] = TRANSPORT_ACTIVE + print(f"[*] Wormhole transport: {TRANSPORT} via SOCKS5 {SOCKS_PROXY}") + except Exception as exc: + print(f"[!] Wormhole transport init failed: {exc}") + print("[!] Continuing without hidden transport.") +write_wormhole_status( + reason="startup", + transport=TRANSPORT, + proxy=SOCKS_PROXY, + transport_active=TRANSPORT_ACTIVE, + proxy_active=PROXY_ACTIVE, + restart=False, + installed=True, + configured=True, + running=True, + ready=False, + pid=os.getpid(), + started_at=int(time.time()), + last_error="", + privacy_level_effective=str(settings.get("privacy_profile", "default")), +) + + +def _watch_transport_settings() -> None: + settings_path = Path(__file__).resolve().parent / "data" / "wormhole.json" + last_mtime = settings_path.stat().st_mtime if settings_path.exists() else 0 + while True: + time.sleep(2) + try: + current_mtime = settings_path.stat().st_mtime if settings_path.exists() else 0 + if current_mtime == last_mtime: + continue + last_mtime = current_mtime + new_settings = read_wormhole_settings() + new_transport = str(new_settings.get("transport", "direct")) + new_proxy = str(new_settings.get("socks_proxy", "")) + new_dns = "true" if bool(new_settings.get("socks_dns", True)) else "false" + if ( + new_transport.lower() != TRANSPORT.lower() + or new_proxy != SOCKS_PROXY + or new_dns != ("true" if SOCKS_DNS else "false") + ): + print("[*] Wormhole transport settings changed — restarting agent to apply.") + write_wormhole_status( + reason="transport_change", + transport=new_transport, + proxy=new_proxy, + transport_active="", + proxy_active="", + restart=True, + installed=True, + configured=True, + running=True, + ready=False, + pid=os.getpid(), + started_at=int(time.time()), + last_error="", + privacy_level_effective=str(new_settings.get("privacy_profile", "default")), + ) + os.environ["WORMHOLE_TRANSPORT"] = new_transport + os.environ["WORMHOLE_SOCKS_PROXY"] = new_proxy + os.environ["WORMHOLE_SOCKS_DNS"] = new_dns + os.execv(sys.executable, [sys.executable, __file__]) + except Exception: + continue + + +if __name__ == "__main__": + threading.Thread(target=_watch_transport_settings, daemon=True).start() + uvicorn.run( + "main:app", + host=HOST, + port=PORT, + reload=RELOAD, + log_level="info", + ) diff --git a/backend/wsdot_sample.json b/backend/wsdot_sample.json deleted file mode 100644 index b232c3b3..00000000 --- a/backend/wsdot_sample.json +++ /dev/null @@ -1 +0,0 @@ -Bad Request \ No newline at end of file diff --git a/desktop-shell/README.md b/desktop-shell/README.md new file mode 100644 index 00000000..ccc80bdc --- /dev/null +++ b/desktop-shell/README.md @@ -0,0 +1,51 @@ +# Desktop Shell Scaffold + +This folder is the first native-side scaffold for the staged desktop boundary. + +## Purpose + +It gives the future Tauri/native shell a concrete shape for: + +- command routing +- handler grouping +- runtime bridge installation + +without forcing a packaging migration yet. + +## Source of truth + +The shared desktop control contract still lives in: + +- `F:\Codebase\Oracle\live-risk-dashboard\frontend\src\lib\desktopControlContract.ts` + +The native-side scaffold imports that contract rather than redefining it. + +## First command scope + +The initial native command set covers only: + +- Wormhole lifecycle +- protected settings get/set +- update trigger + +That is deliberate. The goal is to move the local privileged control plane first, not the entire +mesh data plane. + +## Scaffold layout + +- `F:\Codebase\Oracle\live-risk-dashboard\desktop-shell\src\types.ts` +- `F:\Codebase\Oracle\live-risk-dashboard\desktop-shell\src\handlers\wormholeHandlers.ts` +- `F:\Codebase\Oracle\live-risk-dashboard\desktop-shell\src\handlers\settingsHandlers.ts` +- `F:\Codebase\Oracle\live-risk-dashboard\desktop-shell\src\handlers\updateHandlers.ts` +- `F:\Codebase\Oracle\live-risk-dashboard\desktop-shell\src\nativeControlRouter.ts` +- `F:\Codebase\Oracle\live-risk-dashboard\desktop-shell\src\runtimeBridge.ts` + +## How to use later + +When the Tauri shell is introduced, its command layer should: + +1. receive `invokeLocalControl(command, payload)` +2. dispatch through `createNativeControlRouter(...)` +3. return the handler result back to the frontend bridge + +This keeps the frontend contract stable while shifting privileged ownership into the native shell. diff --git a/desktop-shell/src/handlers/settingsHandlers.ts b/desktop-shell/src/handlers/settingsHandlers.ts new file mode 100644 index 00000000..5027c387 --- /dev/null +++ b/desktop-shell/src/handlers/settingsHandlers.ts @@ -0,0 +1,50 @@ +import type { NativeControlHandlerMap } from '../types'; + +export function createSettingsHandlers(): Pick< + NativeControlHandlerMap, + | 'settings.wormhole.get' + | 'settings.wormhole.set' + | 'settings.privacy.get' + | 'settings.privacy.set' + | 'settings.api_keys.get' + | 'settings.api_keys.set' + | 'settings.news.get' + | 'settings.news.set' + | 'settings.news.reset' +> { + return { + 'settings.wormhole.get': async (_payload, _ctx, exec) => exec('/api/settings/wormhole'), + 'settings.wormhole.set': async (payload, _ctx, exec) => + exec('/api/settings/wormhole', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'settings.privacy.get': async (_payload, _ctx, exec) => + exec('/api/settings/privacy-profile'), + 'settings.privacy.set': async (payload, _ctx, exec) => + exec('/api/settings/privacy-profile', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'settings.api_keys.get': async (_payload, _ctx, exec) => exec('/api/settings/api-keys'), + 'settings.api_keys.set': async (payload, _ctx, exec) => + exec('/api/settings/api-keys', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'settings.news.get': async (_payload, _ctx, exec) => exec('/api/settings/news-feeds'), + 'settings.news.set': async (payload, _ctx, exec) => + exec('/api/settings/news-feeds', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'settings.news.reset': async (_payload, _ctx, exec) => + exec('/api/settings/news-feeds/reset', { + method: 'POST', + }), + }; +} diff --git a/desktop-shell/src/handlers/updateHandlers.ts b/desktop-shell/src/handlers/updateHandlers.ts new file mode 100644 index 00000000..e87e46b4 --- /dev/null +++ b/desktop-shell/src/handlers/updateHandlers.ts @@ -0,0 +1,10 @@ +import type { NativeControlHandlerMap } from '../types'; + +export function createUpdateHandlers(): Pick { + return { + 'system.update': async (_payload, _ctx, exec) => + exec('/api/system/update', { + method: 'POST', + }), + }; +} diff --git a/desktop-shell/src/handlers/wormholeHandlers.ts b/desktop-shell/src/handlers/wormholeHandlers.ts new file mode 100644 index 00000000..b010fab9 --- /dev/null +++ b/desktop-shell/src/handlers/wormholeHandlers.ts @@ -0,0 +1,102 @@ +import type { NativeControlHandlerMap } from '../types'; + +export function createWormholeHandlers(): Pick< + NativeControlHandlerMap, + | 'wormhole.status' + | 'wormhole.connect' + | 'wormhole.disconnect' + | 'wormhole.restart' + | 'wormhole.gate.enter' + | 'wormhole.gate.leave' + | 'wormhole.gate.proof' + | 'wormhole.gate.personas.get' + | 'wormhole.gate.persona.create' + | 'wormhole.gate.persona.activate' + | 'wormhole.gate.persona.clear' + | 'wormhole.gate.key.get' + | 'wormhole.gate.key.rotate' + | 'wormhole.gate.message.compose' + | 'wormhole.gate.message.decrypt' + | 'wormhole.gate.message.post' + | 'wormhole.gate.messages.decrypt' +> { + return { + 'wormhole.status': async (_payload, _ctx, exec) => exec('/api/wormhole/status'), + 'wormhole.connect': async (_payload, _ctx, exec) => + exec('/api/wormhole/connect', { method: 'POST' }), + 'wormhole.disconnect': async (_payload, _ctx, exec) => + exec('/api/wormhole/disconnect', { method: 'POST' }), + 'wormhole.restart': async (_payload, _ctx, exec) => + exec('/api/wormhole/restart', { method: 'POST' }), + 'wormhole.gate.personas.get': async (payload, _ctx, exec) => + exec(`/api/wormhole/gate/${encodeURIComponent(String(payload?.gate_id || ''))}/personas`), + 'wormhole.gate.persona.create': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/persona/create', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.persona.activate': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/persona/activate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.persona.clear': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/persona/clear', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.key.get': async (payload, _ctx, exec) => + exec(`/api/wormhole/gate/${encodeURIComponent(String(payload?.gate_id || ''))}/key`), + 'wormhole.gate.key.rotate': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/key/rotate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.message.compose': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/message/compose', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.message.decrypt': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/message/decrypt', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.enter': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/enter', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.leave': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/leave', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.proof': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/proof', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.message.post': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/message/post', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + 'wormhole.gate.messages.decrypt': async (payload, _ctx, exec) => + exec('/api/wormhole/gate/messages/decrypt', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }), + }; +} diff --git a/desktop-shell/src/index.ts b/desktop-shell/src/index.ts new file mode 100644 index 00000000..6004ed9f --- /dev/null +++ b/desktop-shell/src/index.ts @@ -0,0 +1,4 @@ +export * from './nativeControlRouter'; +export * from './runtimeBridge'; +export * from './nativeControlAudit'; +export * from './types'; diff --git a/desktop-shell/src/nativeControlAudit.ts b/desktop-shell/src/nativeControlAudit.ts new file mode 100644 index 00000000..2146ad6a --- /dev/null +++ b/desktop-shell/src/nativeControlAudit.ts @@ -0,0 +1,60 @@ +import type { + DesktopControlAuditOutcome, + DesktopControlAuditRecord, + DesktopControlAuditReport, +} from '../../frontend/src/lib/desktopControlContract'; +import type { NativeControlAuditEvent, NativeControlAuditTrail } from './types'; + +const DEFAULT_LIMIT = 100; + +function incrementOutcome( + counts: Partial>, + outcome: DesktopControlAuditOutcome, +) { + counts[outcome] = (counts[outcome] || 0) + 1; +} + +export function createNativeControlAuditTrail(maxEntries: number = DEFAULT_LIMIT): NativeControlAuditTrail { + const entries: DesktopControlAuditRecord[] = []; + let totalRecorded = 0; + + return { + record(event: NativeControlAuditEvent) { + totalRecorded += 1; + entries.push({ + ...event, + recordedAt: Date.now(), + }); + if (entries.length > maxEntries) { + entries.splice(0, entries.length - maxEntries); + } + }, + snapshot(limit: number = 25): DesktopControlAuditReport { + const recent = entries.slice(-Math.max(1, limit)).reverse(); + const byOutcome: Partial> = {}; + let lastProfileMismatch: DesktopControlAuditRecord | undefined; + let lastDenied: DesktopControlAuditRecord | undefined; + for (const entry of entries) { + incrementOutcome(byOutcome, entry.outcome); + if (entry.outcome === 'profile_warn' || entry.outcome === 'profile_denied') { + lastProfileMismatch = entry; + } + if (entry.outcome === 'profile_denied' || entry.outcome === 'capability_denied') { + lastDenied = entry; + } + } + return { + totalEvents: entries.length, + totalRecorded, + recent, + byOutcome, + lastProfileMismatch, + lastDenied, + }; + }, + clear() { + totalRecorded = 0; + entries.splice(0, entries.length); + }, + }; +} diff --git a/desktop-shell/src/nativeControlRouter.ts b/desktop-shell/src/nativeControlRouter.ts new file mode 100644 index 00000000..aae350d9 --- /dev/null +++ b/desktop-shell/src/nativeControlRouter.ts @@ -0,0 +1,105 @@ +import type { + DesktopControlCommand, + DesktopControlPayloadMap, +} from '../../frontend/src/lib/desktopControlContract'; +import { + controlCommandCapability as resolveCommandCapability, + extractGateTargetRef, + sessionProfileCapabilities as capabilitiesForProfile, +} from '../../frontend/src/lib/desktopControlContract'; +import { createSettingsHandlers } from './handlers/settingsHandlers'; +import { createUpdateHandlers } from './handlers/updateHandlers'; +import { createWormholeHandlers } from './handlers/wormholeHandlers'; +import type { + NativeControlExecutor, + NativeControlHandlerContext, + NativeControlHandlerMap, + NativeControlInvokeMeta, +} from './types'; + +function createHandlerMap(): NativeControlHandlerMap { + return { + ...createWormholeHandlers(), + ...createSettingsHandlers(), + ...createUpdateHandlers(), + }; +} + +export function createNativeControlRouter( + ctx: NativeControlHandlerContext, + exec: NativeControlExecutor, +) { + const handlers = createHandlerMap(); + return { + async invoke( + command: C, + payload: DesktopControlPayloadMap[C], + meta?: NativeControlInvokeMeta, + ): Promise { + const handler = handlers[command]; + if (!handler) { + throw new Error(`native_control_handler_missing:${command}`); + } + const expectedCapability = resolveCommandCapability(command); + const profile = ctx.sessionProfile; + const profileCapabilities = profile ? capabilitiesForProfile(profile) : []; + const profileAllows = + !profile || profileCapabilities.length === 0 || profileCapabilities.includes(expectedCapability); + const profileEnforced = Boolean((ctx.enforceSessionProfile || meta?.enforceProfileHint) && profile); + const allowedCapabilitiesConfigured = + Array.isArray(ctx.allowedCapabilities) && ctx.allowedCapabilities.length > 0; + const capabilityDenied = + allowedCapabilitiesConfigured && !ctx.allowedCapabilities!.includes(expectedCapability); + const targetRef = extractGateTargetRef(command, payload); + const auditBase = { + command, + expectedCapability, + declaredCapability: meta?.capability, + ...(targetRef ? { targetRef } : {}), + sessionProfile: profile, + sessionProfileHint: meta?.sessionProfileHint, + enforceProfileHint: meta?.enforceProfileHint, + profileAllows, + allowedCapabilitiesConfigured, + enforced: profileEnforced, + } as const; + if (meta?.capability && meta.capability !== expectedCapability) { + ctx.auditControlUse?.({ + ...auditBase, + outcome: 'capability_mismatch', + }); + throw new Error( + `native_control_capability_mismatch:${meta.capability}:${expectedCapability}`, + ); + } + if (capabilityDenied) { + ctx.auditControlUse?.({ + ...auditBase, + outcome: 'capability_denied', + }); + throw new Error(`native_control_capability_denied:${expectedCapability}`); + } + if (!profileAllows) { + const profileMessage = `native_control_profile_mismatch:${profile}:${expectedCapability}`; + ctx.auditControlUse?.({ + ...auditBase, + outcome: profileEnforced ? 'profile_denied' : 'profile_warn', + }); + if (profileEnforced) { + throw new Error(profileMessage); + } + console.warn(profileMessage, { + command, + sessionProfileHint: meta?.sessionProfileHint, + }); + } + if (profileAllows) { + ctx.auditControlUse?.({ + ...auditBase, + outcome: 'allowed', + }); + } + return handler(payload, ctx, exec); + }, + }; +} diff --git a/desktop-shell/src/runtimeBridge.ts b/desktop-shell/src/runtimeBridge.ts new file mode 100644 index 00000000..ad1af0f5 --- /dev/null +++ b/desktop-shell/src/runtimeBridge.ts @@ -0,0 +1,65 @@ +import type { + DesktopControlCommand, + DesktopControlPayloadMap, + LocalControlInvokeMeta, +} from '../../frontend/src/lib/desktopControlContract'; +import { createNativeControlAuditTrail } from './nativeControlAudit'; +import { createNativeControlRouter } from './nativeControlRouter'; +import type { + NativeControlAuditEvent, + NativeControlExecutor, + NativeControlHandlerContext, +} from './types'; + +async function defaultExecutor(baseUrl: string, path: string, init: RequestInit = {}): Promise { + const res = await fetch(`${baseUrl}${path}`, init); + const data = await res.json().catch(() => ({})); + if (!res.ok || data?.ok === false) { + throw new Error(data?.detail || data?.message || 'native_control_request_failed'); + } + return data as T; +} + +export function createRuntimeBridge(ctx: NativeControlHandlerContext) { + const auditTrail = ctx.auditTrail || createNativeControlAuditTrail(); + const auditControlUse = (event: NativeControlAuditEvent) => { + auditTrail.record(event); + ctx.auditControlUse?.(event); + }; + const exec: NativeControlExecutor = (path: string, init: RequestInit = {}) => { + const headers = new Headers(init.headers); + if (ctx.adminKey && !headers.has('X-Admin-Key')) { + headers.set('X-Admin-Key', ctx.adminKey); + } + return defaultExecutor(ctx.backendBaseUrl, path, { ...init, headers }); + }; + function invocationContext(meta?: LocalControlInvokeMeta): NativeControlHandlerContext { + const baseCtx: NativeControlHandlerContext = { + ...ctx, + auditTrail, + auditControlUse, + }; + if (ctx.sessionProfile || !meta?.sessionProfileHint) { + return baseCtx; + } + return { + ...baseCtx, + sessionProfile: meta.sessionProfileHint, + }; + } + return { + invokeLocalControl( + command: C, + payload: DesktopControlPayloadMap[C], + meta?: LocalControlInvokeMeta, + ) { + return createNativeControlRouter(invocationContext(meta), exec).invoke(command, payload, meta); + }, + getNativeControlAuditReport(limit?: number) { + return auditTrail.snapshot(limit); + }, + clearNativeControlAuditReport() { + auditTrail.clear(); + }, + }; +} diff --git a/desktop-shell/src/types.ts b/desktop-shell/src/types.ts new file mode 100644 index 00000000..6f5c2873 --- /dev/null +++ b/desktop-shell/src/types.ts @@ -0,0 +1,43 @@ +import type { + DesktopControlAuditEvent, + DesktopControlAuditReport, + DesktopControlCapability, + DesktopControlCommand, + DesktopControlPayloadMap, + DesktopControlSessionProfile, + LocalControlInvokeMeta, +} from '../../frontend/src/lib/desktopControlContract'; + +export type NativeControlHandlerContext = { + backendBaseUrl: string; + wormholeBaseUrl: string; + adminKey?: string; + allowedCapabilities?: DesktopControlCapability[]; + sessionProfile?: DesktopControlSessionProfile; + enforceSessionProfile?: boolean; + auditControlUse?: (event: NativeControlAuditEvent) => void; + auditTrail?: NativeControlAuditTrail; +}; + +export type NativeControlExecutor = ( + path: string, + init?: RequestInit, +) => Promise; + +export type NativeControlHandlerMap = { + [K in DesktopControlCommand]: ( + payload: DesktopControlPayloadMap[K], + ctx: NativeControlHandlerContext, + exec: NativeControlExecutor, + ) => Promise; +}; + +export type NativeControlInvokeMeta = LocalControlInvokeMeta; + +export type NativeControlAuditEvent = DesktopControlAuditEvent; + +export type NativeControlAuditTrail = { + record: (event: NativeControlAuditEvent) => void; + snapshot: (limit?: number) => DesktopControlAuditReport; + clear: () => void; +}; diff --git a/desktop-shell/tauri-skeleton/README.md b/desktop-shell/tauri-skeleton/README.md new file mode 100644 index 00000000..9d291c6e --- /dev/null +++ b/desktop-shell/tauri-skeleton/README.md @@ -0,0 +1,33 @@ +# Tauri Skeleton + +This folder is the first concrete Tauri-side integration skeleton for the desktop boundary. + +## Scope + +It is intentionally limited to the first trusted local control-plane command set: + +- Wormhole lifecycle +- protected settings reads/writes +- update trigger + +It does **not** attempt to move DM/data-plane operations yet. + +## What this scaffold demonstrates + +1. a native `invoke_local_control` command entrypoint +2. a small Rust-side router for the first command set +3. backend HTTP delegation with native-side admin-key ownership +4. a simple webview runtime injection path for: + - `window.__SHADOWBROKER_DESKTOP__.invokeLocalControl(...)` + +## Important note + +This is a scaffold, not a fully integrated desktop app yet. It exists so the next Tauri pass has a +clear structure instead of starting from scratch. + +## Shared contract + +The command names this scaffold must track are defined in: + +- `F:\Codebase\Oracle\live-risk-dashboard\frontend\src\lib\desktopControlContract.ts` +- `F:\Codebase\Oracle\live-risk-dashboard\frontend\src\lib\desktopControlRouting.ts` diff --git a/desktop-shell/tauri-skeleton/src-tauri/Cargo.toml b/desktop-shell/tauri-skeleton/src-tauri/Cargo.toml new file mode 100644 index 00000000..2c3a44d6 --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "shadowbroker-tauri-shell" +version = "0.1.0" +edition = "2021" + +[build-dependencies] +tauri-build = { version = "2" } + +[dependencies] +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tauri = { version = "2", features = [] } diff --git a/desktop-shell/tauri-skeleton/src-tauri/build.rs b/desktop-shell/tauri-skeleton/src-tauri/build.rs new file mode 100644 index 00000000..d860e1e6 --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/build.rs @@ -0,0 +1,3 @@ +fn main() { + tauri_build::build() +} diff --git a/desktop-shell/tauri-skeleton/src-tauri/src/bridge.rs b/desktop-shell/tauri-skeleton/src-tauri/src/bridge.rs new file mode 100644 index 00000000..84aa39c5 --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/src/bridge.rs @@ -0,0 +1,19 @@ +use serde_json::Value; +use tauri::State; + +use crate::{handlers::dispatch_control_command, DesktopAppState}; + +#[tauri::command] +pub async fn invoke_local_control( + command: String, + payload: Option, + state: State<'_, DesktopAppState>, +) -> Result { + dispatch_control_command( + &state.backend_base_url, + state.admin_key.as_deref(), + &command, + payload, + ) + .await +} diff --git a/desktop-shell/tauri-skeleton/src-tauri/src/handlers.rs b/desktop-shell/tauri-skeleton/src-tauri/src/handlers.rs new file mode 100644 index 00000000..39bc3870 --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/src/handlers.rs @@ -0,0 +1,57 @@ +use reqwest::Method; +use serde_json::Value; + +use crate::http_client::call_backend_json; + +pub async fn dispatch_control_command( + backend_base_url: &str, + admin_key: Option<&str>, + command: &str, + payload: Option, +) -> Result { + match command { + "wormhole.status" => { + call_backend_json(backend_base_url, admin_key, "/api/wormhole/status", Method::GET, None).await + } + "wormhole.connect" => { + call_backend_json(backend_base_url, admin_key, "/api/wormhole/connect", Method::POST, None).await + } + "wormhole.disconnect" => { + call_backend_json(backend_base_url, admin_key, "/api/wormhole/disconnect", Method::POST, None).await + } + "wormhole.restart" => { + call_backend_json(backend_base_url, admin_key, "/api/wormhole/restart", Method::POST, None).await + } + "settings.wormhole.get" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/wormhole", Method::GET, None).await + } + "settings.wormhole.set" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/wormhole", Method::PUT, payload).await + } + "settings.privacy.get" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/privacy-profile", Method::GET, None).await + } + "settings.privacy.set" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/privacy-profile", Method::PUT, payload).await + } + "settings.api_keys.get" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/api-keys", Method::GET, None).await + } + "settings.api_keys.set" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/api-keys", Method::PUT, payload).await + } + "settings.news.get" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/news-feeds", Method::GET, None).await + } + "settings.news.set" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/news-feeds", Method::PUT, payload).await + } + "settings.news.reset" => { + call_backend_json(backend_base_url, admin_key, "/api/settings/news-feeds/reset", Method::POST, None).await + } + "system.update" => { + call_backend_json(backend_base_url, admin_key, "/api/system/update", Method::POST, None).await + } + _ => Err(format!("unsupported_control_command:{command}")), + } +} diff --git a/desktop-shell/tauri-skeleton/src-tauri/src/http_client.rs b/desktop-shell/tauri-skeleton/src-tauri/src/http_client.rs new file mode 100644 index 00000000..050dd80e --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/src/http_client.rs @@ -0,0 +1,40 @@ +use reqwest::Method; +use serde_json::Value; + +pub async fn call_backend_json( + base_url: &str, + admin_key: Option<&str>, + path: &str, + method: Method, + payload: Option, +) -> Result { + let client = reqwest::Client::new(); + let mut request = client.request(method, format!("{base_url}{path}")); + if let Some(key) = admin_key { + if !key.trim().is_empty() { + request = request.header("X-Admin-Key", key); + } + } + if let Some(value) = payload { + request = request.json(&value); + } + let response = request + .send() + .await + .map_err(|e| format!("backend_request_failed:{e}"))?; + let status = response.status(); + let text = response + .text() + .await + .map_err(|e| format!("backend_response_failed:{e}"))?; + let value: Value = serde_json::from_str(&text).unwrap_or_else(|_| serde_json::json!({})); + if !status.is_success() || value.get("ok") == Some(&Value::Bool(false)) { + let detail = value + .get("detail") + .and_then(|v| v.as_str()) + .or_else(|| value.get("message").and_then(|v| v.as_str())) + .unwrap_or("native_control_request_failed"); + return Err(detail.to_string()); + } + Ok(value) +} diff --git a/desktop-shell/tauri-skeleton/src-tauri/src/main.rs b/desktop-shell/tauri-skeleton/src-tauri/src/main.rs new file mode 100644 index 00000000..c45f498d --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/src/main.rs @@ -0,0 +1,37 @@ +mod bridge; +mod handlers; +mod http_client; + +use bridge::invoke_local_control; + +pub struct DesktopAppState { + pub backend_base_url: String, + pub admin_key: Option, +} + +fn main() { + let backend_base_url = + std::env::var("SHADOWBROKER_BACKEND_URL").unwrap_or_else(|_| "http://127.0.0.1:8000".to_string()); + let admin_key = std::env::var("SHADOWBROKER_ADMIN_KEY").ok(); + + tauri::Builder::default() + .manage(DesktopAppState { + backend_base_url, + admin_key, + }) + .invoke_handler(tauri::generate_handler![invoke_local_control]) + .setup(|app| { + if let Some(window) = app.get_webview_window("main") { + let script = r#" + window.__SHADOWBROKER_DESKTOP__ = { + invokeLocalControl: (command, payload) => + window.__TAURI__.core.invoke('invoke_local_control', { command, payload }) + }; + "#; + let _ = window.eval(script); + } + Ok(()) + }) + .run(tauri::generate_context!()) + .expect("failed to run shadowbroker tauri shell"); +} diff --git a/desktop-shell/tauri-skeleton/src-tauri/tauri.conf.json b/desktop-shell/tauri-skeleton/src-tauri/tauri.conf.json new file mode 100644 index 00000000..f70c43ea --- /dev/null +++ b/desktop-shell/tauri-skeleton/src-tauri/tauri.conf.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "productName": "ShadowBroker Desktop Shell", + "version": "0.1.0", + "identifier": "com.shadowbroker.desktop", + "build": { + "frontendDist": "../../frontend/.next", + "devUrl": "http://127.0.0.1:3000" + }, + "app": { + "windows": [ + { + "label": "main", + "title": "ShadowBroker", + "width": 1600, + "height": 1000, + "resizable": true + } + ] + } +} diff --git a/desktop-shell/tsconfig.json b/desktop-shell/tsconfig.json new file mode 100644 index 00000000..1c14f661 --- /dev/null +++ b/desktop-shell/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "lib": ["ES2022", "DOM"] + }, + "include": [ + "src/**/*.ts", + "../frontend/src/lib/desktopControlContract.ts" + ] +} diff --git a/docker-compose.relay.yml b/docker-compose.relay.yml new file mode 100644 index 00000000..68461c0e --- /dev/null +++ b/docker-compose.relay.yml @@ -0,0 +1,27 @@ +# Minimal relay-node compose — backend only, no frontend needed. +services: + backend: + build: + context: . + dockerfile: ./backend/Dockerfile + container_name: shadowbroker-relay + ports: + - "0.0.0.0:8000:8000" + env_file: .env + volumes: + - relay_data:/app/data + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + deploy: + resources: + limits: + memory: 2G + cpus: '2' + +volumes: + relay_data: diff --git a/docker-compose.yml b/docker-compose.yml index 687c1093..f89693e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,17 +11,18 @@ services: - OPENSKY_CLIENT_ID=${OPENSKY_CLIENT_ID} - OPENSKY_CLIENT_SECRET=${OPENSKY_CLIENT_SECRET} - LTA_ACCOUNT_KEY=${LTA_ACCOUNT_KEY} + - ADMIN_KEY=${ADMIN_KEY:-} # Override allowed CORS origins (comma-separated). Auto-detects LAN IPs if empty. - CORS_ORIGINS=${CORS_ORIGINS:-} volumes: - backend_data:/app/data restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/api/live-data/fast"] + test: ["CMD", "curl", "-f", "http://localhost:8000/api/health"] interval: 30s timeout: 10s retries: 3 - start_period: 90s + start_period: 30s deploy: resources: limits: diff --git a/docs/mesh/mesh-canonical-fixtures.json b/docs/mesh/mesh-canonical-fixtures.json new file mode 100644 index 00000000..ff213381 --- /dev/null +++ b/docs/mesh/mesh-canonical-fixtures.json @@ -0,0 +1,152 @@ +[ + { + "name": "message-basic", + "event_type": "message", + "node_id": "!sb_abc12345", + "sequence": 1, + "payload": { + "message": "Hello", + "destination": "broadcast", + "channel": "LongFast", + "priority": "normal", + "ephemeral": false + }, + "expected": "infonet/2|sb-testnet-0|message|!sb_abc12345|1|{\"channel\":\"LongFast\",\"destination\":\"broadcast\",\"ephemeral\":false,\"message\":\"Hello\",\"priority\":\"normal\"}" + }, + { + "name": "gate-message", + "event_type": "gate_message", + "node_id": "!sb_abc12345", + "sequence": 2, + "payload": { + "gate": "Alpha", + "epoch": 2, + "ciphertext": "opaque-gate-payload", + "nonce": "gate-nonce-2", + "sender_ref": "persona-alpha-1", + "format": "g1" + }, + "expected": "infonet/2|sb-testnet-0|gate_message|!sb_abc12345|2|{\"ciphertext\":\"opaque-gate-payload\",\"epoch\":2,\"format\":\"g1\",\"gate\":\"alpha\",\"nonce\":\"gate-nonce-2\",\"sender_ref\":\"persona-alpha-1\"}" + }, + { + "name": "vote-up", + "event_type": "vote", + "node_id": "!sb_voter1", + "sequence": 3, + "payload": { + "target_id": "!sb_target1", + "vote": "1", + "gate": "Alpha" + }, + "expected": "infonet/2|sb-testnet-0|vote|!sb_voter1|3|{\"gate\":\"Alpha\",\"target_id\":\"!sb_target1\",\"vote\":1}" + }, + { + "name": "gate-create", + "event_type": "gate_create", + "node_id": "!sb_creator1", + "sequence": 4, + "payload": { + "gate_id": "AlphaGate", + "display_name": "Alpha Gate", + "rules": { + "min_overall_rep": 3 + } + }, + "expected": "infonet/2|sb-testnet-0|gate_create|!sb_creator1|4|{\"display_name\":\"Alpha Gate\",\"gate_id\":\"alphagate\",\"rules\":{\"min_overall_rep\":3}}" + }, + { + "name": "prediction", + "event_type": "prediction", + "node_id": "!sb_oracle1", + "sequence": 5, + "payload": { + "market_title": "BTC>100k", + "side": "yes", + "stake_amount": "0.5" + }, + "expected": "infonet/2|sb-testnet-0|prediction|!sb_oracle1|5|{\"market_title\":\"BTC>100k\",\"side\":\"yes\",\"stake_amount\":0.5}" + }, + { + "name": "stake", + "event_type": "stake", + "node_id": "!sb_staker1", + "sequence": 6, + "payload": { + "message_id": "m1", + "poster_id": "", + "side": "truth", + "amount": "2.0", + "duration_days": "7" + }, + "expected": "infonet/2|sb-testnet-0|stake|!sb_staker1|6|{\"amount\":2,\"duration_days\":7,\"message_id\":\"m1\",\"poster_id\":\"\",\"side\":\"truth\"}" + }, + { + "name": "dm-key", + "event_type": "dm_key", + "node_id": "!sb_dm1", + "sequence": 0, + "payload": { + "dh_pub_key": "abcd", + "dh_algo": "X25519", + "timestamp": "1700000000" + }, + "expected": "infonet/2|sb-testnet-0|dm_key|!sb_dm1|0|{\"dh_algo\":\"X25519\",\"dh_pub_key\":\"abcd\",\"timestamp\":1700000000}" + }, + { + "name": "dm-message", + "event_type": "dm_message", + "node_id": "!sb_dm1", + "sequence": 7, + "payload": { + "recipient_id": "!sb_dm2", + "delivery_class": "shared", + "recipient_token": "sharedtoken123", + "ciphertext": "deadbeef", + "format": "dm1", + "msg_id": "dm_1", + "timestamp": 1700000001 + }, + "expected": "infonet/2|sb-testnet-0|dm_message|!sb_dm1|7|{\"ciphertext\":\"deadbeef\",\"delivery_class\":\"shared\",\"format\":\"dm1\",\"msg_id\":\"dm_1\",\"recipient_id\":\"!sb_dm2\",\"recipient_token\":\"sharedtoken123\",\"timestamp\":1700000001}" + }, + { + "name": "dm-block", + "event_type": "dm_block", + "node_id": "!sb_dm1", + "sequence": 8, + "payload": { + "blocked_id": "!sb_spam", + "action": "BLOCK" + }, + "expected": "infonet/2|sb-testnet-0|dm_block|!sb_dm1|8|{\"action\":\"block\",\"blocked_id\":\"!sb_spam\"}" + }, + { + "name": "key-rotate", + "event_type": "key_rotate", + "node_id": "!sb_old", + "sequence": 1, + "payload": { + "old_node_id": "!sb_old", + "old_public_key": "oldkey", + "old_public_key_algo": "Ed25519", + "new_public_key": "newkey", + "new_public_key_algo": "Ed25519", + "timestamp": 1700000002, + "old_signature": "abcdef" + }, + "expected": "infonet/2|sb-testnet-0|key_rotate|!sb_old|1|{\"new_public_key\":\"newkey\",\"new_public_key_algo\":\"Ed25519\",\"old_node_id\":\"!sb_old\",\"old_public_key\":\"oldkey\",\"old_public_key_algo\":\"Ed25519\",\"old_signature\":\"abcdef\",\"timestamp\":1700000002}" + }, + { + "name": "key-revoke", + "event_type": "key_revoke", + "node_id": "!sb_revoker", + "sequence": 9, + "payload": { + "revoked_public_key": "revkey", + "revoked_public_key_algo": "Ed25519", + "revoked_at": 1700000100, + "grace_until": 1700003700, + "reason": "compromised" + }, + "expected": "infonet/2|sb-testnet-0|key_revoke|!sb_revoker|9|{\"grace_until\":1700003700,\"reason\":\"compromised\",\"revoked_at\":1700000100,\"revoked_public_key\":\"revkey\",\"revoked_public_key_algo\":\"Ed25519\"}" + } +] diff --git a/docs/mesh/mesh-merkle-fixtures.json b/docs/mesh/mesh-merkle-fixtures.json new file mode 100644 index 00000000..93e7f296 --- /dev/null +++ b/docs/mesh/mesh-merkle-fixtures.json @@ -0,0 +1,40 @@ +{ + "leaves": [ + "a", + "b", + "c", + "d", + "e" + ], + "root": "3615e586768e706351e326736e446554c49123d0e24c169d3ecf9b791a82636b", + "proofs": { + "2": [ + { + "hash": "18ac3e7343f016890c510e93f935261169d9e3f565436429830faf0934f4f8e4", + "side": "right" + }, + { + "hash": "62af5c3cb8da3e4f25061e829ebeea5c7513c54949115b1acc225930a90154da", + "side": "left" + }, + { + "hash": "463bb9d8f7fe77a1f4ea68498899ecec274cdf238783a42cb448ce1e2d8cbb6a", + "side": "right" + } + ], + "4": [ + { + "hash": "3f79bb7b435b05321651daefd374cdc681dc06faa65e374e38337b88ca046dea", + "side": "right" + }, + { + "hash": "1a98a2105977d77929b907710dfad6b5f9cdae2abbcaa989a9387ed62c706cd1", + "side": "right" + }, + { + "hash": "58c89d709329eb37285837b042ab6ff72c7c8f74de0446b091b6a0131c102cfd", + "side": "left" + } + ] + } +} diff --git a/frontend/.prettierignore b/frontend/.prettierignore new file mode 100644 index 00000000..f5516c63 --- /dev/null +++ b/frontend/.prettierignore @@ -0,0 +1,3 @@ +.next +node_modules +public diff --git a/frontend/.prettierrc b/frontend/.prettierrc new file mode 100644 index 00000000..e5ce6355 --- /dev/null +++ b/frontend/.prettierrc @@ -0,0 +1,6 @@ +{ + "singleQuote": true, + "trailingComma": "all", + "printWidth": 100, + "semi": true +} diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 2c5fdd0c..9a95068e 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -25,7 +25,7 @@ ENV NEXT_TELEMETRY_DISABLED=1 RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs -COPY --from=builder /app/public ./public +COPY --from=builder --chown=nextjs:nodejs /app/public ./public RUN mkdir .next RUN chown nextjs:nodejs .next diff --git a/frontend/README.md b/frontend/README.md index 641a7b1e..1260fe05 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -19,14 +19,14 @@ The frontend needs to reach the backend (default port `8000`). Resolution order: ### Common scenarios -| Scenario | Action needed | -|----------|---------------| -| Local dev (`localhost:3000` + `localhost:8000`) | None — auto-detected | -| LAN access (`192.168.x.x:3000`) | None — auto-detected from browser hostname | -| Public deploy (same host, port 8000) | None — auto-detected | -| Backend on different port (e.g. `9096`) | Set `NEXT_PUBLIC_API_URL=http://host:9096` before build | -| Backend on different host | Set `NEXT_PUBLIC_API_URL=http://backend-host:8000` before build | -| Behind reverse proxy (e.g. `/api` path) | Set `NEXT_PUBLIC_API_URL=https://yourdomain.com` before build | +| Scenario | Action needed | +| ----------------------------------------------- | --------------------------------------------------------------- | +| Local dev (`localhost:3000` + `localhost:8000`) | None — auto-detected | +| LAN access (`192.168.x.x:3000`) | None — auto-detected from browser hostname | +| Public deploy (same host, port 8000) | None — auto-detected | +| Backend on different port (e.g. `9096`) | Set `NEXT_PUBLIC_API_URL=http://host:9096` before build | +| Backend on different host | Set `NEXT_PUBLIC_API_URL=http://backend-host:8000` before build | +| Behind reverse proxy (e.g. `/api` path) | Set `NEXT_PUBLIC_API_URL=https://yourdomain.com` before build | ### Setting the variable diff --git a/frontend/eslint.config.mjs b/frontend/eslint.config.mjs index 05e726d1..912ba57e 100644 --- a/frontend/eslint.config.mjs +++ b/frontend/eslint.config.mjs @@ -1,17 +1,49 @@ -import { defineConfig, globalIgnores } from "eslint/config"; -import nextVitals from "eslint-config-next/core-web-vitals"; -import nextTs from "eslint-config-next/typescript"; +import { defineConfig, globalIgnores } from 'eslint/config'; +import nextVitals from 'eslint-config-next/core-web-vitals'; +import nextTs from 'eslint-config-next/typescript'; const eslintConfig = defineConfig([ ...nextVitals, ...nextTs, + { + files: ['**/*.{ts,tsx}'], + rules: { + '@typescript-eslint/no-explicit-any': 'warn', + '@typescript-eslint/ban-ts-comment': 'warn', + 'react-hooks/set-state-in-effect': 'off', + 'react-hooks/purity': 'off', + 'react-hooks/refs': 'off', + 'prefer-const': 'warn', + '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], + }, + }, + { + files: ['**/*.test.{ts,tsx}'], + rules: { + '@typescript-eslint/no-explicit-any': 'off', + }, + }, + { + files: ['scripts/**/*.js'], + rules: { + '@typescript-eslint/no-require-imports': 'off', + }, + }, + { + files: ['**/*.cjs', 'vitest.config.js'], + rules: { + '@typescript-eslint/no-require-imports': 'off', + }, + }, // Override default ignores of eslint-config-next. globalIgnores([ // Default ignores of eslint-config-next: - ".next/**", - "out/**", - "build/**", - "next-env.d.ts", + '.next/**', + 'out/**', + 'build/**', + 'next-env.d.ts', + 'coverage/**', + 'eslint-report.json', ]), ]); diff --git a/frontend/next.config.ts b/frontend/next.config.ts index 6e52e10d..691d4270 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -1,13 +1,73 @@ -import type { NextConfig } from "next"; +import type { NextConfig } from 'next'; // /api/* requests are proxied to the backend by the catch-all route handler at // src/app/api/[...path]/route.ts, which reads BACKEND_URL at request time. // Do NOT add rewrites for /api/* here — next.config is evaluated at build time, // so any URL baked in here ignores the runtime BACKEND_URL env var. +const skipTypecheck = process.env.NEXT_SKIP_TYPECHECK === '1'; +const isDev = process.env.NODE_ENV !== 'production'; +const securityHeaders = [ + { + key: 'Content-Security-Policy', + value: [ + "default-src 'self'", + isDev + ? "script-src 'self' 'unsafe-inline' 'unsafe-eval' blob:" + : "script-src 'self' 'unsafe-inline' blob:", + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data: blob: https:", + isDev + ? "connect-src 'self' ws: wss: http://127.0.0.1:8000 http://127.0.0.1:8787 https:" + : "connect-src 'self' ws: wss: https:", + "font-src 'self' data:", + "object-src 'none'", + "worker-src 'self' blob:", + "child-src 'self' blob:", + "frame-ancestors 'none'", + "base-uri 'self'", + "form-action 'self'", + ].join('; '), + }, + { + key: 'Referrer-Policy', + value: 'no-referrer', + }, + { + key: 'X-Content-Type-Options', + value: 'nosniff', + }, + { + key: 'X-Frame-Options', + value: 'DENY', + }, +]; + const nextConfig: NextConfig = { - transpilePackages: ['react-map-gl', 'mapbox-gl', 'maplibre-gl'], - output: "standalone", + transpilePackages: ['react-map-gl', 'maplibre-gl'], + output: 'standalone', + devIndicators: false, + images: { + remotePatterns: [ + { protocol: 'https', hostname: 'upload.wikimedia.org' }, + { protocol: 'https', hostname: 'via.placeholder.com' }, + { protocol: 'https', hostname: 'services.sentinel-hub.com' }, + { protocol: 'https', hostname: 'data.sentinel-hub.com' }, + { protocol: 'https', hostname: 'sentinel-hub.com' }, + { protocol: 'https', hostname: 'dataspace.copernicus.eu' }, + ], + }, + typescript: { + ignoreBuildErrors: skipTypecheck, + }, + async headers() { + return [ + { + source: '/:path*', + headers: securityHeaders, + }, + ]; + }, }; export default nextConfig; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c3699932..804ea5f6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend", - "version": "0.9.0", + "version": "0.9.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "frontend", - "version": "0.9.0", + "version": "0.9.5", "dependencies": { "@mapbox/point-geometry": "^1.1.0", "framer-motion": "^12.34.3", @@ -17,7 +17,8 @@ "react": "19.2.3", "react-dom": "19.2.3", "react-map-gl": "^8.1.0", - "satellite.js": "^6.0.2" + "satellite.js": "^6.0.2", + "zod": "^4.3.6" }, "devDependencies": { "@tailwindcss/postcss": "^4", @@ -31,6 +32,7 @@ "eslint": "^9", "eslint-config-next": "16.1.6", "jsdom": "^28.1.0", + "prettier": "^3.3.3", "tailwindcss": "^4", "typescript": "^5", "vitest": "^4.1.0" @@ -7350,6 +7352,22 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", @@ -9656,7 +9674,6 @@ "version": "4.3.6", "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/frontend/package.json b/frontend/package.json index ce922771..ede18bcf 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,17 +1,21 @@ { "name": "frontend", - "version": "0.9.5", + "version": "0.9.6", "private": true, "scripts": { - "dev": "concurrently \"npm run dev:frontend\" \"npm run dev:backend\"", + "dev": "node scripts/dev-all.cjs", "dev:frontend": "next dev", "dev:backend": "node ../start-backend.js", "build": "next build", "start": "next start", "lint": "eslint", - "test": "vitest run", + "format": "prettier --write .", + "format:check": "prettier --check .", + "bundle:report": "node scripts/report-bundle-size.js", + "test": "npm run test:ci", "test:watch": "vitest", - "test:coverage": "vitest run --coverage" + "test:coverage": "set NODE_OPTIONS=--require ./scripts/vite-no-net-use.cjs && vitest run --coverage --pool=threads", + "test:ci": "set NODE_OPTIONS=--require ./scripts/vite-no-net-use.cjs && vitest run --pool=threads" }, "dependencies": { "@mapbox/point-geometry": "^1.1.0", @@ -23,7 +27,8 @@ "react": "19.2.3", "react-dom": "19.2.3", "react-map-gl": "^8.1.0", - "satellite.js": "^6.0.2" + "satellite.js": "^6.0.2", + "zod": "^4.3.6" }, "devDependencies": { "@tailwindcss/postcss": "^4", @@ -37,6 +42,7 @@ "eslint": "^9", "eslint-config-next": "16.1.6", "jsdom": "^28.1.0", + "prettier": "^3.3.3", "tailwindcss": "^4", "typescript": "^5", "vitest": "^4.1.0" diff --git a/frontend/postcss.config.mjs b/frontend/postcss.config.mjs index 61e36849..297374d8 100644 --- a/frontend/postcss.config.mjs +++ b/frontend/postcss.config.mjs @@ -1,6 +1,6 @@ const config = { plugins: { - "@tailwindcss/postcss": {}, + '@tailwindcss/postcss': {}, }, }; diff --git a/frontend/scripts/dev-all.cjs b/frontend/scripts/dev-all.cjs new file mode 100644 index 00000000..dc3ff750 --- /dev/null +++ b/frontend/scripts/dev-all.cjs @@ -0,0 +1,61 @@ +const { spawn } = require("child_process"); +const path = require("path"); + +const frontendDir = path.resolve(__dirname, ".."); +const backendLauncher = path.resolve(frontendDir, "..", "start-backend.js"); +const nextBin = require.resolve("next/dist/bin/next"); + +/** @type {import("child_process").ChildProcess[]} */ +const children = []; + +function start(label, file, args, cwd) { + const child = spawn(file, args, { + cwd, + env: process.env, + stdio: "inherit", + windowsHide: false, + }); + + child.on("error", (error) => { + console.error(`[${label}] failed to start:`, error); + shutdown(1); + }); + + child.on("exit", (code, signal) => { + if (signal || (code ?? 0) !== 0) { + console.error(`[${label}] exited with ${signal ?? code}`); + shutdown(typeof code === "number" ? code : 1); + return; + } + shutdown(0); + }); + + children.push(child); + return child; +} + +let shuttingDown = false; + +function shutdown(exitCode) { + if (shuttingDown) { + return; + } + shuttingDown = true; + for (const child of children) { + if (!child.killed) { + child.kill(); + } + } + process.exit(exitCode); +} + +process.on("SIGINT", () => shutdown(0)); +process.on("SIGTERM", () => shutdown(0)); + +start( + "frontend", + process.execPath, + [nextBin, "dev", "--hostname", "127.0.0.1", "--port", "3000"], + frontendDir, +); +start("backend", process.execPath, [backendLauncher], frontendDir); diff --git a/frontend/scripts/report-bundle-size.js b/frontend/scripts/report-bundle-size.js new file mode 100644 index 00000000..4da0fa1f --- /dev/null +++ b/frontend/scripts/report-bundle-size.js @@ -0,0 +1,33 @@ +const fs = require('fs'); +const path = require('path'); + +const root = path.resolve(__dirname, '..'); +const nextDir = path.join(root, '.next'); + +function dirSize(p) { + let total = 0; + if (!fs.existsSync(p)) return 0; + const stats = fs.statSync(p); + if (stats.isFile()) return stats.size; + for (const entry of fs.readdirSync(p)) { + total += dirSize(path.join(p, entry)); + } + return total; +} + +const total = dirSize(nextDir); +const staticSize = dirSize(path.join(nextDir, 'static')); +const serverSize = dirSize(path.join(nextDir, 'server')); + +const toKb = (b) => Math.round(b / 1024); + +console.log('Bundle size report'); +console.log(`.next total: ${toKb(total)} KB`); +console.log(`.next/static: ${toKb(staticSize)} KB`); +console.log(`.next/server: ${toKb(serverSize)} KB`); + +const limitKb = process.env.BUNDLE_SIZE_LIMIT_KB ? Number(process.env.BUNDLE_SIZE_LIMIT_KB) : null; +if (limitKb && toKb(total) > limitKb) { + console.error(`Bundle size exceeds limit: ${toKb(total)} KB > ${limitKb} KB`); + process.exit(1); +} diff --git a/frontend/scripts/vite-no-net-use.cjs b/frontend/scripts/vite-no-net-use.cjs new file mode 100644 index 00000000..cf7d1c6b --- /dev/null +++ b/frontend/scripts/vite-no-net-use.cjs @@ -0,0 +1,21 @@ +const childProcess = require('child_process'); + +const originalExec = childProcess.exec; + +childProcess.exec = function exec(command, options, callback) { + const cmd = typeof command === 'string' ? command.trim().toLowerCase() : ''; + if (cmd === 'net use') { + const cb = typeof options === 'function' ? options : callback; + if (typeof cb === 'function') { + process.nextTick(() => cb(null, '', '')); + } + return { + pid: 0, + stdout: null, + stderr: null, + on() {}, + kill() {}, + }; + } + return originalExec.apply(this, arguments); +}; diff --git a/frontend/src/__tests__/desktop/adminSessionBoundary.test.ts b/frontend/src/__tests__/desktop/adminSessionBoundary.test.ts new file mode 100644 index 00000000..7e8fc2ea --- /dev/null +++ b/frontend/src/__tests__/desktop/adminSessionBoundary.test.ts @@ -0,0 +1,233 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +import { GET as proxyGet } from '@/app/api/[...path]/route'; +import { + DELETE as deleteAdminSession, + GET as getAdminSession, + POST as postAdminSession, +} from '@/app/api/admin/session/route'; + +function extractSessionCookie(setCookie: string): string { + return setCookie.split(';')[0] || ''; +} + +describe('admin/session boundary hardening', () => { + const originalAdminKey = process.env.ADMIN_KEY; + const originalBackendUrl = process.env.BACKEND_URL; + + beforeEach(() => { + process.env.ADMIN_KEY = 'top-secret'; + process.env.BACKEND_URL = 'http://127.0.0.1:8000'; + vi.restoreAllMocks(); + }); + + afterEach(() => { + process.env.ADMIN_KEY = originalAdminKey; + process.env.BACKEND_URL = originalBackendUrl; + vi.restoreAllMocks(); + }); + + it('rejects invalid admin keys before minting a session', async () => { + const req = new NextRequest('http://localhost/api/admin/session', { + method: 'POST', + body: JSON.stringify({ adminKey: 'wrong-key' }), + headers: { 'Content-Type': 'application/json' }, + }); + + const res = await postAdminSession(req); + const body = await res.json(); + + expect(res.status).toBe(403); + expect(body.ok).toBe(false); + expect(body.detail).toBe('Invalid admin key'); + expect(res.headers.get('set-cookie')).toBeNull(); + }); + + it('accepts a verified admin key and reports the minted session as present', async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const req = new NextRequest('http://localhost/api/admin/session', { + method: 'POST', + body: JSON.stringify({ adminKey: 'top-secret' }), + headers: { 'Content-Type': 'application/json' }, + }); + + const res = await postAdminSession(req); + const cookie = extractSessionCookie(res.headers.get('set-cookie') || ''); + + expect(res.status).toBe(200); + expect(cookie).toContain('sb_admin_session='); + expect(res.headers.get('cache-control')).toContain('no-store'); + expect(fetchMock).toHaveBeenCalledTimes(1); + + const getReq = new NextRequest('http://localhost/api/admin/session', { + method: 'GET', + headers: { cookie }, + }); + const getRes = await getAdminSession(getReq); + const getBody = await getRes.json(); + + expect(getBody.ok).toBe(true); + expect(getBody.hasSession).toBe(true); + expect(getRes.headers.get('cache-control')).toContain('no-store'); + + const deleteReq = new NextRequest('http://localhost/api/admin/session', { + method: 'DELETE', + headers: { cookie }, + }); + const deleteRes = await deleteAdminSession(deleteReq); + expect(deleteRes.status).toBe(200); + expect(deleteRes.headers.get('cache-control')).toContain('no-store'); + }); + + it('invalidates the previous admin session token when a new one is minted', async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const firstReq = new NextRequest('http://localhost/api/admin/session', { + method: 'POST', + body: JSON.stringify({ adminKey: 'top-secret' }), + headers: { 'Content-Type': 'application/json' }, + }); + const firstRes = await postAdminSession(firstReq); + const firstCookie = extractSessionCookie(firstRes.headers.get('set-cookie') || ''); + + const secondReq = new NextRequest('http://localhost/api/admin/session', { + method: 'POST', + body: JSON.stringify({ adminKey: 'top-secret' }), + headers: { + 'Content-Type': 'application/json', + cookie: firstCookie, + }, + }); + const secondRes = await postAdminSession(secondReq); + const secondCookie = extractSessionCookie(secondRes.headers.get('set-cookie') || ''); + + expect(secondCookie).toContain('sb_admin_session='); + expect(secondCookie).not.toBe(firstCookie); + + const oldSessionCheck = await getAdminSession( + new NextRequest('http://localhost/api/admin/session', { + method: 'GET', + headers: { cookie: firstCookie }, + }), + ); + const oldBody = await oldSessionCheck.json(); + expect(oldBody.hasSession).toBe(false); + + const newSessionCheck = await getAdminSession( + new NextRequest('http://localhost/api/admin/session', { + method: 'GET', + headers: { cookie: secondCookie }, + }), + ); + const newBody = await newSessionCheck.json(); + expect(newBody.hasSession).toBe(true); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('rejects session minting when frontend admin key is set but backend has no configured admin key', async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ detail: 'Forbidden — admin key not configured' }), { + status: 403, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const req = new NextRequest('http://localhost/api/admin/session', { + method: 'POST', + body: JSON.stringify({ adminKey: 'top-secret' }), + headers: { 'Content-Type': 'application/json' }, + }); + + const res = await postAdminSession(req); + const body = await res.json(); + + expect(res.status).toBe(403); + expect(body.ok).toBe(false); + expect(body.detail).toBe('Forbidden — admin key not configured'); + expect(res.headers.get('set-cookie')).toBeNull(); + }); + + it('does not forward raw x-admin-key headers through the sensitive proxy path', async () => { + process.env.ADMIN_KEY = ''; + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const req = new NextRequest('http://localhost/api/settings/api-keys', { + method: 'GET', + headers: { 'x-admin-key': 'browser-supplied-key' }, + }); + + const res = await proxyGet(req, { params: Promise.resolve({ path: ['settings', 'api-keys'] }) }); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body.ok).toBe(true); + expect(res.headers.get('cache-control')).toContain('no-store'); + + const forwarded = fetchMock.mock.calls[0]?.[1]; + const forwardedHeaders = new Headers((forwarded as RequestInit | undefined)?.headers); + expect(forwardedHeaders.get('X-Admin-Key')).toBeNull(); + }); + + it('forwards the minted admin session to sensitive proxy paths and preserves upstream errors', async () => { + const verifyMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', verifyMock); + + const sessionReq = new NextRequest('http://localhost/api/admin/session', { + method: 'POST', + body: JSON.stringify({ adminKey: 'top-secret' }), + headers: { 'Content-Type': 'application/json' }, + }); + const sessionRes = await postAdminSession(sessionReq); + const cookie = extractSessionCookie(sessionRes.headers.get('set-cookie') || ''); + + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ detail: 'Forbidden upstream' }), { + status: 403, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const req = new NextRequest('http://localhost/api/wormhole/identity', { + method: 'GET', + headers: { cookie }, + }); + + const res = await proxyGet(req, { params: Promise.resolve({ path: ['wormhole', 'identity'] }) }); + const body = await res.json(); + + expect(res.status).toBe(403); + expect(body.detail).toBe('Forbidden upstream'); + expect(res.headers.get('cache-control')).toContain('no-store'); + + const forwarded = fetchMock.mock.calls[0]?.[1]; + const forwardedHeaders = new Headers((forwarded as RequestInit | undefined)?.headers); + expect(forwardedHeaders.get('X-Admin-Key')).toBe('top-secret'); + }); +}); diff --git a/frontend/src/__tests__/desktop/controlPlaneNativeBoundary.test.ts b/frontend/src/__tests__/desktop/controlPlaneNativeBoundary.test.ts new file mode 100644 index 00000000..e7e44129 --- /dev/null +++ b/frontend/src/__tests__/desktop/controlPlaneNativeBoundary.test.ts @@ -0,0 +1,57 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const primeAdminSession = vi.fn(); +const localControlFetch = vi.fn(); +const hasLocalControlBridge = vi.fn(); +const canInvokeLocalControl = vi.fn(); + +vi.mock('@/lib/adminSession', () => ({ + primeAdminSession, +})); + +vi.mock('@/lib/localControlTransport', () => ({ + localControlFetch, + hasLocalControlBridge, + canInvokeLocalControl, +})); + +describe('controlPlane native boundary', () => { + beforeEach(() => { + vi.resetModules(); + primeAdminSession.mockReset(); + localControlFetch.mockReset(); + hasLocalControlBridge.mockReset(); + canInvokeLocalControl.mockReset(); + localControlFetch.mockResolvedValue( + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + }); + + it('skips browser admin-session priming when a native bridge can invoke the request', async () => { + hasLocalControlBridge.mockReturnValue(true); + canInvokeLocalControl.mockReturnValue(true); + + const mod = await import('@/lib/controlPlane'); + await mod.controlPlaneFetch('/api/wormhole/gate/message/compose', { + method: 'POST', + body: JSON.stringify({ gate_id: 'infonet', plaintext: 'hello' }), + }); + + expect(primeAdminSession).not.toHaveBeenCalled(); + expect(localControlFetch).toHaveBeenCalledTimes(1); + }); + + it('still primes browser admin-session when no native invoke path exists', async () => { + hasLocalControlBridge.mockReturnValue(false); + canInvokeLocalControl.mockReturnValue(false); + + const mod = await import('@/lib/controlPlane'); + await mod.controlPlaneFetch('/api/wormhole/identity'); + + expect(primeAdminSession).toHaveBeenCalledTimes(1); + expect(localControlFetch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/src/__tests__/desktop/desktopBridgeAuditReport.test.ts b/frontend/src/__tests__/desktop/desktopBridgeAuditReport.test.ts new file mode 100644 index 00000000..7779b2ac --- /dev/null +++ b/frontend/src/__tests__/desktop/desktopBridgeAuditReport.test.ts @@ -0,0 +1,52 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + getDesktopNativeControlAuditReport, + installDesktopControlBridge, +} from '@/lib/desktopBridge'; + +describe('desktopBridge native audit access', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns the runtime audit report when available', () => { + Object.defineProperty(globalThis, 'window', { + value: {}, + configurable: true, + writable: true, + }); + + installDesktopControlBridge({ + invokeLocalControl: vi.fn(), + getNativeControlAuditReport: vi.fn(() => ({ + totalEvents: 2, + totalRecorded: 2, + recent: [], + byOutcome: { allowed: 2 }, + })), + }); + + expect(getDesktopNativeControlAuditReport(5)).toEqual( + expect.objectContaining({ + totalEvents: 2, + totalRecorded: 2, + byOutcome: { allowed: 2 }, + }), + ); + }); + + it('returns null when no runtime audit report is exposed', () => { + Object.defineProperty(globalThis, 'window', { + value: {}, + configurable: true, + writable: true, + }); + + installDesktopControlBridge({ + invokeLocalControl: vi.fn(), + }); + + expect(getDesktopNativeControlAuditReport(5)).toBeNull(); + }); +}); diff --git a/frontend/src/__tests__/desktop/desktopControlContractHelpers.test.ts b/frontend/src/__tests__/desktop/desktopControlContractHelpers.test.ts new file mode 100644 index 00000000..7325c681 --- /dev/null +++ b/frontend/src/__tests__/desktop/desktopControlContractHelpers.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from 'vitest'; + +import { + describeNativeControlError, + extractGateTargetRef, +} from '../../lib/desktopControlContract'; + +describe('extractGateTargetRef', () => { + it('extracts gate_id from gate key rotation payload', () => { + expect( + extractGateTargetRef('wormhole.gate.key.rotate', { gate_id: 'infonet', reason: 'test' }), + ).toBe('infonet'); + }); + + it('extracts gate_id from gate message compose payload', () => { + expect( + extractGateTargetRef('wormhole.gate.message.compose', { gate_id: 'ops', plaintext: 'hi' }), + ).toBe('ops'); + }); + + it('extracts gate_id from gate proof payload', () => { + expect(extractGateTargetRef('wormhole.gate.proof', { gate_id: 'alpha' })).toBe('alpha'); + }); + + it('extracts gate_id from gate message post payload', () => { + expect( + extractGateTargetRef('wormhole.gate.message.post', { gate_id: 'ops', plaintext: 'hi' }), + ).toBe('ops'); + }); + + it('extracts gate_id from gate persona list payload', () => { + expect( + extractGateTargetRef('wormhole.gate.personas.get', { gate_id: 'alpha' }), + ).toBe('alpha'); + }); + + it('returns undefined for non-gate commands', () => { + expect(extractGateTargetRef('wormhole.status', undefined)).toBeUndefined(); + expect(extractGateTargetRef('settings.news.get', undefined)).toBeUndefined(); + }); + + it('returns undefined when payload has no gate_id', () => { + expect(extractGateTargetRef('wormhole.gate.key.rotate', { reason: 'test' })).toBeUndefined(); + expect(extractGateTargetRef('wormhole.gate.key.rotate', null)).toBeUndefined(); + expect(extractGateTargetRef('wormhole.gate.key.rotate', 'not-an-object')).toBeUndefined(); + }); + + it('returns undefined when gate_id is empty string', () => { + expect(extractGateTargetRef('wormhole.gate.key.get', { gate_id: '' })).toBeUndefined(); + }); +}); + +describe('describeNativeControlError', () => { + it('describes profile mismatch errors', () => { + const err = new Error('native_control_profile_mismatch:settings_only:wormhole_gate_key'); + const msg = describeNativeControlError(err); + expect(msg).toContain('Denied'); + expect(msg).toContain('session profile'); + }); + + it('describes capability denied errors', () => { + const err = new Error('native_control_capability_denied:wormhole_gate_key'); + const msg = describeNativeControlError(err); + expect(msg).toContain('Denied'); + expect(msg).toContain('capability'); + }); + + it('describes capability mismatch errors', () => { + const err = new Error('native_control_capability_mismatch:wormhole_gate_content:wormhole_gate_key'); + const msg = describeNativeControlError(err); + expect(msg).toContain('Denied'); + expect(msg).toContain('capability'); + }); + + it('describes shim enforcement inactivity errors', () => { + const err = new Error('desktop_runtime_shim_enforcement_inactive'); + const msg = describeNativeControlError(err); + expect(msg).toContain('Denied'); + expect(msg).toContain('native runtime'); + }); + + it('returns null for unrelated errors', () => { + expect(describeNativeControlError(new Error('network_error'))).toBeNull(); + expect(describeNativeControlError('some string')).toBeNull(); + expect(describeNativeControlError(null)).toBeNull(); + expect(describeNativeControlError(undefined)).toBeNull(); + }); + + it('handles plain string errors', () => { + expect( + describeNativeControlError('native_control_profile_mismatch:foo'), + ).toContain('Denied'); + }); +}); diff --git a/frontend/src/__tests__/desktop/desktopControlRouting.test.ts b/frontend/src/__tests__/desktop/desktopControlRouting.test.ts new file mode 100644 index 00000000..348f87ad --- /dev/null +++ b/frontend/src/__tests__/desktop/desktopControlRouting.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'vitest'; +import { + commandToHttpRequest, + httpRequestToInvokeRequest, +} from '@/lib/desktopControlRouting'; + +describe('desktopControlRouting', () => { + it('maps invoke commands to HTTP requests', () => { + expect(commandToHttpRequest('wormhole.connect')).toEqual({ + path: '/api/wormhole/connect', + method: 'POST', + }); + expect(commandToHttpRequest('wormhole.gate.key.get', { gate_id: 'infonet' })).toEqual({ + path: '/api/wormhole/gate/infonet/key', + method: 'GET', + }); + expect(commandToHttpRequest('settings.news.reset')).toEqual({ + path: '/api/settings/news-feeds/reset', + method: 'POST', + }); + expect(commandToHttpRequest('wormhole.gate.proof', { gate_id: 'infonet' })).toEqual({ + path: '/api/wormhole/gate/proof', + method: 'POST', + payload: { gate_id: 'infonet' }, + }); + expect( + commandToHttpRequest('wormhole.gate.message.post', { + gate_id: 'ops', + plaintext: 'hello', + }), + ).toEqual({ + path: '/api/wormhole/gate/message/post', + method: 'POST', + payload: { gate_id: 'ops', plaintext: 'hello' }, + }); + }); + + it('maps HTTP settings writes back to invoke requests', () => { + expect( + httpRequestToInvokeRequest( + '/api/settings/privacy-profile', + 'PUT', + JSON.stringify({ profile: 'high' }), + ), + ).toEqual({ + command: 'settings.privacy.set', + payload: { profile: 'high' }, + }); + expect( + httpRequestToInvokeRequest( + '/api/wormhole/gate/key/rotate', + 'POST', + JSON.stringify({ gate_id: 'infonet', reason: 'operator_reset' }), + ), + ).toEqual({ + command: 'wormhole.gate.key.rotate', + payload: { gate_id: 'infonet', reason: 'operator_reset' }, + }); + expect( + httpRequestToInvokeRequest( + '/api/wormhole/gate/proof', + 'POST', + JSON.stringify({ gate_id: 'infonet' }), + ), + ).toEqual({ + command: 'wormhole.gate.proof', + payload: { gate_id: 'infonet' }, + }); + expect( + httpRequestToInvokeRequest( + '/api/wormhole/gate/messages/decrypt', + 'POST', + JSON.stringify({ + messages: [ + { + gate_id: 'infonet', + epoch: 3, + ciphertext: 'ct', + nonce: 'n', + sender_ref: 'ref', + }, + ], + }), + ), + ).toEqual({ + command: 'wormhole.gate.messages.decrypt', + payload: { + messages: [ + { + gate_id: 'infonet', + epoch: 3, + ciphertext: 'ct', + nonce: 'n', + sender_ref: 'ref', + }, + ], + }, + }); + }); + + it('returns null for unsupported paths', () => { + expect(httpRequestToInvokeRequest('/api/mesh/status', 'GET')).toBeNull(); + }); +}); diff --git a/frontend/src/__tests__/desktop/desktopRuntimeShimEnforcement.test.ts b/frontend/src/__tests__/desktop/desktopRuntimeShimEnforcement.test.ts new file mode 100644 index 00000000..2f59d4cb --- /dev/null +++ b/frontend/src/__tests__/desktop/desktopRuntimeShimEnforcement.test.ts @@ -0,0 +1,60 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createHttpBackedDesktopRuntime } from '@/lib/desktopRuntimeShim'; + +describe('desktopRuntimeShim enforcement guard', () => { + const fetchMock = vi.fn(); + const warnMock = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + beforeEach(() => { + fetchMock.mockReset(); + fetchMock.mockResolvedValue( + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('refuses strictly enforced commands in the HTTP-backed shim', async () => { + const runtime = createHttpBackedDesktopRuntime(); + + await expect( + runtime.invokeLocalControl?.( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { + capability: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }, + ), + ).rejects.toThrow('desktop_runtime_shim_enforcement_inactive'); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(warnMock).toHaveBeenCalledWith( + '[desktop-shim] strict native session-profile enforcement is unavailable in the HTTP-backed shim', + expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + sessionProfileHint: 'gate_operator', + }), + ); + expect(runtime.getNativeControlAuditReport?.(5)).toEqual( + expect.objectContaining({ + totalEvents: 1, + totalRecorded: 1, + byOutcome: expect.objectContaining({ shim_refused: 1 }), + lastDenied: expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + targetRef: 'infonet', + outcome: 'shim_refused', + }), + }), + ); + }); +}); diff --git a/frontend/src/__tests__/desktop/localControlTransportCapability.test.ts b/frontend/src/__tests__/desktop/localControlTransportCapability.test.ts new file mode 100644 index 00000000..7f08b0d0 --- /dev/null +++ b/frontend/src/__tests__/desktop/localControlTransportCapability.test.ts @@ -0,0 +1,82 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +describe('localControlTransport capability metadata', () => { + beforeEach(() => { + vi.resetModules(); + }); + + it('attaches capability intent metadata when invoking the native bridge', async () => { + const invoke = vi.fn(async () => ({ ok: true })); + Object.defineProperty(globalThis, 'window', { + value: { + __SHADOWBROKER_LOCAL_CONTROL__: { + invoke, + }, + }, + configurable: true, + writable: true, + }); + + const mod = await import('@/lib/localControlTransport'); + await mod.localControlFetch('/api/wormhole/gate/key/rotate', { + method: 'POST', + capabilityIntent: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + body: JSON.stringify({ gate_id: 'infonet', reason: 'operator_reset' }), + }); + + expect(invoke).toHaveBeenCalledWith({ + command: 'wormhole.gate.key.rotate', + payload: { gate_id: 'infonet', reason: 'operator_reset' }, + meta: { + capability: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }, + }); + }); + + it('falls back to plain fetch when the HTTP-backed shim refuses strict enforcement', async () => { + const invoke = vi.fn(async () => { + throw new Error('desktop_runtime_shim_enforcement_inactive'); + }); + const fetchMock = vi.fn(async () => + new Response(JSON.stringify({ ok: true, gate_id: 'infonet' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + Object.defineProperty(globalThis, 'window', { + value: { + __SHADOWBROKER_LOCAL_CONTROL__: { + invoke, + }, + }, + configurable: true, + writable: true, + }); + + const mod = await import('@/lib/localControlTransport'); + const res = await mod.localControlFetch('/api/wormhole/gate/proof', { + method: 'POST', + capabilityIntent: 'wormhole_gate_content', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ gate_id: 'infonet' }), + }); + const data = await res.json(); + + expect(invoke).toHaveBeenCalledOnce(); + expect(fetchMock).toHaveBeenCalledWith( + '/api/wormhole/gate/proof', + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ gate_id: 'infonet' }), + }), + ); + expect(data).toEqual({ ok: true, gate_id: 'infonet' }); + }); +}); diff --git a/frontend/src/__tests__/desktop/nativeControlRouterCapability.test.ts b/frontend/src/__tests__/desktop/nativeControlRouterCapability.test.ts new file mode 100644 index 00000000..c45add85 --- /dev/null +++ b/frontend/src/__tests__/desktop/nativeControlRouterCapability.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createNativeControlRouter } from '../../../../desktop-shell/src/nativeControlRouter'; + +describe('nativeControlRouter capability scaffolding', () => { + it('rejects mismatched capability intent', async () => { + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + }, + exec, + ); + + await expect( + router.invoke( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { capability: 'wormhole_gate_content' }, + ), + ).rejects.toThrow('native_control_capability_mismatch'); + }); + + it('rejects commands outside the allowed native capability set', async () => { + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + allowedCapabilities: ['wormhole_gate_content'], + }, + exec, + ); + + await expect( + router.invoke( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { capability: 'wormhole_gate_key' }, + ), + ).rejects.toThrow('native_control_capability_denied'); + }); + + it('audits session-profile mismatch without denying by default', async () => { + const auditControlUse = vi.fn(); + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + sessionProfile: 'settings_only', + auditControlUse, + }, + exec, + ); + + const result = await router.invoke( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { capability: 'wormhole_gate_key', sessionProfileHint: 'gate_operator' }, + ); + + expect(result).toEqual({ ok: true }); + expect(auditControlUse).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + expectedCapability: 'wormhole_gate_key', + targetRef: 'infonet', + sessionProfile: 'settings_only', + sessionProfileHint: 'gate_operator', + profileAllows: false, + enforced: false, + outcome: 'profile_warn', + }), + ); + }); + + it('includes targetRef in audit events for gate commands', async () => { + const auditControlUse = vi.fn(); + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + auditControlUse, + }, + exec, + ); + + await router.invoke( + 'wormhole.gate.message.compose', + { gate_id: 'ops-room', plaintext: 'hello' }, + { capability: 'wormhole_gate_content' }, + ); + + expect(auditControlUse).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'wormhole.gate.message.compose', + targetRef: 'ops-room', + outcome: 'allowed', + }), + ); + }); + + it('omits targetRef for non-gate commands', async () => { + const auditControlUse = vi.fn(); + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + auditControlUse, + }, + exec, + ); + + await router.invoke('wormhole.status', undefined); + + const event = auditControlUse.mock.calls[0][0]; + expect(event.command).toBe('wormhole.status'); + expect(event.targetRef).toBeUndefined(); + }); + + it('can enforce session-profile mismatch when explicitly enabled', async () => { + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + sessionProfile: 'settings_only', + enforceSessionProfile: true, + }, + exec, + ); + + await expect( + router.invoke( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { capability: 'wormhole_gate_key', sessionProfileHint: 'gate_operator' }, + ), + ).rejects.toThrow('native_control_profile_mismatch'); + }); + + it('can enforce a hinted session profile for a narrow gate-key command', async () => { + const exec = async (): Promise => ({ ok: true } as T); + const router = createNativeControlRouter( + { + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + sessionProfile: 'settings_only', + }, + exec, + ); + + await expect( + router.invoke( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { + capability: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }, + ), + ).rejects.toThrow('native_control_profile_mismatch'); + }); +}); diff --git a/frontend/src/__tests__/desktop/runtimeBridgeSessionProfile.test.ts b/frontend/src/__tests__/desktop/runtimeBridgeSessionProfile.test.ts new file mode 100644 index 00000000..9a880d30 --- /dev/null +++ b/frontend/src/__tests__/desktop/runtimeBridgeSessionProfile.test.ts @@ -0,0 +1,163 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { createRuntimeBridge } from '../../../../desktop-shell/src/runtimeBridge'; + +describe('runtimeBridge session profile routing', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('uses the invocation session profile hint when the runtime context is unscoped', async () => { + const auditControlUse = vi.fn(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ), + ); + + const runtime = createRuntimeBridge({ + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + auditControlUse, + }); + + await runtime.invokeLocalControl( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { + capability: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }, + ); + + expect(auditControlUse).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + targetRef: 'infonet', + sessionProfile: 'gate_operator', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + profileAllows: true, + outcome: 'allowed', + }), + ); + + const report = runtime.getNativeControlAuditReport?.(5); + expect(report).toEqual( + expect.objectContaining({ + totalEvents: 1, + totalRecorded: 1, + byOutcome: expect.objectContaining({ allowed: 1 }), + }), + ); + expect(report?.recent[0]).toEqual( + expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + targetRef: 'infonet', + sessionProfile: 'gate_operator', + outcome: 'allowed', + }), + ); + }); + + it('preserves an explicitly scoped runtime session profile over the invocation hint', async () => { + const auditControlUse = vi.fn(); + vi.stubGlobal( + 'fetch', + vi.fn(async () => + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ), + ); + + const runtime = createRuntimeBridge({ + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + sessionProfile: 'settings_only', + auditControlUse, + }); + + await runtime.invokeLocalControl( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { + capability: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + }, + ); + + expect(auditControlUse).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + sessionProfile: 'settings_only', + sessionProfileHint: 'gate_operator', + profileAllows: false, + outcome: 'profile_warn', + }), + ); + + const report = runtime.getNativeControlAuditReport?.(5); + expect(report).toEqual( + expect.objectContaining({ + totalEvents: 1, + totalRecorded: 1, + byOutcome: expect.objectContaining({ profile_warn: 1 }), + lastProfileMismatch: expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + sessionProfile: 'settings_only', + outcome: 'profile_warn', + }), + }), + ); + }); + + it('denies a strictly hinted gate-key command when the runtime is pinned to another profile', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => + new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ), + ); + + const runtime = createRuntimeBridge({ + backendBaseUrl: 'http://127.0.0.1:8000', + wormholeBaseUrl: 'http://127.0.0.1:8787', + sessionProfile: 'settings_only', + }); + + await expect( + runtime.invokeLocalControl( + 'wormhole.gate.key.rotate', + { gate_id: 'infonet', reason: 'operator_reset' }, + { + capability: 'wormhole_gate_key', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }, + ), + ).rejects.toThrow('native_control_profile_mismatch'); + + const report = runtime.getNativeControlAuditReport?.(5); + expect(report).toEqual( + expect.objectContaining({ + totalEvents: 1, + totalRecorded: 1, + byOutcome: expect.objectContaining({ profile_denied: 1 }), + lastDenied: expect.objectContaining({ + command: 'wormhole.gate.key.rotate', + outcome: 'profile_denied', + }), + }), + ); + }); +}); diff --git a/frontend/src/__tests__/map/geoJSONBuilders.test.ts b/frontend/src/__tests__/map/geoJSONBuilders.test.ts index cbe1628a..57eb9ba1 100644 --- a/frontend/src/__tests__/map/geoJSONBuilders.test.ts +++ b/frontend/src/__tests__/map/geoJSONBuilders.test.ts @@ -1,10 +1,33 @@ import { describe, it, expect } from 'vitest'; import { - buildEarthquakesGeoJSON, buildJammingGeoJSON, buildCctvGeoJSON, buildKiwisdrGeoJSON, - buildFirmsGeoJSON, buildInternetOutagesGeoJSON, buildDataCentersGeoJSON, - buildGdeltGeoJSON, buildLiveuaGeoJSON, buildFrontlineGeoJSON, buildMilitaryBasesGeoJSON + buildEarthquakesGeoJSON, + buildJammingGeoJSON, + buildCctvGeoJSON, + buildKiwisdrGeoJSON, + buildFirmsGeoJSON, + buildInternetOutagesGeoJSON, + buildDataCentersGeoJSON, + buildGdeltGeoJSON, + buildLiveuaGeoJSON, + buildFrontlineGeoJSON, + buildScannerGeoJSON, + buildMilitaryBasesGeoJSON, + buildTrainsGeoJSON, } from '@/components/map/geoJSONBuilders'; -import type { Earthquake, GPSJammingZone, FireHotspot, InternetOutage, DataCenter, GDELTIncident, LiveUAmapIncident, CCTVCamera, KiwiSDR, MilitaryBase } from '@/types/dashboard'; +import type { + Earthquake, + GPSJammingZone, + FireHotspot, + InternetOutage, + DataCenter, + GDELTIncident, + LiveUAmapIncident, + CCTVCamera, + KiwiSDR, + Scanner, + MilitaryBase, + Train, +} from '@/types/dashboard'; // ─── Military Bases ──────────────────────────────────────────────────────── @@ -34,289 +57,500 @@ describe('buildMilitaryBasesGeoJSON', () => { // ─── Earthquakes ──────────────────────────────────────────────────────────── describe('buildEarthquakesGeoJSON', () => { - it('returns null for empty/undefined input', () => { - expect(buildEarthquakesGeoJSON(undefined)).toBeNull(); - expect(buildEarthquakesGeoJSON([])).toBeNull(); - }); - - it('builds valid FeatureCollection from earthquake data', () => { - const earthquakes: Earthquake[] = [ - { id: 'eq1', mag: 5.2, lat: 35.0, lng: 139.0, place: 'Japan' }, - { id: 'eq2', mag: 3.1, lat: 40.0, lng: -120.0, place: 'California', title: 'Test Title' }, - ]; - const result = buildEarthquakesGeoJSON(earthquakes); - expect(result).not.toBeNull(); - expect(result!.type).toBe('FeatureCollection'); - expect(result!.features).toHaveLength(2); - - const f0 = result!.features[0]; - expect(f0.geometry).toEqual({ type: 'Point', coordinates: [139.0, 35.0] }); - expect(f0.properties?.type).toBe('earthquake'); - expect(f0.properties?.name).toContain('M5.2'); - expect(f0.properties?.name).toContain('Japan'); - }); - - it('filters out entries with null lat/lng', () => { - const earthquakes = [ - { id: 'eq1', mag: 5.0, lat: null as any, lng: 10.0, place: 'X' }, - { id: 'eq2', mag: 3.0, lat: 20.0, lng: 30.0, place: 'Y' }, - ]; - const result = buildEarthquakesGeoJSON(earthquakes); - expect(result!.features).toHaveLength(1); - }); - - it('includes title when present', () => { - const earthquakes: Earthquake[] = [ - { id: 'eq1', mag: 4.0, lat: 10.0, lng: 20.0, place: 'Test', title: 'Big One' }, - ]; - const result = buildEarthquakesGeoJSON(earthquakes); - expect(result!.features[0].properties?.title).toBe('Big One'); - }); + it('returns null for empty/undefined input', () => { + expect(buildEarthquakesGeoJSON(undefined)).toBeNull(); + expect(buildEarthquakesGeoJSON([])).toBeNull(); + }); + + it('builds valid FeatureCollection from earthquake data', () => { + const earthquakes: Earthquake[] = [ + { id: 'eq1', mag: 5.2, lat: 35.0, lng: 139.0, place: 'Japan' }, + { id: 'eq2', mag: 3.1, lat: 40.0, lng: -120.0, place: 'California', title: 'Test Title' }, + ]; + const result = buildEarthquakesGeoJSON(earthquakes); + expect(result).not.toBeNull(); + expect(result!.type).toBe('FeatureCollection'); + expect(result!.features).toHaveLength(2); + + const f0 = result!.features[0]; + expect(f0.geometry).toEqual({ type: 'Point', coordinates: [139.0, 35.0] }); + expect(f0.properties?.type).toBe('earthquake'); + expect(f0.properties?.name).toContain('M5.2'); + expect(f0.properties?.name).toContain('Japan'); + }); + + it('filters out entries with null lat/lng', () => { + const earthquakes = [ + { id: 'eq1', mag: 5.0, lat: null as any, lng: 10.0, place: 'X' }, + { id: 'eq2', mag: 3.0, lat: 20.0, lng: 30.0, place: 'Y' }, + ]; + const result = buildEarthquakesGeoJSON(earthquakes); + expect(result!.features).toHaveLength(1); + }); + + it('includes title when present', () => { + const earthquakes: Earthquake[] = [ + { id: 'eq1', mag: 4.0, lat: 10.0, lng: 20.0, place: 'Test', title: 'Big One' }, + ]; + const result = buildEarthquakesGeoJSON(earthquakes); + expect(result!.features[0].properties?.title).toBe('Big One'); + }); }); // ─── GPS Jamming ──────────────────────────────────────────────────────────── describe('buildJammingGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildJammingGeoJSON(undefined)).toBeNull(); - expect(buildJammingGeoJSON([])).toBeNull(); - }); - - it('builds polygon features with correct opacity mapping', () => { - const zones: GPSJammingZone[] = [ - { lat: 50, lng: 30, severity: 'high', ratio: 0.8, degraded: 100, total: 125 }, - { lat: 45, lng: 35, severity: 'medium', ratio: 0.5, degraded: 50, total: 100 }, - { lat: 40, lng: 25, severity: 'low', ratio: 0.2, degraded: 20, total: 100 }, - ]; - const result = buildJammingGeoJSON(zones); - expect(result!.features).toHaveLength(3); - expect(result!.features[0].properties?.opacity).toBe(0.45); - expect(result!.features[1].properties?.opacity).toBe(0.3); - expect(result!.features[2].properties?.opacity).toBe(0.18); - }); - - it('builds correct 1°×1° polygon geometry', () => { - const zones: GPSJammingZone[] = [ - { lat: 50, lng: 30, severity: 'high', ratio: 0.8, degraded: 100, total: 125 }, - ]; - const result = buildJammingGeoJSON(zones); - const geom = result!.features[0].geometry; - expect(geom.type).toBe('Polygon'); - if (geom.type === 'Polygon') { - const ring = geom.coordinates[0]; - expect(ring).toHaveLength(5); // Closed ring - expect(ring[0]).toEqual([29.5, 49.5]); - expect(ring[2]).toEqual([30.5, 50.5]); - } - }); + it('returns null for empty input', () => { + expect(buildJammingGeoJSON(undefined)).toBeNull(); + expect(buildJammingGeoJSON([])).toBeNull(); + }); + + it('builds polygon features with correct opacity mapping', () => { + const zones: GPSJammingZone[] = [ + { lat: 50, lng: 30, severity: 'high', ratio: 0.8, degraded: 100, total: 125 }, + { lat: 45, lng: 35, severity: 'medium', ratio: 0.5, degraded: 50, total: 100 }, + { lat: 40, lng: 25, severity: 'low', ratio: 0.2, degraded: 20, total: 100 }, + ]; + const result = buildJammingGeoJSON(zones); + expect(result!.features).toHaveLength(3); + expect(result!.features[0].properties?.opacity).toBe(0.45); + expect(result!.features[1].properties?.opacity).toBe(0.3); + expect(result!.features[2].properties?.opacity).toBe(0.18); + }); + + it('builds correct 1°×1° polygon geometry', () => { + const zones: GPSJammingZone[] = [ + { lat: 50, lng: 30, severity: 'high', ratio: 0.8, degraded: 100, total: 125 }, + ]; + const result = buildJammingGeoJSON(zones); + const geom = result!.features[0].geometry; + expect(geom.type).toBe('Polygon'); + if (geom.type === 'Polygon') { + const ring = geom.coordinates[0]; + expect(ring).toHaveLength(5); // Closed ring + expect(ring[0]).toEqual([29.5, 49.5]); + expect(ring[2]).toEqual([30.5, 50.5]); + } + }); }); // ─── CCTV ─────────────────────────────────────────────────────────────────── describe('buildCctvGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildCctvGeoJSON(undefined)).toBeNull(); - }); - - it('builds features from camera data', () => { - const cameras: CCTVCamera[] = [ - { id: 'cam1', lat: 40.7, lon: -74.0, direction_facing: 'North', source_agency: 'DOT' }, - ]; - const result = buildCctvGeoJSON(cameras); - expect(result!.features).toHaveLength(1); - expect(result!.features[0].properties?.type).toBe('cctv'); - expect(result!.features[0].properties?.name).toBe('North'); - }); - - it('respects inView filter', () => { - const cameras: CCTVCamera[] = [ - { id: 'cam1', lat: 40.7, lon: -74.0 }, - { id: 'cam2', lat: 10.0, lon: 20.0 }, - ]; - const inView = (lat: number, _lng: number) => lat > 30; - const result = buildCctvGeoJSON(cameras, inView); - expect(result!.features).toHaveLength(1); - }); + it('returns null for empty input', () => { + expect(buildCctvGeoJSON(undefined)).toBeNull(); + }); + + it('builds features from camera data', () => { + const cameras: CCTVCamera[] = [ + { id: 'cam1', lat: 40.7, lon: -74.0, direction_facing: 'North', source_agency: 'DOT' }, + ]; + const result = buildCctvGeoJSON(cameras); + expect(result!.features).toHaveLength(1); + expect(result!.features[0].properties?.type).toBe('cctv'); + expect(result!.features[0].properties?.name).toBe('North'); + }); + + it('respects inView filter', () => { + const cameras: CCTVCamera[] = [ + { id: 'cam1', lat: 40.7, lon: -74.0 }, + { id: 'cam2', lat: 10.0, lon: 20.0 }, + ]; + const inView = (lat: number, _lng: number) => lat > 30; + const result = buildCctvGeoJSON(cameras, inView); + expect(result!.features).toHaveLength(1); + }); }); // ─── KiwiSDR ──────────────────────────────────────────────────────────────── describe('buildKiwisdrGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildKiwisdrGeoJSON(undefined)).toBeNull(); - }); - - it('builds features with SDR properties', () => { - const receivers: KiwiSDR[] = [ - { lat: 52.0, lon: 13.0, name: 'Berlin SDR', url: 'http://test.com', users: 3, users_max: 8, bands: 'HF', antenna: 'Long Wire', location: 'Berlin' }, - ]; - const result = buildKiwisdrGeoJSON(receivers); - expect(result!.features).toHaveLength(1); - expect(result!.features[0].properties?.name).toBe('Berlin SDR'); - expect(result!.features[0].properties?.users).toBe(3); - }); + it('returns null for empty input', () => { + expect(buildKiwisdrGeoJSON(undefined)).toBeNull(); + }); + + it('builds features with SDR properties', () => { + const receivers: KiwiSDR[] = [ + { + lat: 52.0, + lon: 13.0, + name: 'Berlin SDR', + url: 'http://test.com', + users: 3, + users_max: 8, + bands: 'HF', + antenna: 'Long Wire', + location: 'Berlin', + }, + ]; + const result = buildKiwisdrGeoJSON(receivers); + expect(result!.features).toHaveLength(1); + expect(result!.features[0].properties?.name).toBe('Berlin SDR'); + expect(result!.features[0].properties?.users).toBe(3); + }); }); // ─── FIRMS Fires ──────────────────────────────────────────────────────────── describe('buildFirmsGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildFirmsGeoJSON(undefined)).toBeNull(); - }); - - it('classifies fires by FRP thresholds', () => { - const fires: FireHotspot[] = [ - { lat: 10, lng: 20, frp: 150, brightness: 400, confidence: 'high', daynight: 'D', acq_date: '2024-01-01', acq_time: '1200' }, - { lat: 11, lng: 21, frp: 50, brightness: 350, confidence: 'medium', daynight: 'N', acq_date: '2024-01-01', acq_time: '0100' }, - { lat: 12, lng: 22, frp: 10, brightness: 300, confidence: 'low', daynight: 'D', acq_date: '2024-01-01', acq_time: '1400' }, - { lat: 13, lng: 23, frp: 2, brightness: 250, confidence: 'low', daynight: 'D', acq_date: '2024-01-01', acq_time: '1500' }, - ]; - const result = buildFirmsGeoJSON(fires); - expect(result!.features).toHaveLength(4); - expect(result!.features[0].properties?.iconId).toBe('fire-darkred'); - expect(result!.features[1].properties?.iconId).toBe('fire-red'); - expect(result!.features[2].properties?.iconId).toBe('fire-orange'); - expect(result!.features[3].properties?.iconId).toBe('fire-yellow'); - }); - - it('formats daynight correctly', () => { - const fires: FireHotspot[] = [ - { lat: 10, lng: 20, frp: 5, brightness: 300, confidence: 'low', daynight: 'D', acq_date: '2024-01-01', acq_time: '1200' }, - { lat: 11, lng: 21, frp: 5, brightness: 300, confidence: 'low', daynight: 'N', acq_date: '2024-01-01', acq_time: '0100' }, - ]; - const result = buildFirmsGeoJSON(fires); - expect(result!.features[0].properties?.daynight).toBe('Day'); - expect(result!.features[1].properties?.daynight).toBe('Night'); - }); + it('returns null for empty input', () => { + expect(buildFirmsGeoJSON(undefined)).toBeNull(); + }); + + it('classifies fires by FRP thresholds', () => { + const fires: FireHotspot[] = [ + { + lat: 10, + lng: 20, + frp: 150, + brightness: 400, + confidence: 'high', + daynight: 'D', + acq_date: '2024-01-01', + acq_time: '1200', + }, + { + lat: 11, + lng: 21, + frp: 50, + brightness: 350, + confidence: 'medium', + daynight: 'N', + acq_date: '2024-01-01', + acq_time: '0100', + }, + { + lat: 12, + lng: 22, + frp: 10, + brightness: 300, + confidence: 'low', + daynight: 'D', + acq_date: '2024-01-01', + acq_time: '1400', + }, + { + lat: 13, + lng: 23, + frp: 2, + brightness: 250, + confidence: 'low', + daynight: 'D', + acq_date: '2024-01-01', + acq_time: '1500', + }, + ]; + const result = buildFirmsGeoJSON(fires); + expect(result!.features).toHaveLength(4); + expect(result!.features[0].properties?.iconId).toBe('fire-darkred'); + expect(result!.features[1].properties?.iconId).toBe('fire-red'); + expect(result!.features[2].properties?.iconId).toBe('fire-orange'); + expect(result!.features[3].properties?.iconId).toBe('fire-yellow'); + }); + + it('formats daynight correctly', () => { + const fires: FireHotspot[] = [ + { + lat: 10, + lng: 20, + frp: 5, + brightness: 300, + confidence: 'low', + daynight: 'D', + acq_date: '2024-01-01', + acq_time: '1200', + }, + { + lat: 11, + lng: 21, + frp: 5, + brightness: 300, + confidence: 'low', + daynight: 'N', + acq_date: '2024-01-01', + acq_time: '0100', + }, + ]; + const result = buildFirmsGeoJSON(fires); + expect(result!.features[0].properties?.daynight).toBe('Day'); + expect(result!.features[1].properties?.daynight).toBe('Night'); + }); }); // ─── Internet Outages ─────────────────────────────────────────────────────── describe('buildInternetOutagesGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildInternetOutagesGeoJSON(undefined)).toBeNull(); - }); - - it('builds features with detail string', () => { - const outages: InternetOutage[] = [ - { region_code: 'TX', region_name: 'Texas', country_code: 'US', country_name: 'United States', lat: 31.0, lng: -100.0, severity: 45, level: 'region', datasource: 'bgp' }, - ]; - const result = buildInternetOutagesGeoJSON(outages); - expect(result!.features).toHaveLength(1); - expect(result!.features[0].properties?.detail).toContain('Texas'); - expect(result!.features[0].properties?.detail).toContain('45% drop'); - }); - - it('filters out entries with null coordinates', () => { - const outages: InternetOutage[] = [ - { region_code: 'TX', region_name: 'Texas', country_code: 'US', country_name: 'United States', lat: null as any, lng: null as any, severity: 20, level: 'region', datasource: 'bgp' }, - { region_code: 'CA', region_name: 'California', country_code: 'US', country_name: 'United States', lat: 37.0, lng: -122.0, severity: 30, level: 'region', datasource: 'bgp' }, - ]; - const result = buildInternetOutagesGeoJSON(outages); - expect(result!.features).toHaveLength(1); - }); + it('returns null for empty input', () => { + expect(buildInternetOutagesGeoJSON(undefined)).toBeNull(); + }); + + it('builds features with detail string', () => { + const outages: InternetOutage[] = [ + { + region_code: 'TX', + region_name: 'Texas', + country_code: 'US', + country_name: 'United States', + lat: 31.0, + lng: -100.0, + severity: 45, + level: 'region', + datasource: 'bgp', + }, + ]; + const result = buildInternetOutagesGeoJSON(outages); + expect(result!.features).toHaveLength(1); + expect(result!.features[0].properties?.detail).toContain('Texas'); + expect(result!.features[0].properties?.detail).toContain('45% drop'); + }); + + it('filters out entries with null coordinates', () => { + const outages: InternetOutage[] = [ + { + region_code: 'TX', + region_name: 'Texas', + country_code: 'US', + country_name: 'United States', + lat: null as any, + lng: null as any, + severity: 20, + level: 'region', + datasource: 'bgp', + }, + { + region_code: 'CA', + region_name: 'California', + country_code: 'US', + country_name: 'United States', + lat: 37.0, + lng: -122.0, + severity: 30, + level: 'region', + datasource: 'bgp', + }, + ]; + const result = buildInternetOutagesGeoJSON(outages); + expect(result!.features).toHaveLength(1); + }); }); // ─── Data Centers ─────────────────────────────────────────────────────────── describe('buildDataCentersGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildDataCentersGeoJSON(undefined)).toBeNull(); - }); - - it('builds features with datacenter properties', () => { - const dcs: DataCenter[] = [ - { lat: 40.0, lng: -74.0, name: 'NYC-DC1', company: 'Equinix', street: '123 Main', city: 'New York', country: 'US', zip: '10001' }, - ]; - const result = buildDataCentersGeoJSON(dcs); - expect(result!.features).toHaveLength(1); - expect(result!.features[0].properties?.id).toBe('dc-0'); - expect(result!.features[0].properties?.company).toBe('Equinix'); - }); + it('returns null for empty input', () => { + expect(buildDataCentersGeoJSON(undefined)).toBeNull(); + }); + + it('builds features with datacenter properties', () => { + const dcs: DataCenter[] = [ + { + lat: 40.0, + lng: -74.0, + name: 'NYC-DC1', + company: 'Equinix', + street: '123 Main', + city: 'New York', + country: 'US', + zip: '10001', + }, + ]; + const result = buildDataCentersGeoJSON(dcs); + expect(result!.features).toHaveLength(1); + expect(result!.features[0].properties?.id).toBe('dc-0'); + expect(result!.features[0].properties?.company).toBe('Equinix'); + }); }); // ─── GDELT ────────────────────────────────────────────────────────────────── describe('buildGdeltGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildGdeltGeoJSON(undefined)).toBeNull(); - }); - - it('builds features from GDELT incidents', () => { - const gdelt: GDELTIncident[] = [ - { type: 'Feature', geometry: { type: 'Point', coordinates: [30, 50] }, properties: { name: 'Protest', count: 5, _urls_list: [], _headlines_list: [] } }, - ]; - const result = buildGdeltGeoJSON(gdelt); - expect(result!.features).toHaveLength(1); - expect(result!.features[0].properties?.type).toBe('gdelt'); - expect(result!.features[0].properties?.title).toBe('Protest'); - }); - - it('filters by inView when provided', () => { - const gdelt: GDELTIncident[] = [ - { type: 'Feature', geometry: { type: 'Point', coordinates: [30, 50] }, properties: { name: 'A', count: 1, _urls_list: [], _headlines_list: [] } }, - { type: 'Feature', geometry: { type: 'Point', coordinates: [100, 10] }, properties: { name: 'B', count: 1, _urls_list: [], _headlines_list: [] } }, - ]; - const inView = (lat: number, _lng: number) => lat > 30; - const result = buildGdeltGeoJSON(gdelt, inView); - expect(result!.features).toHaveLength(1); - }); + it('returns null for empty input', () => { + expect(buildGdeltGeoJSON(undefined)).toBeNull(); + }); + + it('builds features from GDELT incidents', () => { + const gdelt: GDELTIncident[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [30, 50] }, + properties: { name: 'Protest', count: 5, _urls_list: [], _headlines_list: [] }, + }, + ]; + const result = buildGdeltGeoJSON(gdelt); + expect(result!.features).toHaveLength(1); + expect(result!.features[0].properties?.type).toBe('gdelt'); + expect(result!.features[0].properties?.title).toBe('Protest'); + }); + + it('filters by inView when provided', () => { + const gdelt: GDELTIncident[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [30, 50] }, + properties: { name: 'A', count: 1, _urls_list: [], _headlines_list: [] }, + }, + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [100, 10] }, + properties: { name: 'B', count: 1, _urls_list: [], _headlines_list: [] }, + }, + ]; + const inView = (lat: number, _lng: number) => lat > 30; + const result = buildGdeltGeoJSON(gdelt, inView); + expect(result!.features).toHaveLength(1); + }); + + it('filters out entries without geometry', () => { + const gdelt: GDELTIncident[] = [ + { + type: 'Feature', + geometry: { type: 'Point', coordinates: [30, 50] }, + properties: { name: 'Good', count: 1, _urls_list: [], _headlines_list: [] }, + }, + { + type: 'Feature', + geometry: null as any, + properties: { name: 'Bad', count: 1, _urls_list: [], _headlines_list: [] }, + }, + ]; + const result = buildGdeltGeoJSON(gdelt); + expect(result!.features).toHaveLength(1); + }); +}); - it('filters out entries without geometry', () => { - const gdelt: GDELTIncident[] = [ - { type: 'Feature', geometry: { type: 'Point', coordinates: [30, 50] }, properties: { name: 'Good', count: 1, _urls_list: [], _headlines_list: [] } }, - { type: 'Feature', geometry: null as any, properties: { name: 'Bad', count: 1, _urls_list: [], _headlines_list: [] } }, - ]; - const result = buildGdeltGeoJSON(gdelt); - expect(result!.features).toHaveLength(1); - }); +describe('buildTrainsGeoJSON', () => { + it('builds all trains when no inView filter is provided', () => { + const trains: Train[] = [ + { + id: 'amtrak-1', + name: 'Empire Builder', + number: '7', + source: 'amtrak', + source_label: 'Amtraker', + operator: 'Amtrak', + country: 'US', + speed_kmh: 88, + heading: 90, + status: 'active', + route: 'SEA-CHI', + lat: 47.6, + lng: -122.3, + }, + { + id: 'fin-1', + name: 'Pendolino', + number: 'S 94', + source: 'digitraffic', + source_label: 'Digitraffic', + operator: 'VR', + country: 'FI', + speed_kmh: 120, + heading: 180, + status: 'active', + route: 'HEL-TKU', + lat: 60.17, + lng: 24.94, + }, + ]; + + const result = buildTrainsGeoJSON(trains); + expect(result).not.toBeNull(); + expect(result!.features).toHaveLength(2); + }); }); // ─── LiveUAMap ────────────────────────────────────────────────────────────── describe('buildLiveuaGeoJSON', () => { - it('returns null for empty input', () => { - expect(buildLiveuaGeoJSON(undefined)).toBeNull(); - }); - - it('classifies violent incidents with red icon', () => { - const incidents: LiveUAmapIncident[] = [ - { id: '1', lat: 48.0, lng: 35.0, title: 'Missile strike in Kharkiv', date: '2024-01-01' }, - { id: '2', lat: 49.0, lng: 36.0, title: 'Humanitarian aid delivery', date: '2024-01-01' }, - ]; - const result = buildLiveuaGeoJSON(incidents); - expect(result!.features).toHaveLength(2); - expect(result!.features[0].properties?.iconId).toBe('icon-liveua-red'); - expect(result!.features[1].properties?.iconId).toBe('icon-liveua-yellow'); - }); - - it('filters by inView when provided', () => { - const incidents: LiveUAmapIncident[] = [ - { id: '1', lat: 48.0, lng: 35.0, title: 'Test', date: '2024-01-01' }, - { id: '2', lat: 10.0, lng: 20.0, title: 'Far away', date: '2024-01-01' }, - ]; - const inView = (lat: number, _lng: number) => lat > 30; - const result = buildLiveuaGeoJSON(incidents, inView); - expect(result!.features).toHaveLength(1); - }); + it('returns null for empty input', () => { + expect(buildLiveuaGeoJSON(undefined)).toBeNull(); + }); + + it('classifies violent incidents with red icon', () => { + const incidents: LiveUAmapIncident[] = [ + { id: '1', lat: 48.0, lng: 35.0, title: 'Missile strike in Kharkiv', date: '2024-01-01' }, + { id: '2', lat: 49.0, lng: 36.0, title: 'Humanitarian aid delivery', date: '2024-01-01' }, + ]; + const result = buildLiveuaGeoJSON(incidents); + expect(result!.features).toHaveLength(2); + expect(result!.features[0].properties?.iconId).toBe('icon-liveua-red'); + expect(result!.features[1].properties?.iconId).toBe('icon-liveua-yellow'); + }); + + it('filters by inView when provided', () => { + const incidents: LiveUAmapIncident[] = [ + { id: '1', lat: 48.0, lng: 35.0, title: 'Test', date: '2024-01-01' }, + { id: '2', lat: 10.0, lng: 20.0, title: 'Far away', date: '2024-01-01' }, + ]; + const inView = (lat: number, _lng: number) => lat > 30; + const result = buildLiveuaGeoJSON(incidents, inView); + expect(result!.features).toHaveLength(1); + }); }); // ─── Frontline ────────────────────────────────────────────────────────────── describe('buildFrontlineGeoJSON', () => { - it('returns null for null/undefined input', () => { - expect(buildFrontlineGeoJSON(null)).toBeNull(); - expect(buildFrontlineGeoJSON(undefined)).toBeNull(); - }); - - it('returns the input unchanged when valid', () => { - const fc = { type: 'FeatureCollection' as const, features: [{ type: 'Feature' as const, properties: { name: 'zone', zone_id: 1 }, geometry: { type: 'Polygon' as const, coordinates: [[[30, 48], [31, 49], [30, 49], [30, 48]]] as [number, number][][] } }] }; - const result = buildFrontlineGeoJSON(fc); - expect(result).toBe(fc); // Same reference — passthrough - }); + it('returns null for null/undefined input', () => { + expect(buildFrontlineGeoJSON(null)).toBeNull(); + expect(buildFrontlineGeoJSON(undefined)).toBeNull(); + }); + + it('returns the input unchanged when valid', () => { + const fc = { + type: 'FeatureCollection' as const, + features: [ + { + type: 'Feature' as const, + properties: { name: 'zone', zone_id: 1 }, + geometry: { + type: 'Polygon' as const, + coordinates: [ + [ + [30, 48], + [31, 49], + [30, 49], + [30, 48], + ], + ] as [number, number][][], + }, + }, + ], + }; + const result = buildFrontlineGeoJSON(fc); + expect(result).toBe(fc); // Same reference — passthrough + }); + + it('returns null for empty features array', () => { + const fc = { type: 'FeatureCollection' as const, features: [] }; + expect(buildFrontlineGeoJSON(fc)).toBeNull(); + }); +}); - it('returns null for empty features array', () => { - const fc = { type: 'FeatureCollection' as const, features: [] }; - expect(buildFrontlineGeoJSON(fc)).toBeNull(); - }); +// ─── Scanners ─────────────────────────────────────────────────────────────── + +describe('buildScannerGeoJSON', () => { + it('returns null for empty input', () => { + expect(buildScannerGeoJSON(undefined)).toBeNull(); + expect(buildScannerGeoJSON([])).toBeNull(); + }); + + it('builds features with scanner properties', () => { + const scanners: Scanner[] = [ + { + shortName: 'TEST', + name: 'Test System', + lat: 39.0, + lng: -104.0, + city: 'Denver', + state: 'CO', + clientCount: 5, + description: 'Demo', + }, + ]; + const result = buildScannerGeoJSON(scanners); + expect(result!.features).toHaveLength(1); + expect(result!.features[0].properties?.type).toBe('scanner'); + expect(result!.features[0].properties?.name).toBe('Test System'); + }); }); diff --git a/frontend/src/__tests__/mesh/gateEnvelope.test.ts b/frontend/src/__tests__/mesh/gateEnvelope.test.ts new file mode 100644 index 00000000..217d1813 --- /dev/null +++ b/frontend/src/__tests__/mesh/gateEnvelope.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; + +import { + gateEnvelopeDisplayText, + gateEnvelopeState, + isEncryptedGateEnvelope, +} from '@/mesh/gateEnvelope'; +import { normalizePayload } from '@/mesh/meshProtocol'; +import { validateEventPayload } from '@/mesh/meshSchema'; + +describe('gate envelope protocol', () => { + it('normalizes encrypted gate-message payloads', () => { + expect( + normalizePayload('gate_message', { + gate: 'Finance', + epoch: '2', + ciphertext: 'opaque', + nonce: 'nonce-2', + sender_ref: 'persona-fin-1', + }), + ).toEqual({ + gate: 'finance', + epoch: 2, + ciphertext: 'opaque', + nonce: 'nonce-2', + sender_ref: 'persona-fin-1', + format: 'g1', + }); + }); + + it('accepts encrypted gate-message envelopes and rejects plaintext ones', () => { + expect( + validateEventPayload('gate_message', { + gate: 'finance', + epoch: 2, + ciphertext: 'opaque', + nonce: 'nonce-2', + sender_ref: 'persona-fin-1', + format: 'g1', + }), + ).toEqual({ ok: true }); + + expect( + validateEventPayload('gate_message', { + gate: 'finance', + message: 'plaintext', + }), + ).toEqual({ ok: false, reason: 'Payload is not normalized' }); + }); +}); + +describe('gate envelope display', () => { + it('detects encrypted gate messages and shows placeholders honestly', () => { + const encrypted = { + event_type: 'gate_message', + gate: 'finance', + epoch: 2, + ciphertext: 'opaque', + nonce: 'nonce-2', + sender_ref: 'persona-fin-1', + }; + + expect(isEncryptedGateEnvelope(encrypted)).toBe(true); + expect(gateEnvelopeState(encrypted)).toBe('locked'); + expect(gateEnvelopeDisplayText(encrypted)).toBe('ENCRYPTED GATE MESSAGE - KEY UNAVAILABLE'); + expect( + gateEnvelopeState({ + ...encrypted, + decrypted_message: 'decoded text', + }), + ).toBe('decrypted'); + expect( + gateEnvelopeDisplayText({ + ...encrypted, + decrypted_message: 'decoded text', + }), + ).toBe('decoded text'); + expect( + gateEnvelopeState({ + event_type: 'gate_notice', + message: 'legacy plaintext', + }), + ).toBe('plaintext'); + expect( + gateEnvelopeDisplayText({ + event_type: 'gate_notice', + message: 'legacy plaintext', + }), + ).toBe('legacy plaintext'); + }); +}); diff --git a/frontend/src/__tests__/mesh/mailboxClaimPrivacy.test.ts b/frontend/src/__tests__/mesh/mailboxClaimPrivacy.test.ts new file mode 100644 index 00000000..a7cb1514 --- /dev/null +++ b/frontend/src/__tests__/mesh/mailboxClaimPrivacy.test.ts @@ -0,0 +1,158 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const deadDropTokensForContacts = vi.fn(); +const mailboxClaimToken = vi.fn(); +const mailboxDecoySharedToken = vi.fn(); + +vi.mock('@/mesh/meshDeadDrop', () => ({ + deadDropToken: vi.fn(), + deadDropTokensForContacts, +})); + +vi.mock('@/mesh/meshMailbox', () => ({ + mailboxClaimToken, + mailboxDecoySharedToken, +})); + +vi.mock('@/mesh/meshIdentity', () => ({ + deriveSenderSealKey: vi.fn(), + ensureDhKeysFresh: vi.fn(), + deriveSharedKey: vi.fn(), + encryptDM: vi.fn(), + getDHAlgo: vi.fn(() => 'X25519'), + getNodeIdentity: vi.fn(() => ({ nodeId: '!self', publicKey: 'pub' })), + getPublicKeyAlgo: vi.fn(() => 'Ed25519'), + nextSequence: vi.fn(() => 1), + verifyNodeIdBindingFromPublicKey: vi.fn(async () => true), +})); + +vi.mock('@/mesh/wormholeIdentityClient', () => ({ + buildWormholeSenderSeal: vi.fn(), + getActiveSigningContext: vi.fn(async () => null), + isWormholeSecureRequired: vi.fn(async () => false), + issueWormholeDmSenderToken: vi.fn(), + issueWormholeDmSenderTokens: vi.fn(), + registerWormholeDmKey: vi.fn(), + signRawMeshMessage: vi.fn(), + signMeshEvent: vi.fn(), +})); + +vi.mock('@/mesh/meshSchema', () => ({ + validateEventPayload: vi.fn(() => ({ ok: true, reason: 'ok' })), +})); + +describe('mailbox claim privacy padding', () => { + beforeEach(() => { + vi.resetModules(); + vi.unstubAllEnvs(); + vi.stubEnv('NEXT_PUBLIC_ENABLE_RFC2A_CLAIM_SHAPE', '1'); + deadDropTokensForContacts.mockReset(); + mailboxClaimToken.mockReset(); + mailboxDecoySharedToken.mockReset(); + mailboxClaimToken.mockImplementation(async (type: string) => `${type}-token`); + mailboxDecoySharedToken.mockImplementation(async (index: number) => `decoy-${index}`); + }); + + function buildSharedTokens(count: number): string[] { + return Array.from({ length: count }, (_, index) => `shared-${index + 1}`); + } + + it('uses bucketed shared-claim envelopes across multiple contact counts', async () => { + const mod = await import('@/mesh/meshDmClient'); + + for (const testCase of [ + { realSharedClaims: 0, expectedSharedClaims: 3, expectedTotalClaims: 5 }, + { realSharedClaims: 1, expectedSharedClaims: 3, expectedTotalClaims: 5 }, + { realSharedClaims: 3, expectedSharedClaims: 3, expectedTotalClaims: 5 }, + { realSharedClaims: 4, expectedSharedClaims: 6, expectedTotalClaims: 8 }, + { realSharedClaims: 7, expectedSharedClaims: 12, expectedTotalClaims: 14 }, + { realSharedClaims: 25, expectedSharedClaims: 30, expectedTotalClaims: 32 }, + { realSharedClaims: 30, expectedSharedClaims: 30, expectedTotalClaims: 32 }, + ]) { + deadDropTokensForContacts.mockResolvedValue(buildSharedTokens(testCase.realSharedClaims)); + + const claims = await mod.buildMailboxClaims({}); + expect(claims.slice(0, 2)).toEqual([ + { type: 'self', token: 'self-token' }, + { type: 'requests', token: 'requests-token' }, + ]); + expect(claims.filter((claim) => claim.type === 'shared')).toHaveLength( + testCase.expectedSharedClaims, + ); + expect(claims).toHaveLength(testCase.expectedTotalClaims); + } + }); + + it('falls back to the legacy shared-claim floor when the experiment is disabled', async () => { + vi.resetModules(); + vi.stubEnv('NEXT_PUBLIC_ENABLE_RFC2A_CLAIM_SHAPE', '0'); + deadDropTokensForContacts.mockResolvedValue(['shared-1', 'shared-2', 'shared-3', 'shared-4']); + + const mod = await import('@/mesh/meshDmClient'); + const claims = await mod.buildMailboxClaims({}); + const sharedClaims = claims.filter((claim) => claim.type === 'shared'); + + expect(mod.MAILBOX_SHARED_CLAIM_SHAPE_VERSION).toBe('legacy-floor-v1'); + expect(sharedClaims).toEqual([ + { type: 'shared', token: 'shared-1' }, + { type: 'shared', token: 'shared-2' }, + { type: 'shared', token: 'shared-3' }, + { type: 'shared', token: 'shared-4' }, + ]); + expect(claims).toHaveLength(6); + }); + + it('deduplicates real shared tokens before filling the bucketed envelope', async () => { + deadDropTokensForContacts.mockResolvedValue(['shared-real', 'shared-real']); + + const mod = await import('@/mesh/meshDmClient'); + const claims = await mod.buildMailboxClaims({ + alice: { blocked: false, dhPubKey: 'dh-a' }, + }); + + const sharedClaims = claims.filter((claim) => claim.type === 'shared'); + expect(sharedClaims).toEqual([ + { type: 'shared', token: 'decoy-0' }, + { type: 'shared', token: 'shared-real' }, + { type: 'shared', token: 'decoy-1' }, + ]); + }); + + it('preserves every real shared token within the supported 30-claim shared range', async () => { + const realSharedTokens = buildSharedTokens(30); + deadDropTokensForContacts.mockResolvedValue(realSharedTokens); + + const mod = await import('@/mesh/meshDmClient'); + const claims = await mod.buildMailboxClaims({}); + + const sharedTokens = claims + .filter((claim) => claim.type === 'shared') + .map((claim) => claim.token); + expect(sharedTokens).toHaveLength(30); + expect(new Set(sharedTokens)).toEqual(new Set(realSharedTokens)); + }); + + it('keeps decoy shared tokens distinct from real shared tokens', async () => { + const realSharedTokens = ['shared-1', 'shared-2', 'shared-3', 'shared-4']; + deadDropTokensForContacts.mockResolvedValue(realSharedTokens); + + const mod = await import('@/mesh/meshDmClient'); + const claims = await mod.buildMailboxClaims({}); + + const sharedTokens = claims + .filter((claim) => claim.type === 'shared') + .map((claim) => String(claim.token || '')); + const decoyTokens = sharedTokens.filter((token) => !realSharedTokens.includes(token)); + + expect(sharedTokens).toEqual([ + 'shared-1', + 'decoy-0', + 'shared-2', + 'shared-3', + 'decoy-1', + 'shared-4', + ]); + expect(decoyTokens).toEqual(['decoy-0', 'decoy-1']); + expect(decoyTokens.every((token) => !realSharedTokens.includes(token))).toBe(true); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshCanonical.test.ts b/frontend/src/__tests__/mesh/meshCanonical.test.ts new file mode 100644 index 00000000..b8d49683 --- /dev/null +++ b/frontend/src/__tests__/mesh/meshCanonical.test.ts @@ -0,0 +1,32 @@ +import { readFileSync } from 'fs'; +import path from 'path'; +import { buildSignaturePayload, type JsonValue } from '@/mesh/meshProtocol'; + +type Fixture = { + name: string; + event_type: string; + node_id: string; + sequence: number; + payload: Record; + expected: string; +}; + +describe('mesh canonical signature payloads', () => { + const cwd = process.cwd(); + const fixturePath = cwd.endsWith('frontend') + ? path.resolve(cwd, '..', 'docs', 'mesh', 'mesh-canonical-fixtures.json') + : path.resolve(cwd, 'docs', 'mesh', 'mesh-canonical-fixtures.json'); + const fixtures = JSON.parse(readFileSync(fixturePath, 'utf-8')) as Fixture[]; + + for (const fixture of fixtures) { + it(`matches fixture: ${fixture.name}`, () => { + const result = buildSignaturePayload({ + eventType: fixture.event_type, + nodeId: fixture.node_id, + sequence: fixture.sequence, + payload: fixture.payload, + }); + expect(result).toBe(fixture.expected); + }); + } +}); diff --git a/frontend/src/__tests__/mesh/meshContactStorage.test.ts b/frontend/src/__tests__/mesh/meshContactStorage.test.ts new file mode 100644 index 00000000..0fbd75aa --- /dev/null +++ b/frontend/src/__tests__/mesh/meshContactStorage.test.ts @@ -0,0 +1,244 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const controlPlaneJson = vi.fn(); +const idbStore = new Map(); + +vi.mock('@/lib/controlPlane', () => ({ + controlPlaneJson, +})); + +vi.mock('@/mesh/meshKeyStore', () => ({ + getKey: vi.fn(async (id: string) => idbStore.get(id) ?? null), + setKey: vi.fn(async (id: string, key: unknown) => { + idbStore.set(id, key); + }), + deleteKey: vi.fn(async (id: string) => { + idbStore.delete(id); + }), +})); + +async function flushStoragePersistence(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +async function waitForEncryptedContacts(): Promise { + for (let i = 0; i < 20; i += 1) { + await flushStoragePersistence(); + const stored = + sessionStorage.getItem('sb_mesh_contacts') || localStorage.getItem('sb_mesh_contacts'); + if (typeof stored === 'string' && stored.startsWith('enc:')) { + return stored; + } + } + return sessionStorage.getItem('sb_mesh_contacts') || localStorage.getItem('sb_mesh_contacts'); +} + +function bufToBase64(buf: ArrayBuffer): string { + return btoa(String.fromCharCode(...new Uint8Array(buf))); +} + +describe('meshIdentity contact storage hardening', () => { + beforeEach(() => { + vi.resetModules(); + controlPlaneJson.mockReset(); + idbStore.clear(); + const makeStorage = () => { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + }; + }; + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + }); + + async function provisionLocalIdentity(mod: typeof import('@/mesh/meshIdentity')) { + localStorage.setItem('sb_mesh_pubkey', 'test-pub'); + localStorage.setItem('sb_mesh_node_id', '!sb_contacts123456'); + localStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + const keyPair = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveKey', 'deriveBits'], + )) as CryptoKeyPair; + const publicRaw = await crypto.subtle.exportKey('raw', keyPair.publicKey); + localStorage.setItem('sb_mesh_dh_pubkey', bufToBase64(publicRaw)); + localStorage.setItem('sb_mesh_dh_algo', 'ECDH'); + idbStore.set('sb_mesh_dh_priv', keyPair.privateKey); + } + + it('hydrates secure-mode contacts from Wormhole and avoids localStorage persistence', async () => { + controlPlaneJson + .mockResolvedValueOnce({ + ok: true, + contacts: { + alice: { blocked: false, dhPubKey: 'dh_a', sharedAlias: 'alias_a' }, + }, + }) + .mockResolvedValueOnce({ + ok: true, + peer_id: 'alice', + contact: { blocked: false, dhPubKey: 'dh_a2', sharedAlias: 'alias_a' }, + }); + + const mod = await import('@/mesh/meshIdentity'); + mod.setSecureModeCached(true); + + const contacts = await mod.hydrateWormholeContacts(true); + expect(contacts.alice.dhPubKey).toBe('dh_a'); + + mod.addContact('alice', 'dh_a2'); + await Promise.resolve(); + + expect(localStorage.getItem('sb_mesh_contacts')).toBeNull(); + expect(mod.getContacts().alice.dhPubKey).toBe('dh_a2'); + expect(controlPlaneJson).toHaveBeenLastCalledWith('/api/wormhole/dm/contact', expect.any(Object)); + }); + + it('stores local contacts as encrypted ciphertext and hydrates them back', async () => { + const mod = await import('@/mesh/meshIdentity'); + await provisionLocalIdentity(mod); + + mod.addContact('alice', 'dh_a', 'Alice', 'X25519'); + mod.updateContact('alice', { + remotePrekeyFingerprint: 'fp-1', + remotePrekeyObservedFingerprint: 'fp-1', + remotePrekeyPinnedAt: 111, + remotePrekeyLastSeenAt: 222, + remotePrekeySequence: 3, + remotePrekeySignedAt: 444, + remotePrekeyMismatch: false, + }); + const stored = await waitForEncryptedContacts(); + expect(String(stored ?? '')).toMatch(/^enc:/); + expect(String(stored ?? '')).not.toContain('"alice"'); + expect(String(stored ?? '')).not.toContain('"dh_a"'); + + const hydrated = await mod.hydrateWormholeContacts(true); + expect(hydrated.alice.dhPubKey).toBe('dh_a'); + expect(hydrated.alice.alias).toBe('Alice'); + expect(hydrated.alice.remotePrekeyFingerprint).toBe('fp-1'); + expect(hydrated.alice.remotePrekeyObservedFingerprint).toBe('fp-1'); + expect(hydrated.alice.remotePrekeyPinnedAt).toBe(111); + expect(hydrated.alice.remotePrekeyLastSeenAt).toBe(222); + expect(hydrated.alice.remotePrekeySequence).toBe(3); + expect(hydrated.alice.remotePrekeySignedAt).toBe(444); + expect(hydrated.alice.remotePrekeyMismatch).toBe(false); + }); + + it('migrates legacy plaintext contacts to encrypted storage on first hydrate', async () => { + const mod = await import('@/mesh/meshIdentity'); + await provisionLocalIdentity(mod); + + localStorage.setItem( + 'sb_mesh_contacts', + JSON.stringify({ alice: { blocked: false, dhPubKey: 'legacy_dh', alias: 'Legacy Alice' } }), + ); + + const hydrated = await mod.hydrateWormholeContacts(true); + expect(hydrated.alice.dhPubKey).toBe('legacy_dh'); + expect(hydrated.alice.alias).toBe('Legacy Alice'); + + const stored = await waitForEncryptedContacts(); + expect(String(stored ?? '')).toMatch(/^enc:/); + expect(String(stored ?? '')).not.toContain('legacy_dh'); + }); + + it('encrypts identity-bound browser payloads under distinct info domains', async () => { + const mod = await import('@/mesh/meshIdentity'); + await provisionLocalIdentity(mod); + + const accessCipher = await mod.encryptIdentityBoundStoragePayload( + [{ sender_id: 'alice', timestamp: 1 }], + 'SB-ACCESS-REQUESTS-STORAGE-V1', + ); + expect(accessCipher).toMatch(/^enc:/); + expect(accessCipher).not.toContain('alice'); + + const decrypted = await mod.decryptIdentityBoundStoragePayload( + accessCipher, + 'SB-ACCESS-REQUESTS-STORAGE-V1', + [], + ); + expect(decrypted).toEqual([{ sender_id: 'alice', timestamp: 1 }]); + + await expect( + mod.decryptIdentityBoundStoragePayload( + accessCipher, + 'SB-PENDING-CONTACTS-STORAGE-V1', + [], + ), + ).rejects.toThrow(); + }); + + it('treats unreadable encrypted contacts as empty and warns instead of crashing', async () => { + const mod = await import('@/mesh/meshIdentity'); + await provisionLocalIdentity(mod); + + localStorage.setItem('sb_mesh_contacts', 'enc:not-valid-ciphertext'); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const hydrated = await mod.hydrateWormholeContacts(true); + + expect(hydrated).toEqual({}); + expect(warn).toHaveBeenCalledWith( + '[mesh] contact storage unreadable — treating as empty contacts', + expect.anything(), + ); + warn.mockRestore(); + }); + + it('purges browser-persisted contact graph when secure mode boundary is applied', async () => { + const mod = await import('@/mesh/meshIdentity'); + localStorage.setItem( + 'sb_mesh_contacts', + JSON.stringify({ bob: { blocked: false, sharedAlias: 'peer-b' } }), + ); + + mod.purgeBrowserContactGraph(); + + expect(localStorage.getItem('sb_mesh_contacts')).toBeNull(); + expect(mod.getContacts()).toEqual({}); + }); + + it('rotates the mailbox-claim secret when identity state is cleared', async () => { + const { mailboxClaimToken } = await import('@/mesh/meshMailbox'); + const mod = await import('@/mesh/meshIdentity'); + await provisionLocalIdentity(mod); + + const first = await mailboxClaimToken('requests', '!sb_contacts123456'); + const second = await mailboxClaimToken('requests', '!sb_contacts123456'); + expect(second).toBe(first); + + await mod.clearBrowserIdentityState(); + + localStorage.setItem('sb_mesh_pubkey', 'test-pub'); + localStorage.setItem('sb_mesh_node_id', '!sb_contacts123456'); + localStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + const keyPair = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveKey', 'deriveBits'], + )) as CryptoKeyPair; + const publicRaw = await crypto.subtle.exportKey('raw', keyPair.publicKey); + localStorage.setItem('sb_mesh_dh_pubkey', bufToBase64(publicRaw)); + localStorage.setItem('sb_mesh_dh_algo', 'ECDH'); + idbStore.set('sb_mesh_dh_priv', keyPair.privateKey); + + const rotated = await mailboxClaimToken('requests', '!sb_contacts123456'); + expect(rotated).not.toBe(first); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshDmConsent.test.ts b/frontend/src/__tests__/mesh/meshDmConsent.test.ts new file mode 100644 index 00000000..d2306800 --- /dev/null +++ b/frontend/src/__tests__/mesh/meshDmConsent.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest'; + +import { + allDmPeerIds, + buildAliasRotateMessage, + buildAccessGrantedMessage, + buildContactAcceptMessage, + buildContactDenyMessage, + buildContactOfferMessage, + mergeAliasHistory, + parseAliasRotateMessage, + parseAccessGrantedMessage, + parseDmConsentMessage, + preferredDmPeerId, +} from '@/mesh/meshDmConsent'; + +describe('mesh DM consent helpers', () => { + it('builds and parses access-granted payloads', () => { + const message = buildAccessGrantedMessage('dmx_alpha'); + expect(parseAccessGrantedMessage(message)).toEqual({ shared_alias: 'dmx_alpha' }); + }); + + it('builds and parses off-ledger contact offer payloads', () => { + const message = buildContactOfferMessage('dh_pub', 'X25519', '40.12,-105.27'); + expect(parseDmConsentMessage(message)).toEqual({ + kind: 'contact_offer', + dh_pub_key: 'dh_pub', + dh_algo: 'X25519', + geo_hint: '40.12,-105.27', + }); + }); + + it('builds and parses off-ledger contact accept payloads', () => { + const message = buildContactAcceptMessage('dmx_pairwise'); + expect(parseDmConsentMessage(message)).toEqual({ + kind: 'contact_accept', + shared_alias: 'dmx_pairwise', + }); + }); + + it('builds and parses off-ledger contact deny payloads', () => { + const message = buildContactDenyMessage('declined'); + expect(parseDmConsentMessage(message)).toEqual({ + kind: 'contact_deny', + reason: 'declined', + }); + }); + + it('prefers the pairwise alias for shared DM routing', () => { + expect(preferredDmPeerId('node_public', { sharedAlias: 'dmx_pairwise' })).toBe('dmx_pairwise'); + expect(preferredDmPeerId('node_public', { sharedAlias: '' })).toBe('node_public'); + }); + + it('keeps both alias and public ids during the transition window', () => { + expect(allDmPeerIds('node_public', { sharedAlias: 'dmx_pairwise' })).toEqual([ + 'dmx_pairwise', + 'node_public', + ]); + expect(allDmPeerIds('node_public', { sharedAlias: 'node_public' })).toEqual(['node_public']); + }); + + it('builds and parses alias rotation control payloads', () => { + const message = buildAliasRotateMessage('dmx_next'); + expect(parseAliasRotateMessage(message)).toEqual({ shared_alias: 'dmx_next' }); + }); + + it('promotes pending alias after the grace window elapses', () => { + const now = Date.now(); + expect( + preferredDmPeerId('node_public', { + sharedAlias: 'dmx_current', + pendingSharedAlias: 'dmx_next', + sharedAliasGraceUntil: now - 1, + }), + ).toBe('dmx_next'); + }); + + it('keeps alias history compact and unique', () => { + expect(mergeAliasHistory(['dmx_a', 'dmx_b', 'dmx_a', 'dmx_c', 'dmx_d'], 3)).toEqual([ + 'dmx_a', + 'dmx_b', + 'dmx_c', + ]); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshDmWorkerVault.test.ts b/frontend/src/__tests__/mesh/meshDmWorkerVault.test.ts new file mode 100644 index 00000000..898b1dad --- /dev/null +++ b/frontend/src/__tests__/mesh/meshDmWorkerVault.test.ts @@ -0,0 +1,286 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +type StoreRecord = Map; +type DbRecord = { + version: number; + stores: Map; +}; + +const databases = new Map(); +const deletedDatabases: string[] = []; +const workerInstances: FakeWorker[] = []; + +vi.mock('@/lib/controlPlane', () => ({ + controlPlaneJson: vi.fn(), +})); + +vi.mock('@/mesh/wormholeIdentityClient', () => ({ + ensureWormholeReadyForSecureAction: vi.fn(async () => undefined), + isWormholeReady: vi.fn(async () => false), +})); + +vi.mock('@/mesh/meshIdentity', () => ({ + getDHAlgo: vi.fn(() => 'X25519'), +})); + +function makeStorage() { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + get length() { + return values.size; + }, + key: (_i: number) => null as string | null, + }; +} + +class FakeWorker { + onmessage: ((event: MessageEvent<{ id: string; ok: boolean; result?: string }>) => void) | null = + null; + terminated = false; + + constructor() { + workerInstances.push(this); + } + + postMessage(message: { id: string }) { + queueMicrotask(() => { + this.onmessage?.({ + data: { id: message.id, ok: true, result: '' }, + } as MessageEvent<{ id: string; ok: boolean; result?: string }>); + }); + } + + terminate() { + this.terminated = true; + } +} + +function domStringList(record: DbRecord): DOMStringList { + return { + contains: (name: string) => record.stores.has(name), + item: (index: number) => Array.from(record.stores.keys())[index] ?? null, + get length() { + return record.stores.size; + }, + } as DOMStringList; +} + +function makeRequest( + executor: (request: IDBRequest) => void, + tx?: IDBTransaction, +): IDBRequest { + const request = {} as IDBRequest; + queueMicrotask(() => { + executor(request); + tx?.oncomplete?.(new Event('complete') as Event); + }); + return request; +} + +function makeObjectStore(record: DbRecord, name: string, tx: IDBTransaction): IDBObjectStore { + const store = record.stores.get(name); + if (!store) throw new Error(`missing object store ${name}`); + return { + get(key: IDBValidKey) { + return makeRequest((request) => { + (request as { result?: unknown }).result = store.get(String(key)); + request.onsuccess?.(new Event('success') as Event); + }, tx); + }, + put(value: unknown, key?: IDBValidKey) { + return makeRequest((request) => { + store.set(String(key ?? ''), value); + (request as { result?: unknown }).result = key; + request.onsuccess?.(new Event('success') as Event); + }, tx); + }, + delete(key: IDBValidKey) { + return makeRequest((request) => { + store.delete(String(key)); + request.onsuccess?.(new Event('success') as Event); + }, tx); + }, + } as unknown as IDBObjectStore; +} + +function makeTransaction(record: DbRecord): IDBTransaction { + const tx = { + oncomplete: null, + onerror: null, + onabort: null, + objectStore: (name: string) => makeObjectStore(record, name, tx as unknown as IDBTransaction), + } as unknown as IDBTransaction; + return tx; +} + +function makeDb(name: string, record: DbRecord): IDBDatabase { + return { + name, + version: record.version, + objectStoreNames: domStringList(record), + createObjectStore(storeName: string) { + if (!record.stores.has(storeName)) { + record.stores.set(storeName, new Map()); + } + return {} as IDBObjectStore; + }, + transaction(storeName: string | string[]) { + return makeTransaction(record); + }, + close() { + /* noop */ + }, + } as unknown as IDBDatabase; +} + +function createFakeIndexedDb() { + return { + open(name: string, version?: number) { + const request = {} as IDBOpenDBRequest; + queueMicrotask(() => { + const resolvedVersion = Number(version || 1); + let record = databases.get(name); + const upgrading = !record || resolvedVersion > record.version; + if (!record) { + record = { version: resolvedVersion, stores: new Map() }; + databases.set(name, record); + } + if (upgrading) { + record.version = resolvedVersion; + (request as { result?: IDBDatabase }).result = makeDb(name, record); + request.onupgradeneeded?.(new Event('upgradeneeded') as IDBVersionChangeEvent); + } + (request as { result?: IDBDatabase }).result = makeDb(name, record); + request.onsuccess?.(new Event('success') as Event); + }); + return request; + }, + deleteDatabase(name: string) { + const request = {} as IDBOpenDBRequest; + queueMicrotask(() => { + deletedDatabases.push(name); + databases.delete(name); + request.onsuccess?.(new Event('success') as Event); + }); + return request; + }, + }; +} + +function ensureStore(name: string, version: number, storeName: string): StoreRecord { + let record = databases.get(name); + if (!record) { + record = { version, stores: new Map() }; + databases.set(name, record); + } + record.version = Math.max(record.version, version); + if (!record.stores.has(storeName)) { + record.stores.set(storeName, new Map()); + } + return record.stores.get(storeName)!; +} + +function getStoredValue(name: string, storeName: string, key: string): unknown { + return databases.get(name)?.stores.get(storeName)?.get(key); +} + +describe('worker ratchet vault hardening', () => { + beforeEach(() => { + vi.resetModules(); + databases.clear(); + deletedDatabases.length = 0; + workerInstances.length = 0; + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'Worker', { + value: FakeWorker, + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'indexedDB', { + value: createFakeIndexedDb(), + configurable: true, + writable: true, + }); + }); + + it('persists worker ratchet state as an encrypted blob instead of raw state', async () => { + const mod = await import('@/mesh/meshDmWorkerVault'); + const sample = { + alice: { + algo: 'X25519', + rk: 'root-key', + cks: 'send-chain', + ckr: 'recv-chain', + dhSelfPub: 'pub', + dhSelfPriv: 'private-material', + dhRemote: 'remote', + ns: 1, + nr: 2, + pn: 3, + skipped: { 'remote:1': 'mk' }, + updated: 123, + }, + }; + + await mod.writeWorkerRatchetStates(sample); + + const raw = getStoredValue(mod.WORKER_RATCHET_DB, 'ratchet', 'state'); + expect(typeof raw).toBe('string'); + expect(String(raw)).not.toContain('dhSelfPriv'); + expect(String(raw)).not.toContain('private-material'); + + const loaded = await mod.readWorkerRatchetStates(); + expect(loaded).toEqual(sample); + }); + + it('migrates legacy plaintext worker state into encrypted storage on read', async () => { + const legacyStore = ensureStore('sb_mesh_dm_worker', 1, 'ratchet'); + legacyStore.set('state', { + bob: { + algo: 'X25519', + rk: 'legacy-rk', + dhSelfPub: 'legacy-pub', + dhSelfPriv: 'legacy-private', + dhRemote: 'legacy-remote', + ns: 0, + nr: 0, + pn: 0, + updated: 999, + }, + }); + + const mod = await import('@/mesh/meshDmWorkerVault'); + const loaded = await mod.readWorkerRatchetStates(); + const raw = getStoredValue(mod.WORKER_RATCHET_DB, 'ratchet', 'state'); + + expect(loaded.bob?.dhSelfPriv).toBe('legacy-private'); + expect(typeof raw).toBe('string'); + expect(String(raw)).not.toContain('legacy-private'); + }); + + it('purgeBrowserDmState clears worker persistence and legacy browser copies', async () => { + localStorage.setItem('sb_mesh_dm_ratchet', 'legacy'); + sessionStorage.setItem('sb_mesh_ratchet_telemetry', '{"seen":1}'); + const mod = await import('@/mesh/meshDmWorkerClient'); + + await mod.purgeBrowserDmState(); + + expect(localStorage.getItem('sb_mesh_dm_ratchet')).toBeNull(); + expect(sessionStorage.getItem('sb_mesh_ratchet_telemetry')).toBeNull(); + expect(deletedDatabases).toContain('sb_mesh_dm_worker'); + expect(workerInstances[0]?.terminated).toBe(true); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshIdentitySeparation.test.ts b/frontend/src/__tests__/mesh/meshIdentitySeparation.test.ts new file mode 100644 index 00000000..7ba6aac2 --- /dev/null +++ b/frontend/src/__tests__/mesh/meshIdentitySeparation.test.ts @@ -0,0 +1,112 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('@/lib/controlPlane', () => ({ + controlPlaneJson: vi.fn(), +})); + +vi.mock('@/mesh/meshKeyStore', () => ({ + getKey: vi.fn().mockResolvedValue(null), + setKey: vi.fn().mockResolvedValue(undefined), + deleteKey: vi.fn().mockResolvedValue(undefined), +})); + +describe('mesh identity storage separation', () => { + beforeEach(() => { + vi.resetModules(); + const makeStorage = () => { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + }; + }; + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + }); + + it('keeps public browser identity separate from Wormhole descriptor cache', async () => { + const mod = await import('@/mesh/meshIdentity'); + + mod.cachePublicIdentity({ + nodeId: '!sb_public', + publicKey: 'public-key', + publicKeyAlgo: 'Ed25519', + }); + mod.cacheWormholeIdentityDescriptor({ + nodeId: '!sb_wormhole', + publicKey: 'wormhole-key', + publicKeyAlgo: 'Ed25519', + }); + + expect(mod.getStoredNodeDescriptor()).toEqual({ + nodeId: '!sb_public', + publicKey: 'public-key', + publicKeyAlgo: 'Ed25519', + }); + expect(mod.getWormholeIdentityDescriptor()).toEqual({ + nodeId: '!sb_wormhole', + publicKey: 'wormhole-key', + publicKeyAlgo: 'Ed25519', + }); + }); + + it('clears browser public identity and Wormhole descriptor cache together on full reset', async () => { + const mod = await import('@/mesh/meshIdentity'); + + mod.cachePublicIdentity({ + nodeId: '!sb_public', + publicKey: 'public-key', + publicKeyAlgo: 'Ed25519', + }); + mod.cacheWormholeIdentityDescriptor({ + nodeId: '!sb_wormhole', + publicKey: 'wormhole-key', + publicKeyAlgo: 'Ed25519', + }); + + await mod.clearBrowserIdentityState(); + + expect(mod.getStoredNodeDescriptor()).toBeNull(); + expect(mod.getWormholeIdentityDescriptor()).toBeNull(); + }); + + it('migrates legacy browser and Wormhole node ids to the current format', async () => { + const mod = await import('@/mesh/meshIdentity'); + const publicKey = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + const currentNodeId = await mod.deriveNodeIdFromPublicKey(publicKey); + + mod.cachePublicIdentity({ + nodeId: '!sb_deadbeef', + publicKey, + publicKeyAlgo: 'Ed25519', + }); + mod.cacheWormholeIdentityDescriptor({ + nodeId: '!sb_deadbeef', + publicKey, + publicKeyAlgo: 'Ed25519', + }); + + await mod.migrateLegacyNodeIds(); + + expect(mod.getStoredNodeDescriptor()).toEqual({ + nodeId: currentNodeId, + publicKey, + publicKeyAlgo: 'Ed25519', + }); + expect(mod.getWormholeIdentityDescriptor()).toEqual({ + nodeId: currentNodeId, + publicKey, + publicKeyAlgo: 'Ed25519', + }); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshMerkle.test.ts b/frontend/src/__tests__/mesh/meshMerkle.test.ts new file mode 100644 index 00000000..8a711365 --- /dev/null +++ b/frontend/src/__tests__/mesh/meshMerkle.test.ts @@ -0,0 +1,32 @@ +import { readFileSync } from 'fs'; +import path from 'path'; +import { buildMerkleRoot, verifyMerkleProof } from '@/mesh/meshMerkle'; + +type Fixture = { + leaves: string[]; + root: string; + proofs: Record; +}; + +describe('mesh merkle fixtures', () => { + const cwd = process.cwd(); + const fixturePath = cwd.endsWith('frontend') + ? path.resolve(cwd, '..', 'docs', 'mesh', 'mesh-merkle-fixtures.json') + : path.resolve(cwd, 'docs', 'mesh', 'mesh-merkle-fixtures.json'); + const fixtures = JSON.parse(readFileSync(fixturePath, 'utf-8')) as Fixture; + + it('builds the expected root', async () => { + const root = await buildMerkleRoot(fixtures.leaves); + expect(root).toBe(fixtures.root); + }); + + it('verifies provided proofs', async () => { + const root = fixtures.root; + for (const [idxStr, proof] of Object.entries(fixtures.proofs)) { + const idx = Number(idxStr); + const leaf = fixtures.leaves[idx]; + const ok = await verifyMerkleProof(leaf, idx, proof, root); + expect(ok).toBe(true); + } + }); +}); diff --git a/frontend/src/__tests__/mesh/meshPrivacyHints.test.ts b/frontend/src/__tests__/mesh/meshPrivacyHints.test.ts new file mode 100644 index 00000000..ffa17a78 --- /dev/null +++ b/frontend/src/__tests__/mesh/meshPrivacyHints.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildDmTrustHint, + buildPrivateLaneHint, + dmTrustPrimaryActionLabel, + isFirstContactTrustOnly, + shortTrustFingerprint, + shouldAutoRevealSasForTrust, +} from '@/mesh/meshPrivacyHints'; + +describe('meshPrivacyHints', () => { + it('flags recent private-lane fallback as a danger hint', () => { + const hint = buildPrivateLaneHint({ + activeTab: 'dms', + recentPrivateFallback: true, + recentPrivateFallbackReason: 'Tor transport failed and clearnet relay was used.', + dmTransportMode: 'relay', + }); + + expect(hint).toEqual( + expect.objectContaining({ + severity: 'danger', + title: 'RECENT PRIVACY DOWNGRADE', + }), + ); + expect(hint?.detail).toContain('clearnet relay'); + }); + + it('flags remote prekey mismatch as a danger trust hint', () => { + const hint = buildDmTrustHint({ + remotePrekeyMismatch: true, + }); + + expect(hint).toEqual( + expect.objectContaining({ + severity: 'danger', + title: 'REMOTE PREKEY CHANGED', + }), + ); + }); + + it('flags first-seen pinned contacts as TOFU until verified', () => { + const contact = { + remotePrekeyFingerprint: 'abc123', + remotePrekeyPinnedAt: 123, + verify_registry: false, + verify_inband: false, + verified: false, + }; + + expect(isFirstContactTrustOnly(contact)).toBe(true); + expect(buildDmTrustHint(contact)).toEqual( + expect.objectContaining({ + severity: 'warn', + title: 'FIRST CONTACT (TOFU ONLY)', + }), + ); + expect(buildDmTrustHint(contact)?.detail).toContain('not proof of sender identity'); + expect(dmTrustPrimaryActionLabel(contact)).toBe('VERIFY SAS NOW'); + expect(shouldAutoRevealSasForTrust(contact)).toBe(true); + }); + + it('auto-reveals SAS for trust hazards but keeps ordinary verified contacts quiet', () => { + expect( + shouldAutoRevealSasForTrust({ + remotePrekeyMismatch: true, + }), + ).toBe(true); + expect( + shouldAutoRevealSasForTrust({ + verify_mismatch: true, + }), + ).toBe(true); + expect( + shouldAutoRevealSasForTrust({ + verified: true, + verify_inband: true, + verify_registry: true, + }), + ).toBe(false); + expect( + dmTrustPrimaryActionLabel({ + verified: true, + verify_inband: true, + verify_registry: true, + }), + ).toBe('SHOW SAS'); + }); + + it('shortens long trust fingerprints for display', () => { + expect(shortTrustFingerprint('abcdef0123456789fedcba9876543210')).toBe('abcdef01..543210'); + expect(shortTrustFingerprint('abcd1234')).toBe('abcd1234'); + expect(shortTrustFingerprint('')).toBe('unknown'); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshSigningKeyHardening.test.ts b/frontend/src/__tests__/mesh/meshSigningKeyHardening.test.ts new file mode 100644 index 00000000..65ec2eba --- /dev/null +++ b/frontend/src/__tests__/mesh/meshSigningKeyHardening.test.ts @@ -0,0 +1,175 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +// Track what gets stored in IndexedDB +const idbStore = new Map(); +const deletedDatabases: string[] = []; + +vi.mock('@/lib/controlPlane', () => ({ + controlPlaneJson: vi.fn(), +})); + +vi.mock('@/mesh/meshKeyStore', () => ({ + getKey: vi.fn(async (id: string) => idbStore.get(id) ?? null), + setKey: vi.fn(async (id: string, key: unknown) => { + idbStore.set(id, key); + }), + deleteKey: vi.fn(async (id: string) => { + idbStore.delete(id); + }), +})); + +function makeStorage() { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + get length() { + return values.size; + }, + key: (_i: number) => null as string | null, + }; +} + +describe('signing key storage hardening', () => { + beforeEach(() => { + vi.resetModules(); + idbStore.clear(); + deletedDatabases.length = 0; + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'indexedDB', { + value: { + deleteDatabase: vi.fn((name: string) => { + deletedDatabases.push(name); + const request = {} as IDBOpenDBRequest; + queueMicrotask(() => { + request.onsuccess?.(new Event('success') as Event); + }); + return request; + }), + }, + configurable: true, + writable: true, + }); + }); + + it('getNodeIdentity returns identity even when privateKey is empty (post-migration)', async () => { + const mod = await import('@/mesh/meshIdentity'); + + // Simulate a state where the signing key has already been migrated: + // publicKey and nodeId exist, but privateKey does not. + localStorage.setItem('sb_mesh_pubkey', 'test-pub'); + localStorage.setItem('sb_mesh_node_id', '!sb_abcd1234abcd1234'); + localStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + // No sb_mesh_privkey — simulates post-migration state + + const identity = mod.getNodeIdentity(); + expect(identity).not.toBeNull(); + expect(identity!.publicKey).toBe('test-pub'); + expect(identity!.nodeId).toBe('!sb_abcd1234abcd1234'); + expect(identity!.privateKey).toBe(''); + }); + + it('getNodeIdentity triggers eager migration and does not expose legacy privateKey', async () => { + const mod = await import('@/mesh/meshIdentity'); + + localStorage.setItem('sb_mesh_pubkey', 'test-pub'); + localStorage.setItem('sb_mesh_node_id', '!sb_abcd1234abcd1234'); + localStorage.setItem('sb_mesh_privkey', '{"fake":"jwk"}'); + localStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + + const identity = mod.getNodeIdentity(); + expect(identity).not.toBeNull(); + expect(identity!.privateKey).toBe(''); + + // The eager migration fires asynchronously (void ensureSigningPrivateKey()). + // In this test environment crypto.subtle.importKey will fail on the fake JWK, + // but the extractable browser copy should still be scrubbed. + await new Promise((r) => setTimeout(r, 10)); + expect(localStorage.getItem('sb_mesh_privkey')).toBeNull(); + expect(identity!.publicKey).toBe('test-pub'); + }); + + it('purgeBrowserSigningMaterial clears IndexedDB signing key', async () => { + const { deleteKey } = await import('@/mesh/meshKeyStore'); + const mod = await import('@/mesh/meshIdentity'); + + idbStore.set('sb_mesh_sign_priv', 'mock-crypto-key'); + localStorage.setItem('sb_mesh_privkey', '{"fake":"jwk"}'); + localStorage.setItem('sb_mesh_sequence', '42'); + + await mod.purgeBrowserSigningMaterial(); + + expect(deleteKey).toHaveBeenCalledWith('sb_mesh_sign_priv'); + expect(localStorage.getItem('sb_mesh_privkey')).toBeNull(); + expect(localStorage.getItem('sb_mesh_sequence')).toBeNull(); + }); + + it('clearBrowserIdentityState clears both DH and signing keys from IndexedDB', async () => { + const { deleteKey } = await import('@/mesh/meshKeyStore'); + const mod = await import('@/mesh/meshIdentity'); + + localStorage.setItem('sb_mesh_pubkey', 'test-pub'); + localStorage.setItem('sb_mesh_node_id', '!sb_test'); + localStorage.setItem('sb_mesh_privkey', '{"fake":"jwk"}'); + localStorage.setItem('sb_mesh_session_mode', 'true'); + localStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + localStorage.setItem('sb_dm_bundle_fingerprint', 'bundle-fp'); + sessionStorage.setItem('sb_wormhole_desc_node_id', '!sb_gate'); + sessionStorage.setItem('sb_mesh_dm_ratchet', 'encrypted'); + sessionStorage.setItem('sb_mesh_ratchet_telemetry', '{"seen":1}'); + + await mod.clearBrowserIdentityState(); + + expect(deleteKey).toHaveBeenCalledWith('sb_mesh_dh_priv'); + expect(deleteKey).toHaveBeenCalledWith('sb_mesh_sign_priv'); + expect(localStorage.getItem('sb_mesh_pubkey')).toBeNull(); + expect(localStorage.getItem('sb_mesh_privkey')).toBeNull(); + expect(localStorage.getItem('sb_dm_bundle_fingerprint')).toBeNull(); + expect(sessionStorage.getItem('sb_wormhole_desc_node_id')).toBeNull(); + expect(sessionStorage.getItem('sb_mesh_dm_ratchet')).toBeNull(); + expect(sessionStorage.getItem('sb_mesh_ratchet_telemetry')).toBeNull(); + expect(deletedDatabases).toContain('sb_mesh_ratchet_crypto'); + }); + + it('generateDHKeys fails closed when non-extractable DH key storage is unavailable', async () => { + const { setKey } = await import('@/mesh/meshKeyStore'); + vi.mocked(setKey).mockRejectedValueOnce(new Error('idb unavailable')); + const mod = await import('@/mesh/meshIdentity'); + + await expect(mod.generateDHKeys()).rejects.toThrow('IndexedDB required for DH key storage'); + expect(localStorage.getItem('sb_mesh_dh_privkey')).toBeNull(); + expect(sessionStorage.getItem('sb_mesh_dh_privkey')).toBeNull(); + }); + + it('signWithStoredKey is exported and throws when no key available', async () => { + const mod = await import('@/mesh/meshIdentity'); + // No key in IndexedDB or localStorage + await expect(mod.signWithStoredKey('test message')).rejects.toThrow( + 'No signing key available', + ); + }); + + it('signEvent fails closed when only public identity metadata exists', async () => { + const mod = await import('@/mesh/meshIdentity'); + + sessionStorage.setItem('sb_mesh_pubkey', 'test-pub'); + sessionStorage.setItem('sb_mesh_node_id', '!sb_abcd1234abcd1234'); + sessionStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + sessionStorage.setItem('sb_mesh_algo', 'Ed25519'); + + await expect( + mod.signEvent('message', '!sb_abcd1234abcd1234', 1, { message: 'hello' }), + ).rejects.toThrow('No signing key available'); + }); +}); diff --git a/frontend/src/__tests__/mesh/meshTerminalPolicy.test.ts b/frontend/src/__tests__/mesh/meshTerminalPolicy.test.ts new file mode 100644 index 00000000..1628b479 --- /dev/null +++ b/frontend/src/__tests__/mesh/meshTerminalPolicy.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; + +import { + getMeshTerminalWriteLockReason, + isMeshTerminalWriteCommand, +} from '@/lib/meshTerminalPolicy'; + +describe('mesh terminal policy', () => { + it('blocks sensitive terminal writes while anonymous mode is active', () => { + const reason = getMeshTerminalWriteLockReason({ + wormholeRequired: true, + wormholeReady: true, + anonymousMode: true, + anonymousModeReady: true, + }); + + expect(reason).toContain('Anonymous Infonet mode'); + expect(isMeshTerminalWriteCommand('dm', ['add', '!sb_test'])).toBe(true); + expect(isMeshTerminalWriteCommand('mesh', ['send', 'hello'])).toBe(true); + }); + + it('blocks sensitive terminal writes until Wormhole secure mode is ready', () => { + const reason = getMeshTerminalWriteLockReason({ + wormholeRequired: true, + wormholeReady: false, + anonymousMode: false, + anonymousModeReady: false, + }); + + expect(reason).toContain('until Wormhole secure mode is ready'); + expect(isMeshTerminalWriteCommand('gate', ['create', 'newsroom'])).toBe(true); + expect(isMeshTerminalWriteCommand('send', ['broadcast', 'hello'])).toBe(true); + }); + + it('keeps read-only terminal commands available', () => { + expect(isMeshTerminalWriteCommand('status', [])).toBe(false); + expect(isMeshTerminalWriteCommand('signals', ['10'])).toBe(false); + expect(isMeshTerminalWriteCommand('mesh', ['listen', '20'])).toBe(false); + expect(isMeshTerminalWriteCommand('messages', [])).toBe(false); + }); +}); diff --git a/frontend/src/__tests__/mesh/requestSenderRecovery.test.ts b/frontend/src/__tests__/mesh/requestSenderRecovery.test.ts new file mode 100644 index 00000000..f3ee05f2 --- /dev/null +++ b/frontend/src/__tests__/mesh/requestSenderRecovery.test.ts @@ -0,0 +1,224 @@ +import { + getSenderRecoveryState, + REQUEST_V2_REDUCED_VERSION, + recoverSenderSealWithFallback, + requiresSenderRecovery, + shouldAllowRequestActions, + shouldKeepUnresolvedRequestVisible, + shouldPromoteRecoveredSenderForBootstrap, + shouldPromoteRecoveredSenderForKnownContact, +} from '@/mesh/requestSenderRecovery'; + +describe('requestSenderRecovery', () => { + it('only promotes a known-contact sender when the seal verified and the sender matches', () => { + expect( + shouldPromoteRecoveredSenderForKnownContact( + { sender_id: 'alice', seal_verified: true }, + 'alice', + ), + ).toBe(true); + expect( + shouldPromoteRecoveredSenderForKnownContact( + { sender_id: 'alice', seal_verified: false }, + 'alice', + ), + ).toBe(false); + expect( + shouldPromoteRecoveredSenderForKnownContact( + { sender_id: 'mallory', seal_verified: true }, + 'alice', + ), + ).toBe(false); + }); + + it('only promotes a bootstrap-recovered sender when the seal verified', () => { + expect( + shouldPromoteRecoveredSenderForBootstrap({ + sender_id: 'alice', + seal_verified: true, + }), + ).toBe(true); + expect( + shouldPromoteRecoveredSenderForBootstrap({ + sender_id: 'alice', + seal_verified: false, + }), + ).toBe(false); + expect(shouldPromoteRecoveredSenderForBootstrap(null)).toBe(false); + }); + + it('prefers explicit request-v2 recovery markers over sealed-string inference', () => { + expect( + requiresSenderRecovery({ + sender_id: 'opaque', + sender_seal: 'v3:test', + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + }), + ).toBe(true); + expect( + getSenderRecoveryState({ + sender_id: 'opaque', + sender_seal: 'v3:test', + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + }), + ).toBe('pending'); + expect( + requiresSenderRecovery({ + sender_id: 'sealed:abcd', + sender_seal: 'v2:test', + }), + ).toBe(true); + }); + + it('only allows request actions once canonical recovery reaches verified', () => { + expect( + shouldAllowRequestActions({ + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'verified', + }), + ).toBe(true); + expect( + shouldAllowRequestActions({ + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'pending', + }), + ).toBe(false); + expect( + shouldAllowRequestActions({ + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'failed', + }), + ).toBe(false); + expect( + shouldAllowRequestActions({ + request_contract_version: undefined, + sender_recovery_required: undefined, + sender_recovery_state: undefined, + }), + ).toBe(true); + }); + + it('keeps only pending or failed canonical request-v2 mail visible in the unresolved inbox flow', () => { + expect( + shouldKeepUnresolvedRequestVisible({ + delivery_class: 'request', + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'pending', + }), + ).toBe(true); + expect( + shouldKeepUnresolvedRequestVisible({ + delivery_class: 'request', + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'failed', + }), + ).toBe(true); + expect( + shouldKeepUnresolvedRequestVisible({ + delivery_class: 'request', + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'verified', + }), + ).toBe(false); + expect( + shouldKeepUnresolvedRequestVisible({ + delivery_class: 'request', + request_contract_version: undefined, + sender_recovery_required: undefined, + sender_recovery_state: 'pending', + }), + ).toBe(false); + expect( + shouldKeepUnresolvedRequestVisible({ + delivery_class: 'shared', + request_contract_version: REQUEST_V2_REDUCED_VERSION, + sender_recovery_required: true, + sender_recovery_state: 'pending', + }), + ).toBe(false); + }); + + it('prefers local recovery and only falls back to the helper on local failure', async () => { + const openLocal = vi.fn().mockResolvedValue({ + sender_id: 'alice', + seal_verified: true, + }); + const openHelper = vi.fn().mockResolvedValue({ + sender_id: 'helper-alice', + seal_verified: true, + }); + + await expect( + recoverSenderSealWithFallback({ + wormholeReady: true, + openLocal, + openHelper, + }), + ).resolves.toEqual({ sender_id: 'alice', seal_verified: true }); + + expect(openLocal).toHaveBeenCalledTimes(1); + expect(openHelper).not.toHaveBeenCalled(); + }); + + it('uses the helper only as fallback when local recovery cannot open the seal', async () => { + const openLocal = vi.fn().mockResolvedValue(null); + const openHelper = vi.fn().mockResolvedValue({ + sender_id: 'alice', + seal_verified: true, + }); + + await expect( + recoverSenderSealWithFallback({ + wormholeReady: true, + openLocal, + openHelper, + }), + ).resolves.toEqual({ sender_id: 'alice', seal_verified: true }); + + expect(openLocal).toHaveBeenCalledTimes(1); + expect(openHelper).toHaveBeenCalledTimes(1); + }); + + it('does not invoke the helper when Wormhole fallback is unavailable', async () => { + const openLocal = vi.fn().mockResolvedValue(null); + const openHelper = vi.fn().mockResolvedValue({ + sender_id: 'alice', + seal_verified: true, + }); + + await expect( + recoverSenderSealWithFallback({ + wormholeReady: false, + openLocal, + openHelper, + }), + ).resolves.toBeNull(); + + expect(openLocal).toHaveBeenCalledTimes(1); + expect(openHelper).not.toHaveBeenCalled(); + }); + + it('treats helper failure as unresolved instead of promoting helper authority', async () => { + const openLocal = vi.fn().mockResolvedValue(null); + const openHelper = vi.fn().mockRejectedValue(new Error('helper_failed')); + + await expect( + recoverSenderSealWithFallback({ + wormholeReady: true, + openLocal, + openHelper, + }), + ).resolves.toBeNull(); + + expect(openLocal).toHaveBeenCalledTimes(1); + expect(openHelper).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/src/__tests__/mesh/requestSenderSealPolicy.test.ts b/frontend/src/__tests__/mesh/requestSenderSealPolicy.test.ts new file mode 100644 index 00000000..d882fd9a --- /dev/null +++ b/frontend/src/__tests__/mesh/requestSenderSealPolicy.test.ts @@ -0,0 +1,40 @@ +import { + ensureCanonicalRequestV2SenderSeal, + REQUEST_V2_SENDER_SEAL_VERSION_ERROR, + requiresCanonicalRequestV2SenderSeal, +} from '@/mesh/requestSenderSealPolicy'; + +describe('requestSenderSealPolicy', () => { + it('requires canonical v3 seals only for request-class sealed sender', () => { + expect( + requiresCanonicalRequestV2SenderSeal({ + deliveryClass: 'request', + useSealedSender: true, + }), + ).toBe(true); + expect( + requiresCanonicalRequestV2SenderSeal({ + deliveryClass: 'request', + useSealedSender: false, + }), + ).toBe(false); + expect( + requiresCanonicalRequestV2SenderSeal({ + deliveryClass: 'shared', + useSealedSender: true, + }), + ).toBe(false); + }); + + it('accepts v3 seals and rejects non-v3 seals for canonical request-v2 sender sealing', () => { + expect(ensureCanonicalRequestV2SenderSeal('v3:ephemeral:payload')).toBe( + 'v3:ephemeral:payload', + ); + expect(() => ensureCanonicalRequestV2SenderSeal('v2:legacy-payload')).toThrow( + REQUEST_V2_SENDER_SEAL_VERSION_ERROR, + ); + expect(() => ensureCanonicalRequestV2SenderSeal('')).toThrow( + REQUEST_V2_SENDER_SEAL_VERSION_ERROR, + ); + }); +}); diff --git a/frontend/src/__tests__/mesh/requestSenderSealRecoveryWindow.test.ts b/frontend/src/__tests__/mesh/requestSenderSealRecoveryWindow.test.ts new file mode 100644 index 00000000..cc410c99 --- /dev/null +++ b/frontend/src/__tests__/mesh/requestSenderSealRecoveryWindow.test.ts @@ -0,0 +1,178 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const controlPlaneJson = vi.fn(); +const idbStore = new Map(); + +vi.mock('@/lib/controlPlane', () => ({ + controlPlaneJson, +})); + +vi.mock('@/mesh/meshKeyStore', () => ({ + getKey: vi.fn(async (id: string) => idbStore.get(id) ?? null), + setKey: vi.fn(async (id: string, key: unknown) => { + idbStore.set(id, key); + }), + deleteKey: vi.fn(async (id: string) => { + idbStore.delete(id); + }), +})); + +function bufToBase64(buf: ArrayBuffer): string { + return btoa(String.fromCharCode(...new Uint8Array(buf))); +} + +async function buildV3SealForRecipient(params: { + recipientPublicKey: CryptoKey; + recipientId: string; + msgId: string; + plaintext: string; +}) { + const { encryptDM } = await import('@/mesh/meshIdentity'); + const { PROTOCOL_VERSION } = await import('@/mesh/meshProtocol'); + const ephemeral = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + true, + ['deriveBits', 'deriveKey'], + )) as CryptoKeyPair; + const ephemeralPubRaw = await crypto.subtle.exportKey('raw', ephemeral.publicKey); + const ephemeralPub = bufToBase64(ephemeralPubRaw); + const secret = await crypto.subtle.deriveBits( + { name: 'ECDH', public: params.recipientPublicKey }, + ephemeral.privateKey, + 256, + ); + const salt = await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode( + `SB-SEAL-SALT|${params.recipientId}|${params.msgId}|${PROTOCOL_VERSION}|${ephemeralPub}`, + ), + ); + const hkdfKey = await crypto.subtle.importKey('raw', secret, 'HKDF', false, ['deriveKey']); + const sealKey = await crypto.subtle.deriveKey( + { + name: 'HKDF', + hash: 'SHA-256', + salt, + info: new TextEncoder().encode('SB-SENDER-SEAL-V3'), + }, + hkdfKey, + { name: 'AES-GCM', length: 256 }, + false, + ['encrypt', 'decrypt'], + ); + const ciphertext = await encryptDM(params.plaintext, sealKey); + return `v3:${ephemeralPub}:${ciphertext}`; +} + +describe('request sender seal recovery window', () => { + beforeEach(() => { + vi.resetModules(); + controlPlaneJson.mockReset(); + idbStore.clear(); + const makeStorage = () => { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + }; + }; + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + }); + + it('opens a v3 sender seal with the immediately previous retained recipient key', async () => { + const mod = await import('@/mesh/meshIdentity'); + const previousRecipient = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveBits', 'deriveKey'], + )) as CryptoKeyPair; + const currentRecipient = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveBits', 'deriveKey'], + )) as CryptoKeyPair; + + idbStore.set('sb_mesh_dh_priv', currentRecipient.privateKey); + idbStore.set('sb_mesh_dh_prev_priv', previousRecipient.privateKey); + localStorage.setItem('sb_mesh_dh_algo', 'ECDH'); + + const plaintext = JSON.stringify({ sender_id: 'alice', msg_id: 'msg-rotation' }); + const senderSeal = await buildV3SealForRecipient({ + recipientPublicKey: previousRecipient.publicKey, + recipientId: '!sb_recipient', + msgId: 'msg-rotation', + plaintext, + }); + + await expect( + mod.decryptSenderSealPayloadLocally(senderSeal, '', '!sb_recipient', 'msg-rotation'), + ).resolves.toBe(plaintext); + }); + + it('returns null when the prior retained recipient key is unavailable', async () => { + const mod = await import('@/mesh/meshIdentity'); + const previousRecipient = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveBits', 'deriveKey'], + )) as CryptoKeyPair; + const currentRecipient = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveBits', 'deriveKey'], + )) as CryptoKeyPair; + + idbStore.set('sb_mesh_dh_priv', currentRecipient.privateKey); + localStorage.setItem('sb_mesh_dh_algo', 'ECDH'); + + const senderSeal = await buildV3SealForRecipient({ + recipientPublicKey: previousRecipient.publicKey, + recipientId: '!sb_recipient', + msgId: 'msg-rotation-miss', + plaintext: JSON.stringify({ sender_id: 'alice', msg_id: 'msg-rotation-miss' }), + }); + + await expect( + mod.decryptSenderSealPayloadLocally(senderSeal, '', '!sb_recipient', 'msg-rotation-miss'), + ).resolves.toBeNull(); + }); + + it('retains the current DH private key in the previous-key slot when rotating', async () => { + const mod = await import('@/mesh/meshIdentity'); + const existing = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveBits', 'deriveKey'], + )) as CryptoKeyPair; + const originalGenerateKey = crypto.subtle.generateKey.bind(crypto.subtle); + const generateKeySpy = vi + .spyOn(crypto.subtle, 'generateKey') + .mockImplementation(((algorithm: AlgorithmIdentifier, extractable: boolean, keyUsages: KeyUsage[]) => { + if (algorithm === 'X25519') { + return Promise.reject(new Error('x25519_unavailable_for_test')); + } + return originalGenerateKey(algorithm, extractable, keyUsages); + }) as typeof crypto.subtle.generateKey); + + idbStore.set('sb_mesh_dh_priv', existing.privateKey); + try { + await mod.generateDHKeys(); + + expect(idbStore.get('sb_mesh_dh_prev_priv')).toBe(existing.privateKey); + expect(idbStore.get('sb_mesh_dh_priv')).not.toBe(existing.privateKey); + } finally { + generateKeySpy.mockRestore(); + } + }); +}); diff --git a/frontend/src/__tests__/mesh/wormholeIdentityClientProfiles.test.ts b/frontend/src/__tests__/mesh/wormholeIdentityClientProfiles.test.ts new file mode 100644 index 00000000..ec7a93c3 --- /dev/null +++ b/frontend/src/__tests__/mesh/wormholeIdentityClientProfiles.test.ts @@ -0,0 +1,125 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const controlPlaneJson = vi.fn(); +const getNodeIdentity = vi.fn< + () => { nodeId: string; publicKey: string; privateKey: string } | null +>(() => null); +const signEvent = vi.fn(); +const signMessage = vi.fn(); +const signWithStoredKey = vi.fn(); +const isSecureModeCached = vi.fn(() => true); +const fetchWormholeSettings = vi.fn(async () => ({ enabled: true })); +const fetchWormholeState = vi.fn(async () => ({ ready: true })); + +vi.mock('@/lib/controlPlane', () => ({ + controlPlaneJson, +})); + +vi.mock('@/mesh/meshIdentity', () => ({ + cacheWormholeIdentityDescriptor: vi.fn(), + getNodeIdentity, + getPublicKeyAlgo: vi.fn(() => 'ed25519'), + isSecureModeCached, + purgeBrowserSigningMaterial: vi.fn(async () => {}), + setSecureModeCached: vi.fn(), + signEvent, + signMessage, + signWithStoredKey, +})); + +vi.mock('@/mesh/meshProtocol', () => ({ + PROTOCOL_VERSION: 'sb-test', +})); + +vi.mock('@/mesh/wormholeClient', () => ({ + fetchWormholeSettings, + fetchWormholeState, +})); + +describe('wormholeIdentityClient strict profile hints', () => { + beforeEach(() => { + vi.resetModules(); + controlPlaneJson.mockReset(); + controlPlaneJson.mockResolvedValue({ ok: true }); + getNodeIdentity.mockReset(); + getNodeIdentity.mockReturnValue(null); + signEvent.mockReset(); + signMessage.mockReset(); + signWithStoredKey.mockReset(); + isSecureModeCached.mockReset(); + isSecureModeCached.mockReturnValue(true); + fetchWormholeSettings.mockReset(); + fetchWormholeSettings.mockResolvedValue({ enabled: true }); + fetchWormholeState.mockReset(); + fetchWormholeState.mockResolvedValue({ ready: true }); + }); + + it('applies strict gate_operator enforcement to gate persona and compose operations', async () => { + const mod = await import('@/mesh/wormholeIdentityClient'); + + await mod.listWormholeGatePersonas('infonet'); + await mod.createWormholeGatePersona('infonet', 'persona-1'); + await mod.activateWormholeGatePersona('infonet', 'persona-1'); + await mod.clearWormholeGatePersona('infonet'); + await mod.retireWormholeGatePersona('infonet', 'persona-1'); + await mod.composeWormholeGateMessage('infonet', 'hello'); + + expect(controlPlaneJson).toHaveBeenNthCalledWith( + 1, + '/api/wormhole/gate/infonet/personas', + expect.objectContaining({ + capabilityIntent: 'wormhole_gate_persona', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }), + ); + for (let i = 2; i <= 5; i += 1) { + expect(controlPlaneJson).toHaveBeenNthCalledWith( + i, + expect.any(String), + expect.objectContaining({ + capabilityIntent: 'wormhole_gate_persona', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }), + ); + } + expect(controlPlaneJson).toHaveBeenNthCalledWith( + 6, + '/api/wormhole/gate/message/compose', + expect.objectContaining({ + capabilityIntent: 'wormhole_gate_content', + sessionProfileHint: 'gate_operator', + enforceProfileHint: true, + }), + ); + }); + + it('browser raw signing fails closed instead of falling back to legacy jwk signing', async () => { + fetchWormholeSettings.mockResolvedValue({ enabled: false }); + fetchWormholeState.mockResolvedValue({ ready: false }); + getNodeIdentity.mockReturnValue({ + nodeId: '!sb_browser', + publicKey: 'browser-pub', + privateKey: '', + }); + signWithStoredKey.mockRejectedValue(new Error('no key')); + + const mod = await import('@/mesh/wormholeIdentityClient'); + + await expect(mod.signRawMeshMessage('payload')).rejects.toThrow( + 'browser_signing_key_unavailable', + ); + expect(signWithStoredKey).toHaveBeenCalledWith('payload'); + expect(signMessage).not.toHaveBeenCalled(); + }); + + it('keeps the cached secure boundary when wormhole settings fetch fails', async () => { + fetchWormholeSettings.mockRejectedValue(new Error('network down')); + isSecureModeCached.mockReturnValue(true); + + const mod = await import('@/mesh/wormholeIdentityClient'); + + await expect(mod.isWormholeSecureRequired()).resolves.toBe(true); + }); +}); diff --git a/frontend/src/__tests__/utils/aircraftClassification.test.ts b/frontend/src/__tests__/utils/aircraftClassification.test.ts index 62930c73..b147692b 100644 --- a/frontend/src/__tests__/utils/aircraftClassification.test.ts +++ b/frontend/src/__tests__/utils/aircraftClassification.test.ts @@ -1,5 +1,10 @@ import { describe, it, expect } from 'vitest'; -import { classifyAircraft, HELI_TYPES, TURBOPROP_TYPES, BIZJET_TYPES } from '@/utils/aircraftClassification'; +import { + classifyAircraft, + HELI_TYPES, + TURBOPROP_TYPES, + BIZJET_TYPES, +} from '@/utils/aircraftClassification'; describe('classifyAircraft', () => { // ─── Helicopter classification ──────────────────────────────────────────── diff --git a/frontend/src/__tests__/utils/identityBoundSensitiveStorage.test.ts b/frontend/src/__tests__/utils/identityBoundSensitiveStorage.test.ts new file mode 100644 index 00000000..24b2b219 --- /dev/null +++ b/frontend/src/__tests__/utils/identityBoundSensitiveStorage.test.ts @@ -0,0 +1,101 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const idbStore = new Map(); + +vi.mock('@/mesh/meshKeyStore', () => ({ + getKey: vi.fn(async (id: string) => idbStore.get(id) ?? null), + setKey: vi.fn(async (id: string, key: unknown) => { + idbStore.set(id, key); + }), + deleteKey: vi.fn(async (id: string) => { + idbStore.delete(id); + }), +})); + +function makeStorage() { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + }; +} + +function bufToBase64(buf: ArrayBuffer): string { + return btoa(String.fromCharCode(...new Uint8Array(buf))); +} + +async function provisionLocalIdentity(): Promise { + const meshIdentity = await import('@/mesh/meshIdentity'); + localStorage.setItem('sb_mesh_pubkey', 'test-pub'); + localStorage.setItem('sb_mesh_node_id', '!sb_sensitive123456'); + localStorage.setItem('sb_mesh_sovereignty_accepted', 'true'); + const keyPair = (await crypto.subtle.generateKey( + { name: 'ECDH', namedCurve: 'P-256' }, + false, + ['deriveKey', 'deriveBits'], + )) as CryptoKeyPair; + const publicRaw = await crypto.subtle.exportKey('raw', keyPair.publicKey); + localStorage.setItem('sb_mesh_dh_pubkey', bufToBase64(publicRaw)); + localStorage.setItem('sb_mesh_dh_algo', 'ECDH'); + idbStore.set('sb_mesh_dh_priv', keyPair.privateKey); + meshIdentity.getNodeIdentity(); +} + +describe('identityBoundSensitiveStorage', () => { + beforeEach(() => { + vi.resetModules(); + idbStore.clear(); + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + }); + + it('stores encrypted values in sensitive storage and keeps them out of localStorage', async () => { + await provisionLocalIdentity(); + const storage = await import('@/lib/identityBoundSensitiveStorage'); + + await storage.persistIdentityBoundSensitiveValue( + 'sb_access_requests:test', + 'SB-ACCESS-REQUESTS-STORAGE-V1', + [{ sender_id: 'alice', ts: 1 }], + ); + + expect(String(sessionStorage.getItem('sb_access_requests:test') ?? '')).toMatch(/^enc:/); + expect(localStorage.getItem('sb_access_requests:test')).toBeNull(); + + const hydrated = await storage.loadIdentityBoundSensitiveValue( + 'sb_access_requests:test', + 'SB-ACCESS-REQUESTS-STORAGE-V1', + [], + ); + expect(hydrated).toEqual([{ sender_id: 'alice', ts: 1 }]); + }); + + it('migrates legacy plaintext sensitive values into encrypted session-backed storage', async () => { + await provisionLocalIdentity(); + const storage = await import('@/lib/identityBoundSensitiveStorage'); + + localStorage.setItem('sb_mesh_muted', JSON.stringify(['alice', 'bob'])); + + const hydrated = await storage.loadIdentityBoundSensitiveValue( + 'sb_mesh_muted:!sb_sensitive123456', + 'SB-MUTED-LIST-V1', + [], + { legacyKey: 'sb_mesh_muted' }, + ); + + expect(hydrated).toEqual(['alice', 'bob']); + expect(String(sessionStorage.getItem('sb_mesh_muted:!sb_sensitive123456') ?? '')).toMatch(/^enc:/); + expect(localStorage.getItem('sb_mesh_muted')).toBeNull(); + expect(sessionStorage.getItem('sb_mesh_muted')).toBeNull(); + }); +}); diff --git a/frontend/src/__tests__/utils/privacyBrowserStorage.test.ts b/frontend/src/__tests__/utils/privacyBrowserStorage.test.ts new file mode 100644 index 00000000..c05a9711 --- /dev/null +++ b/frontend/src/__tests__/utils/privacyBrowserStorage.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +function makeStorage() { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => void values.set(key, value), + removeItem: (key: string) => void values.delete(key), + clear: () => void values.clear(), + }; +} + +describe('privacyBrowserStorage', () => { + beforeEach(() => { + vi.resetModules(); + Object.defineProperty(globalThis, 'localStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + Object.defineProperty(globalThis, 'sessionStorage', { + value: makeStorage(), + configurable: true, + writable: true, + }); + }); + + it('stores sensitive items in sessionStorage by default', async () => { + const mod = await import('@/lib/privacyBrowserStorage'); + + mod.setSensitiveBrowserItem('secret-key', 'alpha'); + + expect(mod.getSensitiveBrowserStorageMode()).toBe('session'); + expect(sessionStorage.getItem('secret-key')).toBe('alpha'); + expect(localStorage.getItem('secret-key')).toBeNull(); + expect(mod.getSensitiveBrowserItem('secret-key')).toBe('alpha'); + }); + + it('stores privacy preferences in session storage when session mode is enabled', async () => { + const mod = await import('@/lib/privacyBrowserStorage'); + + mod.setSessionModePreference(true); + mod.setPrivacyStrictPreference(true, { sessionMode: true }); + mod.setPrivacyProfilePreference('high', { sessionMode: true }); + + expect(mod.getSessionModePreference()).toBe(true); + expect(mod.getPrivacyStrictPreference()).toBe(true); + expect(mod.getPrivacyProfilePreference()).toBe('high'); + expect(sessionStorage.getItem('sb_mesh_session_mode')).toBe('true'); + expect(sessionStorage.getItem('sb_privacy_strict')).toBe('true'); + expect(sessionStorage.getItem('sb_privacy_profile')).toBe('high'); + expect(localStorage.getItem('sb_mesh_session_mode')).toBeNull(); + expect(localStorage.getItem('sb_privacy_strict')).toBeNull(); + expect(localStorage.getItem('sb_privacy_profile')).toBeNull(); + }); + + it('persists session mode locally only when the user explicitly disables it', async () => { + const mod = await import('@/lib/privacyBrowserStorage'); + + mod.setSessionModePreference(false); + + expect(mod.getSessionModePreference()).toBe(false); + expect(localStorage.getItem('sb_mesh_session_mode')).toBe('false'); + expect(sessionStorage.getItem('sb_mesh_session_mode')).toBeNull(); + }); + + it('stores sensitive items in sessionStorage when privacy strict is enabled', async () => { + localStorage.setItem('sb_privacy_strict', 'true'); + const mod = await import('@/lib/privacyBrowserStorage'); + + mod.setSensitiveBrowserItem('secret-key', 'bravo'); + + expect(mod.getSensitiveBrowserStorageMode()).toBe('session'); + expect(sessionStorage.getItem('secret-key')).toBe('bravo'); + expect(localStorage.getItem('secret-key')).toBeNull(); + }); + + it('migrates legacy localStorage values into sessionStorage in strict mode', async () => { + localStorage.setItem('sb_privacy_strict', 'true'); + localStorage.setItem('secret-key', 'charlie'); + const mod = await import('@/lib/privacyBrowserStorage'); + + expect(mod.getSensitiveBrowserItem('secret-key')).toBe('charlie'); + expect(sessionStorage.getItem('secret-key')).toBe('charlie'); + expect(localStorage.getItem('secret-key')).toBeNull(); + }); +}); diff --git a/frontend/src/__tests__/utils/solarTerminator.test.ts b/frontend/src/__tests__/utils/solarTerminator.test.ts index 2996269d..b52988cf 100644 --- a/frontend/src/__tests__/utils/solarTerminator.test.ts +++ b/frontend/src/__tests__/utils/solarTerminator.test.ts @@ -70,7 +70,8 @@ describe('computeNightPolygon', () => { .filter(([lng]: number[]) => lng >= -180 && lng <= 180) .slice(0, 361) .map(([, lat]: number[]) => lat); - const avgLat = terminatorLats.reduce((a: number, b: number) => a + b, 0) / terminatorLats.length; + const avgLat = + terminatorLats.reduce((a: number, b: number) => a + b, 0) / terminatorLats.length; expect(avgLat).toBeLessThan(15); }); diff --git a/frontend/src/__tests__/utils/viewportPrivacy.test.ts b/frontend/src/__tests__/utils/viewportPrivacy.test.ts new file mode 100644 index 00000000..c5be8a24 --- /dev/null +++ b/frontend/src/__tests__/utils/viewportPrivacy.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from 'vitest'; + +import { + buildBoundsQuery, + coarsenViewBounds, + expandBoundsToRadius, +} from '@/lib/viewportPrivacy'; + +describe('viewport privacy helper', () => { + it('coarsens narrow bounds outward without clipping the original view', () => { + const original = { + south: 33.612, + west: -84.452, + north: 33.781, + east: -84.211, + }; + + const coarse = coarsenViewBounds(original); + + expect(coarse.south).toBeLessThanOrEqual(original.south); + expect(coarse.west).toBeLessThanOrEqual(original.west); + expect(coarse.north).toBeGreaterThanOrEqual(original.north); + expect(coarse.east).toBeGreaterThanOrEqual(original.east); + expect(coarse.south).toBe(33.6); + expect(coarse.west).toBe(-84.5); + expect(coarse.north).toBe(33.8); + expect(coarse.east).toBe(-84.2); + }); + + it('canonicalizes the bounds query so nearby pans in the same coarse cell dedupe', () => { + const a = buildBoundsQuery({ + south: 47.6011, + west: -122.3484, + north: 47.6902, + east: -122.2012, + }); + const b = buildBoundsQuery({ + south: 47.6039, + west: -122.3441, + north: 47.6883, + east: -122.2051, + }); + + expect(a).toBe('?s=47.60&w=-122.35&n=47.70&e=-122.20'); + expect(b).toBe(a); + }); + + it('expands bounds to a fixed preload radius around the current view center', () => { + const original = { + south: 39.55, + west: -105.25, + north: 39.95, + east: -104.75, + }; + + const expanded = expandBoundsToRadius(original, 3000); + + expect(expanded.south).toBeLessThanOrEqual(original.south); + expect(expanded.west).toBeLessThanOrEqual(original.west); + expect(expanded.north).toBeGreaterThanOrEqual(original.north); + expect(expanded.east).toBeGreaterThanOrEqual(original.east); + expect(expanded.north - expanded.south).toBeGreaterThan(80); + expect(expanded.east - expanded.west).toBeGreaterThan(90); + }); +}); diff --git a/frontend/src/app/api/[...path]/route.ts b/frontend/src/app/api/[...path]/route.ts index ce4edd4f..c64769b9 100644 --- a/frontend/src/app/api/[...path]/route.ts +++ b/frontend/src/app/api/[...path]/route.ts @@ -5,16 +5,27 @@ * the client bundle or the build manifest. * * Set BACKEND_URL in docker-compose `environment:` (e.g. http://backend:8000) - * to use Docker internal networking. Defaults to http://localhost:8000 for + * to use Docker internal networking. Defaults to http://127.0.0.1:8000 for * local development where both services run on the same host. */ -import { NextRequest, NextResponse } from "next/server"; +import { NextRequest, NextResponse } from 'next/server'; +import fs from 'fs'; +import path from 'path'; +import { resolveAdminSessionToken } from '@/lib/server/adminSessionStore'; // Headers that must not be forwarded to the backend. const STRIP_REQUEST = new Set([ - "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", - "te", "trailers", "transfer-encoding", "upgrade", "host", + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'x-admin-key', + 'te', + 'trailers', + 'transfer-encoding', + 'upgrade', + 'host', ]); // Headers that must not be forwarded back to the browser. @@ -22,60 +33,222 @@ const STRIP_REQUEST = new Set([ // automatically decompresses gzip/br responses — forwarding these headers // would cause ERR_CONTENT_DECODING_FAILED in the browser. const STRIP_RESPONSE = new Set([ - "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", - "te", "trailers", "transfer-encoding", "upgrade", - "content-encoding", "content-length", + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailers', + 'transfer-encoding', + 'upgrade', + 'content-encoding', + 'content-length', ]); -async function proxy(req: NextRequest, path: string[]): Promise { - const backendUrl = process.env.BACKEND_URL ?? "http://localhost:8000"; - const targetUrl = new URL(`/api/${path.join("/")}`, backendUrl); - targetUrl.search = req.nextUrl.search; +const ADMIN_COOKIE = 'sb_admin_session'; +const NO_STORE_PROXY_HEADERS = { + 'Cache-Control': 'no-store, max-age=0', + Pragma: 'no-cache', +}; - // Forward relevant request headers - const forwardHeaders = new Headers(); - req.headers.forEach((value, key) => { - if (!STRIP_REQUEST.has(key.toLowerCase())) { - forwardHeaders.set(key, value); - } - }); +function isSensitiveProxyPath(pathSegments: string[]): boolean { + const joined = pathSegments.join('/'); + if (!joined) return false; + if (pathSegments[0] === 'wormhole') return true; + if (joined === 'refresh') return true; + if (joined === 'debug-latest') return true; + if (joined === 'system/update') return true; + if (pathSegments[0] === 'settings') return true; + if (joined === 'mesh/infonet/ingest') return true; + return false; +} - const isBodyless = req.method === "GET" || req.method === "HEAD"; - let upstream: Response; +async function proxy(req: NextRequest, pathSegments: string[]): Promise { try { - upstream = await fetch(targetUrl.toString(), { + const isMesh = pathSegments[0] === 'mesh'; + const meshSegments = pathSegments.slice(1); + const isSensitiveMeshPath = isMesh && meshSegments[0] === 'dm'; + const isAnonymousMeshWritePath = + isMesh && + !isSensitiveMeshPath && + ['POST', 'PUT', 'DELETE'].includes(req.method.toUpperCase()) && + (meshSegments.join('/') === 'send' || + meshSegments.join('/') === 'vote' || + meshSegments.join('/') === 'report' || + meshSegments.join('/') === 'gate/create' || + (meshSegments[0] === 'gate' && meshSegments[2] === 'message') || + meshSegments.join('/') === 'oracle/predict' || + meshSegments.join('/') === 'oracle/resolve' || + meshSegments.join('/') === 'oracle/stake' || + meshSegments.join('/') === 'oracle/resolve-stakes'); + const backendUrl = process.env.BACKEND_URL ?? 'http://127.0.0.1:8000'; + let targetBase = backendUrl; + + if (isMesh) { + const envEnabled = (process.env.WORMHOLE_ENABLED || '').toLowerCase(); + let wormholeEnabled = ['1', 'true', 'yes'].includes(envEnabled); + let privacyProfile = (process.env.WORMHOLE_PRIVACY_PROFILE || '').toLowerCase(); + let anonymousMode = ['1', 'true', 'yes'].includes( + (process.env.WORMHOLE_ANONYMOUS_MODE || '').toLowerCase(), + ); + let wormholeReady = false; + let effectiveTransport = ''; + + if (!wormholeEnabled || !privacyProfile || !anonymousMode) { + try { + const cwd = process.cwd(); + const repoRoot = cwd.endsWith(path.sep + 'frontend') ? path.resolve(cwd, '..') : cwd; + const wormholeFile = path.join(repoRoot, 'backend', 'data', 'wormhole.json'); + if (fs.existsSync(wormholeFile)) { + const raw = fs.readFileSync(wormholeFile, 'utf8'); + const data = JSON.parse(raw); + if (!wormholeEnabled) { + wormholeEnabled = Boolean(data && data.enabled); + } + privacyProfile = privacyProfile || String(data?.privacy_profile || '').toLowerCase(); + if (!anonymousMode) { + anonymousMode = Boolean(data?.anonymous_mode); + } + } + const wormholeStatusFile = path.join(repoRoot, 'backend', 'data', 'wormhole_status.json'); + if (fs.existsSync(wormholeStatusFile)) { + const raw = fs.readFileSync(wormholeStatusFile, 'utf8'); + const data = JSON.parse(raw); + wormholeReady = Boolean(data?.running) && Boolean(data?.ready); + effectiveTransport = String(data?.transport_active || data?.transport || '').toLowerCase(); + } + } catch { + wormholeEnabled = false; + } + } + + if (privacyProfile === 'high' && !wormholeEnabled && isSensitiveMeshPath) { + return new NextResponse( + JSON.stringify({ + ok: false, + detail: 'High privacy requires Wormhole. Enable it in Settings and restart.', + }), + { status: 428, headers: { 'Content-Type': 'application/json' } }, + ); + } + + if (wormholeEnabled && isSensitiveMeshPath) { + if (!wormholeReady) { + return new NextResponse( + JSON.stringify({ + ok: false, + detail: 'Wormhole is enabled but not connected yet. Start Wormhole to use secure DM features.', + }), + { status: 503, headers: { 'Content-Type': 'application/json' } }, + ); + } + targetBase = process.env.WORMHOLE_URL ?? 'http://127.0.0.1:8787'; + } + + if (anonymousMode && isAnonymousMeshWritePath) { + if (!wormholeEnabled) { + return new NextResponse( + JSON.stringify({ + ok: false, + detail: 'Anonymous mode requires Wormhole to be enabled before public posting.', + }), + { status: 428, headers: { 'Content-Type': 'application/json' } }, + ); + } + const hiddenReady = wormholeReady && ['tor', 'i2p', 'mixnet'].includes(effectiveTransport); + if (!hiddenReady) { + return new NextResponse( + JSON.stringify({ + ok: false, + detail: 'Anonymous mode requires Wormhole hidden transport (Tor/I2P/Mixnet) to be ready.', + }), + { status: 428, headers: { 'Content-Type': 'application/json' } }, + ); + } + targetBase = process.env.WORMHOLE_URL ?? 'http://127.0.0.1:8787'; + } + } + + const targetUrl = new URL(`/api/${pathSegments.join('/')}`, targetBase); + targetUrl.search = req.nextUrl.search; + + const forwardHeaders = new Headers(); + req.headers.forEach((value, key) => { + if (!STRIP_REQUEST.has(key.toLowerCase())) { + forwardHeaders.set(key, value); + } + }); + if (isSensitiveProxyPath(pathSegments)) { + const cookieToken = req.cookies.get(ADMIN_COOKIE)?.value || ''; + const injectedAdmin = process.env.ADMIN_KEY || resolveAdminSessionToken(cookieToken) || ''; + if (injectedAdmin) { + forwardHeaders.set('X-Admin-Key', injectedAdmin); + } + } + + const isBodyless = req.method === 'GET' || req.method === 'HEAD'; + let upstream: Response; + const requestInit: RequestInit & { duplex?: 'half' } = { method: req.method, headers: forwardHeaders, - body: isBodyless ? undefined : req.body, + cache: 'no-store', + }; + if (!isBodyless) { + requestInit.body = req.body; // Required for streaming request bodies in Node.js fetch - // @ts-ignore - duplex: "half", - }); - } catch (err) { - // Backend unreachable — return a clean 502 so the UI can handle it gracefully - return new NextResponse(JSON.stringify({ error: "Backend unavailable" }), { - status: 502, - headers: { "Content-Type": "application/json" }, + requestInit.duplex = 'half'; + } + try { + upstream = await fetch(targetUrl.toString(), requestInit); + } catch { + return new NextResponse(JSON.stringify({ error: 'Backend unavailable' }), { + status: 502, + headers: { 'Content-Type': 'application/json' }, + }); + } + + const responseHeaders = new Headers(); + upstream.headers.forEach((value, key) => { + if (!STRIP_RESPONSE.has(key.toLowerCase())) { + responseHeaders.set(key, value); + } }); - } + if (isSensitiveProxyPath(pathSegments) || isSensitiveMeshPath) { + Object.entries(NO_STORE_PROXY_HEADERS).forEach(([key, value]) => { + responseHeaders.set(key, value); + }); + } - // Forward response headers - const responseHeaders = new Headers(); - upstream.headers.forEach((value, key) => { - if (!STRIP_RESPONSE.has(key.toLowerCase())) { - responseHeaders.set(key, value); + if (upstream.status === 304) { + return new NextResponse(null, { status: 304, headers: responseHeaders }); } - }); - // 304 responses must have no body - if (upstream.status === 304) { - return new NextResponse(null, { status: 304, headers: responseHeaders }); + // Stream the upstream body directly instead of buffering the full response. + // This reduces TTFB and memory pressure for large payloads (flights, ships). + return new NextResponse(upstream.body, { + status: upstream.status, + headers: responseHeaders, + }); + } catch (error) { + console.error('api proxy unexpected error', { + pathSegments, + method: req.method, + error, + }); + return new NextResponse( + JSON.stringify({ + error: 'Proxy failed', + detail: error instanceof Error ? error.message : 'unknown_error', + }), + { + status: 500, + headers: { + 'Content-Type': 'application/json', + ...NO_STORE_PROXY_HEADERS, + }, + }, + ); } - - return new NextResponse(upstream.body, { - status: upstream.status, - headers: responseHeaders, - }); } export async function GET(req: NextRequest, { params }: { params: Promise<{ path: string[] }> }) { @@ -90,6 +263,9 @@ export async function PUT(req: NextRequest, { params }: { params: Promise<{ path return proxy(req, (await params).path); } -export async function DELETE(req: NextRequest, { params }: { params: Promise<{ path: string[] }> }) { +export async function DELETE( + req: NextRequest, + { params }: { params: Promise<{ path: string[] }> }, +) { return proxy(req, (await params).path); } diff --git a/frontend/src/app/api/admin/session/route.ts b/frontend/src/app/api/admin/session/route.ts new file mode 100644 index 00000000..39488548 --- /dev/null +++ b/frontend/src/app/api/admin/session/route.ts @@ -0,0 +1,106 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { + clearAdminSessionToken, + createAdminSessionToken, + hasAdminSessionToken, +} from '@/lib/server/adminSessionStore'; + +const COOKIE_NAME = 'sb_admin_session'; +const COOKIE_MAX_AGE = 60 * 60 * 8; +const NO_STORE_HEADERS = { + 'Cache-Control': 'no-store, max-age=0', + Pragma: 'no-cache', +}; + +function cookieOptions() { + return { + httpOnly: true, + sameSite: 'strict' as const, + secure: process.env.NODE_ENV === 'production', + path: '/', + maxAge: COOKIE_MAX_AGE, + }; +} + +async function verifyAdminKey(adminKey: string): Promise<{ ok: true } | { ok: false; detail: string }> { + const backendUrl = process.env.BACKEND_URL ?? 'http://127.0.0.1:8000'; + const verifyAgainstBackend = async (): Promise< + { ok: true } | { ok: false; detail: string } + > => { + try { + const res = await fetch(`${backendUrl}/api/settings/privacy-profile`, { + method: 'GET', + headers: { 'X-Admin-Key': adminKey }, + cache: 'no-store', + }); + if (res.ok) return { ok: true }; + const data = await res.json().catch(() => ({})); + return { + ok: false, + detail: String(data?.detail || data?.message || 'Unable to verify admin key'), + }; + } catch { + return { + ok: false, + detail: 'Unable to verify admin key against backend', + }; + } + }; + + const configuredAdmin = String(process.env.ADMIN_KEY || '').trim(); + if (configuredAdmin) { + if (adminKey !== configuredAdmin) { + return { ok: false, detail: 'Invalid admin key' }; + } + return verifyAgainstBackend(); + } + + return verifyAgainstBackend(); +} + +export async function POST(req: NextRequest) { + const body = await req.json().catch(() => ({})); + const adminKey = String(body?.adminKey || '').trim(); + if (!adminKey) { + return NextResponse.json( + { ok: false, detail: 'Missing admin key' }, + { status: 400, headers: NO_STORE_HEADERS }, + ); + } + const verification = await verifyAdminKey(adminKey); + if (!verification.ok) { + return NextResponse.json( + { ok: false, detail: verification.detail }, + { status: 403, headers: NO_STORE_HEADERS }, + ); + } + const existingToken = req.cookies.get(COOKIE_NAME)?.value || ''; + if (existingToken) { + clearAdminSessionToken(existingToken); + } + const sessionToken = createAdminSessionToken(adminKey, COOKIE_MAX_AGE); + const res = NextResponse.json({ ok: true }, { headers: NO_STORE_HEADERS }); + res.cookies.set(COOKIE_NAME, sessionToken, cookieOptions()); + return res; +} + +export async function DELETE(req: NextRequest) { + const existingToken = req.cookies.get(COOKIE_NAME)?.value || ''; + if (existingToken) { + clearAdminSessionToken(existingToken); + } + const res = NextResponse.json({ ok: true }, { headers: NO_STORE_HEADERS }); + res.cookies.set(COOKIE_NAME, '', { + ...cookieOptions(), + maxAge: 0, + }); + return res; +} + +export async function GET(req: NextRequest) { + const token = req.cookies.get(COOKIE_NAME)?.value || ''; + return NextResponse.json( + { ok: true, hasSession: hasAdminSessionToken(token) }, + { headers: NO_STORE_HEADERS }, + ); +} diff --git a/frontend/src/app/globals.css b/frontend/src/app/globals.css index 44c90132..0a666998 100644 --- a/frontend/src/app/globals.css +++ b/frontend/src/app/globals.css @@ -1,4 +1,4 @@ -@import "tailwindcss"; +@import 'tailwindcss'; :root { --background: #000000; @@ -7,47 +7,75 @@ --bg-secondary: rgb(5, 5, 8); --bg-tertiary: rgb(12, 12, 16); --bg-panel: rgba(0, 0, 0, 0.85); - --border-primary: rgb(10, 12, 15); - --border-secondary: rgb(20, 24, 28); + --border-primary: rgba(8, 145, 178, 0.18); + --border-secondary: rgba(8, 145, 178, 0.30); + --border-glow: rgba(6, 182, 212, 0.12); --text-primary: rgb(243, 244, 246); --text-secondary: rgb(34, 211, 238); --text-muted: rgb(8, 145, 178); --text-heading: rgb(236, 254, 255); --hover-accent: rgba(8, 51, 68, 0.2); - --scrollbar-thumb: rgba(8, 145, 178, 0.3); - --scrollbar-thumb-hover: rgba(8, 145, 178, 0.5); + --scrollbar-thumb: rgba(255, 255, 255, 0.12); + --scrollbar-thumb-hover: rgba(255, 255, 255, 0.25); + --font-geist-sans: + ui-sans-serif, system-ui, -apple-system, 'Segoe UI', 'Helvetica Neue', Arial, 'Noto Sans', + sans-serif; + --font-roboto-mono: + ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', 'Courier New', + monospace; } /* Light theme: only the map basemap changes — UI stays dark */ -[data-theme="light"] { +[data-theme='light'] { --background: #000000; --foreground: #ededed; --bg-primary: #000000; --bg-secondary: rgb(5, 5, 8); --bg-tertiary: rgb(12, 12, 16); --bg-panel: rgba(0, 0, 0, 0.85); - --border-primary: rgb(10, 12, 15); - --border-secondary: rgb(20, 24, 28); + --border-primary: rgba(8, 145, 178, 0.18); + --border-secondary: rgba(8, 145, 178, 0.30); + --border-glow: rgba(6, 182, 212, 0.12); --text-primary: rgb(243, 244, 246); --text-secondary: rgb(34, 211, 238); --text-muted: rgb(8, 145, 178); --text-heading: rgb(236, 254, 255); --hover-accent: rgba(8, 51, 68, 0.2); - --scrollbar-thumb: rgba(8, 145, 178, 0.3); - --scrollbar-thumb-hover: rgba(8, 145, 178, 0.5); + --scrollbar-thumb: rgba(255, 255, 255, 0.12); + --scrollbar-thumb-hover: rgba(255, 255, 255, 0.25); } @theme inline { --color-background: var(--background); --color-foreground: var(--foreground); --font-sans: var(--font-geist-sans); - --font-mono: var(--font-geist-mono); + --font-mono: var(--font-roboto-mono); } body { background: var(--background); color: var(--foreground); - font-family: Arial, Helvetica, sans-serif; + font-family: var(--font-roboto-mono), 'Roboto Mono', monospace; +} + +/* Global interactive cursor hints */ +button:not(:disabled), +[role='button']:not([aria-disabled='true']), +a[href], +summary, +label[for], +input[type='button']:not(:disabled), +input[type='submit']:not(:disabled), +input[type='reset']:not(:disabled) { + cursor: pointer; +} + +button:disabled, +[role='button'][aria-disabled='true'], +input:disabled, +select:disabled, +textarea:disabled { + cursor: not-allowed; } /* Styled thin scrollbar for dark HUD UI */ @@ -70,15 +98,49 @@ body { .styled-scrollbar { scrollbar-width: thin; + scrollbar-color: var(--scrollbar-thumb) transparent; +} + +/* DOS block cursor blink */ +@keyframes blink { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0; + } +} + +/* ── TERMINAL UTILITY CLASSES ── */ + +/* Subtle text glow for cyan headings */ +.text-glow { + text-shadow: 0 0 8px rgba(34, 211, 238, 0.3); +} + +/* Terminal input — prompt style */ +.terminal-input { + border-radius: 0; + border: 1px solid rgba(8, 145, 178, 0.25); + background: rgba(0, 0, 0, 0.4); +} +.terminal-input:focus { + border-color: rgba(34, 211, 238, 0.5); + box-shadow: 0 0 6px rgba(34, 211, 238, 0.15); + outline: none; } /* Map popup shared utilities */ .map-popup { background: rgba(10, 14, 26, 0.95); - border-radius: 6px; + border-radius: 2px; + border: 1px solid rgba(8, 145, 178, 0.25); padding: 10px 14px; color: #e0e6f0; - font-family: monospace; + font-family: + var(--font-roboto-mono), 'Roboto Mono', monospace, 'Microsoft YaHei', 'PingFang SC', + 'Noto Sans SC', 'Noto Sans JP', 'Noto Sans KR', sans-serif; font-size: 11px; min-width: 220px; max-width: 320px; @@ -116,77 +178,181 @@ body { /* ── MATRIX HUD COLOR THEME ── */ /* Remaps cyan accents → green within .hud-zone containers only */ -[data-hud="matrix"] .hud-zone { +[data-hud='matrix'] .hud-zone { --text-secondary: #4ade80; --text-muted: #16a34a; --text-heading: #bbf7d0; --hover-accent: rgba(5, 46, 22, 0.2); - --scrollbar-thumb: rgba(22, 163, 74, 0.3); - --scrollbar-thumb-hover: rgba(22, 163, 74, 0.5); + --scrollbar-thumb: rgba(255, 255, 255, 0.12); + --scrollbar-thumb-hover: rgba(255, 255, 255, 0.25); + --border-primary: rgba(22, 163, 74, 0.18); + --border-secondary: rgba(22, 163, 74, 0.30); + --border-glow: rgba(34, 197, 94, 0.12); +} +[data-hud='matrix'] .hud-zone .text-glow { + text-shadow: 0 0 8px rgba(74, 222, 128, 0.3); } /* --- Text color overrides --- */ -[data-hud="matrix"] .hud-zone .text-cyan-300 { color: #86efac !important; } -[data-hud="matrix"] .hud-zone .text-cyan-400 { color: #4ade80 !important; } -[data-hud="matrix"] .hud-zone .text-cyan-500 { color: #22c55e !important; } -[data-hud="matrix"] .hud-zone .text-cyan-600 { color: #16a34a !important; } -[data-hud="matrix"] .hud-zone .text-cyan-700 { color: #15803d !important; } -[data-hud="matrix"] .hud-zone .text-cyan-500\/50 { color: rgba(34, 197, 94, 0.5) !important; } -[data-hud="matrix"] .hud-zone .text-cyan-500\/70 { color: rgba(34, 197, 94, 0.7) !important; } -[data-hud="matrix"] .hud-zone .text-cyan-500\/80 { color: rgba(34, 197, 94, 0.8) !important; } +[data-hud='matrix'] .hud-zone .text-cyan-300 { + color: #86efac !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-400 { + color: #4ade80 !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-500 { + color: #22c55e !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-600 { + color: #16a34a !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-700 { + color: #15803d !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-500\/50 { + color: rgba(34, 197, 94, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-500\/70 { + color: rgba(34, 197, 94, 0.7) !important; +} +[data-hud='matrix'] .hud-zone .text-cyan-500\/80 { + color: rgba(34, 197, 94, 0.8) !important; +} /* --- Background color overrides --- */ -[data-hud="matrix"] .hud-zone .bg-cyan-400 { background-color: #4ade80 !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-300 { background-color: #86efac !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-500 { background-color: #22c55e !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-500\/10 { background-color: rgba(34, 197, 94, 0.1) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-500\/20 { background-color: rgba(34, 197, 94, 0.2) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-500\/30 { background-color: rgba(34, 197, 94, 0.3) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-900\/30 { background-color: rgba(20, 83, 45, 0.3) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-900\/50 { background-color: rgba(20, 83, 45, 0.5) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-900\/60 { background-color: rgba(20, 83, 45, 0.6) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-950\/10 { background-color: rgba(5, 46, 22, 0.1) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-950\/30 { background-color: rgba(5, 46, 22, 0.3) !important; } -[data-hud="matrix"] .hud-zone .bg-cyan-950\/40 { background-color: rgba(5, 46, 22, 0.4) !important; } +[data-hud='matrix'] .hud-zone .bg-cyan-400 { + background-color: #4ade80 !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-300 { + background-color: #86efac !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-500 { + background-color: #22c55e !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-500\/10 { + background-color: rgba(34, 197, 94, 0.1) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-500\/20 { + background-color: rgba(34, 197, 94, 0.2) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-500\/30 { + background-color: rgba(34, 197, 94, 0.3) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-900\/30 { + background-color: rgba(20, 83, 45, 0.3) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-900\/50 { + background-color: rgba(20, 83, 45, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-900\/60 { + background-color: rgba(20, 83, 45, 0.6) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-950\/10 { + background-color: rgba(5, 46, 22, 0.1) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-950\/30 { + background-color: rgba(5, 46, 22, 0.3) !important; +} +[data-hud='matrix'] .hud-zone .bg-cyan-950\/40 { + background-color: rgba(5, 46, 22, 0.4) !important; +} /* --- Border color overrides --- */ -[data-hud="matrix"] .hud-zone .border-cyan-400 { border-color: #4ade80 !important; } -[data-hud="matrix"] .hud-zone .border-cyan-500 { border-color: #22c55e !important; } -[data-hud="matrix"] .hud-zone .border-cyan-700 { border-color: #15803d !important; } -[data-hud="matrix"] .hud-zone .border-cyan-800 { border-color: #166534 !important; } -[data-hud="matrix"] .hud-zone .border-cyan-900 { border-color: #14532d !important; } -[data-hud="matrix"] .hud-zone .border-cyan-500\/10 { border-color: rgba(34, 197, 94, 0.1) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-500\/20 { border-color: rgba(34, 197, 94, 0.2) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-500\/30 { border-color: rgba(34, 197, 94, 0.3) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-500\/40 { border-color: rgba(34, 197, 94, 0.4) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-500\/50 { border-color: rgba(34, 197, 94, 0.5) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-800\/40 { border-color: rgba(22, 101, 52, 0.4) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-800\/50 { border-color: rgba(22, 101, 52, 0.5) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-800\/60 { border-color: rgba(22, 101, 52, 0.6) !important; } -[data-hud="matrix"] .hud-zone .border-cyan-900\/50 { border-color: rgba(20, 83, 45, 0.5) !important; } -[data-hud="matrix"] .hud-zone .border-b-cyan-900 { border-bottom-color: #14532d !important; } -[data-hud="matrix"] .hud-zone .border-l-cyan-500 { border-left-color: #22c55e !important; } +[data-hud='matrix'] .hud-zone .border-cyan-400 { + border-color: #4ade80 !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-500 { + border-color: #22c55e !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-700 { + border-color: #15803d !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-800 { + border-color: #166534 !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-900 { + border-color: #14532d !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-500\/10 { + border-color: rgba(34, 197, 94, 0.1) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-500\/20 { + border-color: rgba(34, 197, 94, 0.2) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-500\/30 { + border-color: rgba(34, 197, 94, 0.3) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-500\/40 { + border-color: rgba(34, 197, 94, 0.4) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-500\/50 { + border-color: rgba(34, 197, 94, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-800\/40 { + border-color: rgba(22, 101, 52, 0.4) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-800\/50 { + border-color: rgba(22, 101, 52, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-800\/60 { + border-color: rgba(22, 101, 52, 0.6) !important; +} +[data-hud='matrix'] .hud-zone .border-cyan-900\/50 { + border-color: rgba(20, 83, 45, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .border-b-cyan-900 { + border-bottom-color: #14532d !important; +} +[data-hud='matrix'] .hud-zone .border-l-cyan-500 { + border-left-color: #22c55e !important; +} /* --- Hover text --- */ -[data-hud="matrix"] .hud-zone .hover\:text-cyan-300:hover { color: #86efac !important; } -[data-hud="matrix"] .hud-zone .hover\:text-cyan-400:hover { color: #4ade80 !important; } +[data-hud='matrix'] .hud-zone .hover\:text-cyan-300:hover { + color: #86efac !important; +} +[data-hud='matrix'] .hud-zone .hover\:text-cyan-400:hover { + color: #4ade80 !important; +} /* --- Hover background --- */ -[data-hud="matrix"] .hud-zone .hover\:bg-cyan-300:hover { background-color: #86efac !important; } -[data-hud="matrix"] .hud-zone .hover\:bg-cyan-500\/20:hover { background-color: rgba(34, 197, 94, 0.2) !important; } -[data-hud="matrix"] .hud-zone .hover\:bg-cyan-900\/50:hover { background-color: rgba(20, 83, 45, 0.5) !important; } -[data-hud="matrix"] .hud-zone .hover\:bg-cyan-950\/30:hover { background-color: rgba(5, 46, 22, 0.3) !important; } +[data-hud='matrix'] .hud-zone .hover\:bg-cyan-300:hover { + background-color: #86efac !important; +} +[data-hud='matrix'] .hud-zone .hover\:bg-cyan-500\/20:hover { + background-color: rgba(34, 197, 94, 0.2) !important; +} +[data-hud='matrix'] .hud-zone .hover\:bg-cyan-900\/50:hover { + background-color: rgba(20, 83, 45, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .hover\:bg-cyan-950\/30:hover { + background-color: rgba(5, 46, 22, 0.3) !important; +} /* --- Hover border --- */ -[data-hud="matrix"] .hud-zone .hover\:border-cyan-300:hover { border-color: #86efac !important; } -[data-hud="matrix"] .hud-zone .hover\:border-cyan-500:hover { border-color: #22c55e !important; } -[data-hud="matrix"] .hud-zone .hover\:border-cyan-500\/40:hover { border-color: rgba(34, 197, 94, 0.4) !important; } -[data-hud="matrix"] .hud-zone .hover\:border-cyan-500\/50:hover { border-color: rgba(34, 197, 94, 0.5) !important; } -[data-hud="matrix"] .hud-zone .hover\:border-cyan-600:hover { border-color: #16a34a !important; } -[data-hud="matrix"] .hud-zone .hover\:border-cyan-800:hover { border-color: #166534 !important; } +[data-hud='matrix'] .hud-zone .hover\:border-cyan-300:hover { + border-color: #86efac !important; +} +[data-hud='matrix'] .hud-zone .hover\:border-cyan-500:hover { + border-color: #22c55e !important; +} +[data-hud='matrix'] .hud-zone .hover\:border-cyan-500\/40:hover { + border-color: rgba(34, 197, 94, 0.4) !important; +} +[data-hud='matrix'] .hud-zone .hover\:border-cyan-500\/50:hover { + border-color: rgba(34, 197, 94, 0.5) !important; +} +[data-hud='matrix'] .hud-zone .hover\:border-cyan-600:hover { + border-color: #16a34a !important; +} +[data-hud='matrix'] .hud-zone .hover\:border-cyan-800:hover { + border-color: #166534 !important; +} /* --- Accent (range inputs) --- */ -[data-hud="matrix"] .hud-zone .accent-cyan-500 { accent-color: #22c55e !important; } +[data-hud='matrix'] .hud-zone .accent-cyan-500 { + accent-color: #22c55e !important; +} /* Focus mode: dim the map canvas (tiles + drawn layers) when a popup is active. Inside MapLibre's DOM, .maplibregl-canvas-container is a SIBLING of .maplibregl-popup, @@ -200,3 +366,94 @@ body { .map-focus-active .maplibregl-popup { z-index: 10 !important; } + +/* ── INFONET CRT TERMINAL EFFECTS ── */ + +.infonet-font { + font-family: 'JetBrains Mono', ui-monospace, SFMono-Regular, monospace; +} + +/* CRT scanline overlay — scoped to .crt containers only */ +.crt { + position: relative; + animation: crt-flicker 0.15s infinite; + text-shadow: 0 0 2px rgba(255, 255, 255, 0.2); +} + +.crt::before { + content: ' '; + display: block; + position: absolute; + top: 0; + left: 0; + bottom: 0; + right: 0; + background: + linear-gradient(rgba(18, 16, 16, 0) 50%, rgba(0, 0, 0, 0.25) 50%), + linear-gradient( + 90deg, + rgba(255, 0, 0, 0.06), + rgba(0, 255, 0, 0.02), + rgba(0, 0, 255, 0.06) + ); + z-index: 2; + background-size: 100% 2px, 3px 100%; + pointer-events: none; +} + +@keyframes crt-flicker { + 0% { + opacity: 0.95; + } + 5% { + opacity: 0.85; + } + 10% { + opacity: 0.95; + } + 15% { + opacity: 1; + } + 100% { + opacity: 1; + } +} + +@media (prefers-reduced-motion: reduce) { + .crt { + animation: none; + } + .crt::before { + display: none; + } +} + +/* Ticker animation for InfoNet */ +@keyframes infonet-ticker { + 0% { + transform: translateX(100%); + } + 100% { + transform: translateX(-100%); + } +} + +.animate-ticker { + display: inline-block; + white-space: nowrap; + animation: infonet-ticker 90s linear infinite; +} + +/* Scoped scrollbar for CRT terminal */ +.crt ::-webkit-scrollbar { + width: 8px; +} +.crt ::-webkit-scrollbar-track { + background: #0a0a0a; +} +.crt ::-webkit-scrollbar-thumb { + background: #4b5563; +} +.crt ::-webkit-scrollbar-thumb:hover { + background: #9ca3af; +} diff --git a/frontend/src/app/layout.tsx b/frontend/src/app/layout.tsx index a5075e2c..9f2254aa 100644 --- a/frontend/src/app/layout.tsx +++ b/frontend/src/app/layout.tsx @@ -1,21 +1,11 @@ -import type { Metadata } from "next"; -import { Geist, Geist_Mono } from "next/font/google"; -import { ThemeProvider } from "@/lib/ThemeContext"; -import "./globals.css"; - -const geistSans = Geist({ - variable: "--font-geist-sans", - subsets: ["latin"], -}); - -const geistMono = Geist_Mono({ - variable: "--font-geist-mono", - subsets: ["latin"], -}); +import type { Metadata } from 'next'; +import DesktopBridgeBootstrap from '@/components/DesktopBridgeBootstrap'; +import { ThemeProvider } from '@/lib/ThemeContext'; +import './globals.css'; export const metadata: Metadata = { - title: "WORLDVIEW // ORBITAL TRACKING", - description: "Advanced Geopolitical Risk Dashboard", + title: 'WORLDVIEW // ORBITAL TRACKING', + description: 'Advanced Geopolitical Risk Dashboard', }; export default function RootLayout({ @@ -25,12 +15,16 @@ export default function RootLayout({ }>) { return ( - - - {children} + + + + + + + + + {children} + ); diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index 2b147a69..55b73cde 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -1,49 +1,88 @@ -"use client"; +'use client'; -import { useEffect, useState, useRef } from "react"; +import { useEffect, useState, useRef, useCallback, useMemo } from 'react'; import dynamic from 'next/dynamic'; -import { motion } from "framer-motion"; -import { ChevronLeft, ChevronRight } from "lucide-react"; -import WorldviewLeftPanel from "@/components/WorldviewLeftPanel"; - -import NewsFeed from "@/components/NewsFeed"; -import MarketsPanel from "@/components/MarketsPanel"; -import FilterPanel from "@/components/FilterPanel"; -import FindLocateBar from "@/components/FindLocateBar"; -import TopRightControls from "@/components/TopRightControls"; -import RadioInterceptPanel from "@/components/RadioInterceptPanel"; -import SettingsPanel from "@/components/SettingsPanel"; -import MapLegend from "@/components/MapLegend"; -import ScaleBar from "@/components/ScaleBar"; -import ErrorBoundary from "@/components/ErrorBoundary"; -import { DashboardDataProvider } from "@/lib/DashboardDataContext"; -import OnboardingModal, { useOnboarding } from "@/components/OnboardingModal"; -import ChangelogModal, { useChangelog } from "@/components/ChangelogModal"; -import type { SelectedEntity } from "@/types/dashboard"; -import { NOMINATIM_DEBOUNCE_MS } from "@/lib/constants"; -import { useDataPolling } from "@/hooks/useDataPolling"; -import { useReverseGeocode } from "@/hooks/useReverseGeocode"; -import { useRegionDossier } from "@/hooks/useRegionDossier"; +import { motion } from 'framer-motion'; +import { ChevronLeft, ChevronRight, ChevronUp, ChevronDown } from 'lucide-react'; +import WorldviewLeftPanel from '@/components/WorldviewLeftPanel'; + +import NewsFeed from '@/components/NewsFeed'; +import MarketsPanel from '@/components/MarketsPanel'; +import FilterPanel from '@/components/FilterPanel'; +import FindLocateBar from '@/components/FindLocateBar'; +import TopRightControls from '@/components/TopRightControls'; +import PredictionsPanel from '@/components/PredictionsPanel'; +import SettingsPanel from '@/components/SettingsPanel'; +import MapLegend from '@/components/MapLegend'; +import ScaleBar from '@/components/ScaleBar'; +import MeshTerminal from '@/components/MeshTerminal'; +import MeshChat from '@/components/MeshChat'; +import InfonetTerminal from '@/components/InfonetTerminal'; +import { leaveWormhole, fetchWormholeState } from '@/mesh/wormholeClient'; +import ShodanPanel from '@/components/ShodanPanel'; +import GlobalTicker from '@/components/GlobalTicker'; +import ErrorBoundary from '@/components/ErrorBoundary'; +import OnboardingModal, { useOnboarding } from '@/components/OnboardingModal'; +import ChangelogModal, { useChangelog } from '@/components/ChangelogModal'; +import type { ActiveLayers, KiwiSDR, Scanner, SelectedEntity } from '@/types/dashboard'; +import type { ShodanSearchMatch } from '@/types/shodan'; +import { NOMINATIM_DEBOUNCE_MS } from '@/lib/constants'; +import { API_BASE } from '@/lib/api'; +import { useDataPolling, LAYER_TOGGLE_EVENT } from '@/hooks/useDataPolling'; +import { useBackendStatus, useDataKey } from '@/hooks/useDataStore'; +import { useReverseGeocode } from '@/hooks/useReverseGeocode'; +import { useRegionDossier } from '@/hooks/useRegionDossier'; +import { + requestSecureMeshTerminalLauncherOpen, + subscribeMeshTerminalOpen, +} from '@/lib/meshTerminalLauncher'; +import { + hasSentinelInfoBeenSeen, + markSentinelInfoSeen, + hasSentinelCredentials, + getSentinelUsage, +} from '@/lib/sentinelHub'; // Use dynamic loads for Maplibre to avoid SSR window is not defined errors const MaplibreViewer = dynamic(() => import('@/components/MaplibreViewer'), { ssr: false }); /* ── LOCATE BAR ── coordinate / place-name search above bottom status bar ── */ -function LocateBar({ onLocate }: { onLocate: (lat: number, lng: number) => void }) { +function LocateBar({ onLocate, onOpenChange }: { onLocate: (lat: number, lng: number) => void; onOpenChange?: (open: boolean) => void }) { const [open, setOpen] = useState(false); + + useEffect(() => { onOpenChange?.(open); }, [open]); const [value, setValue] = useState(''); const [results, setResults] = useState<{ label: string; lat: number; lng: number }[]>([]); const [loading, setLoading] = useState(false); const inputRef = useRef(null); const timerRef = useRef | null>(null); + const searchAbortRef = useRef(null); + const containerRef = useRef(null); + + useEffect(() => { + if (open) inputRef.current?.focus(); + }, [open]); - useEffect(() => { if (open) inputRef.current?.focus(); }, [open]); + // Close when clicking outside + useEffect(() => { + if (!open) return; + const handler = (e: MouseEvent) => { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { + setOpen(false); + setValue(''); + setResults([]); + } + }; + document.addEventListener('mousedown', handler); + return () => document.removeEventListener('mousedown', handler); + }, [open]); // Parse raw coordinate input: "31.8, 34.8" or "31.8 34.8" or "-12.3, 45.6" const parseCoords = (s: string): { lat: number; lng: number } | null => { const m = s.trim().match(/^([+-]?\d+\.?\d*)[,\s]+([+-]?\d+\.?\d*)$/); if (!m) return null; - const lat = parseFloat(m[1]), lng = parseFloat(m[2]); + const lat = parseFloat(m[1]), + lng = parseFloat(m[2]); if (lat >= -90 && lat <= 90 && lng >= -180 && lng <= 180) return { lat, lng }; return null; }; @@ -58,17 +97,72 @@ function LocateBar({ onLocate }: { onLocate: (lat: number, lng: number) => void } // Geocode with Nominatim (debounced) if (timerRef.current) clearTimeout(timerRef.current); - if (q.trim().length < 2) { setResults([]); return; } + if (searchAbortRef.current) searchAbortRef.current.abort(); + if (q.trim().length < 2) { + setResults([]); + return; + } timerRef.current = setTimeout(async () => { setLoading(true); + searchAbortRef.current = new AbortController(); + const signal = searchAbortRef.current.signal; try { - const res = await fetch(`https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(q)}&format=json&limit=5`, { - headers: { 'Accept-Language': 'en' }, - }); - const data = await res.json(); - setResults(data.map((r: { display_name: string; lat: string; lon: string }) => ({ label: r.display_name, lat: parseFloat(r.lat), lng: parseFloat(r.lon) }))); - } catch { setResults([]); } - setLoading(false); + // Try backend proxy first (has caching + rate-limit compliance) + const res = await fetch( + `${API_BASE}/api/geocode/search?q=${encodeURIComponent(q)}&limit=5`, + { signal }, + ); + if (res.ok) { + const data = await res.json(); + const mapped = (data?.results || []).map( + (r: { label: string; lat: number; lng: number }) => ({ + label: r.label, + lat: r.lat, + lng: r.lng, + }), + ); + setResults(mapped); + } else { + // Backend proxy returned an error — fall back to direct Nominatim + console.warn(`[Locate] Proxy returned HTTP ${res.status}, falling back to Nominatim`); + const directRes = await fetch( + `https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(q)}&format=json&limit=5`, + { headers: { 'Accept-Language': 'en' }, signal }, + ); + const data = await directRes.json(); + setResults( + data.map((r: { display_name: string; lat: string; lon: string }) => ({ + label: r.display_name, + lat: parseFloat(r.lat), + lng: parseFloat(r.lon), + })), + ); + } + } catch (err) { + if ((err as Error)?.name !== 'AbortError') { + // Proxy completely failed — try direct Nominatim as last resort + try { + const directRes = await fetch( + `https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(q)}&format=json&limit=5`, + { headers: { 'Accept-Language': 'en' } }, + ); + const data = await directRes.json(); + setResults( + data.map((r: { display_name: string; lat: string; lon: string }) => ({ + label: r.display_name, + lat: parseFloat(r.lat), + lng: parseFloat(r.lon), + })), + ); + } catch { + setResults([]); + } + } else { + setResults([]); + } + } finally { + setLoading(false); + } }, NOMINATIM_DEBOUNCE_MS); }; @@ -83,37 +177,113 @@ function LocateBar({ onLocate }: { onLocate: (lat: number, lng: number) => void return ( ); } return ( -
-
- +
+
+ + + + handleSearch(e.target.value)} - onKeyDown={(e) => { if (e.key === 'Escape') { setOpen(false); setValue(''); setResults([]); } if (e.key === 'Enter' && results.length > 0) handleSelect(results[0]); }} + onKeyDown={(e) => { + if (e.key === 'Escape') { + setOpen(false); + setValue(''); + setResults([]); + } + if (e.key === 'Enter' && results.length > 0) handleSelect(results[0]); + }} placeholder="Enter coordinates (31.8, 34.8) or place name..." - className="flex-1 bg-transparent text-[10px] text-[var(--text-primary)] font-mono tracking-wider outline-none placeholder:text-[var(--text-muted)]" + className="flex-1 bg-transparent text-[12px] text-[var(--text-primary)] font-mono tracking-wider outline-none placeholder:text-[var(--text-muted)]" /> - {loading &&
} -
{results.length > 0 && ( -
+
{results.map((r, i) => ( - ))}
@@ -123,22 +293,78 @@ function LocateBar({ onLocate }: { onLocate: (lat: number, lng: number) => void } export default function Dashboard() { - const { data, dataVersion, backendStatus } = useDataPolling(); + const viewBoundsRef = useRef<{ south: number; west: number; north: number; east: number } | null>(null); const { mouseCoords, locationLabel, handleMouseCoords } = useReverseGeocode(); const [selectedEntity, setSelectedEntity] = useState(null); - const [trackedSdr, setTrackedSdr] = useState(null); - const { regionDossier, regionDossierLoading, handleMapRightClick } = useRegionDossier(selectedEntity, setSelectedEntity); + const [trackedSdr, setTrackedSdr] = useState(null); + const [trackedScanner, setTrackedScanner] = useState(null); + const { regionDossier, regionDossierLoading, handleMapRightClick } = useRegionDossier( + selectedEntity, + setSelectedEntity, + ); const [uiVisible, setUiVisible] = useState(true); const [leftOpen, setLeftOpen] = useState(true); const [rightOpen, setRightOpen] = useState(true); + const [tickerOpen, setTickerOpen] = useState(true); + + // Persist UI panel states + useEffect(() => { + const l = localStorage.getItem('sb_left_open'); + const r = localStorage.getItem('sb_right_open'); + const t = localStorage.getItem('sb_ticker_open'); + if (l !== null) setLeftOpen(l === 'true'); + if (r !== null) setRightOpen(r === 'true'); + if (t !== null) setTickerOpen(t === 'true'); + }, []); + + useEffect(() => { + localStorage.setItem('sb_left_open', leftOpen.toString()); + }, [leftOpen]); + + useEffect(() => { + localStorage.setItem('sb_right_open', rightOpen.toString()); + }, [rightOpen]); + + useEffect(() => { + localStorage.setItem('sb_ticker_open', tickerOpen.toString()); + }, [tickerOpen]); const [settingsOpen, setSettingsOpen] = useState(false); const [legendOpen, setLegendOpen] = useState(false); + const [terminalOpen, setTerminalOpen] = useState(false); + const [terminalLaunchToken, setTerminalLaunchToken] = useState(0); + const [infonetOpen, setInfonetOpen] = useState(false); + const [meshChatLaunchRequest, setMeshChatLaunchRequest] = useState<{ + tab: 'infonet' | 'meshtastic' | 'dms'; + gate?: string; + nonce: number; + } | null>(null); + const [dmCount, setDmCount] = useState(0); const [mapView, setMapView] = useState({ zoom: 2, latitude: 20 }); + const [locateBarOpen, setLocateBarOpen] = useState(false); const [measureMode, setMeasureMode] = useState(false); const [measurePoints, setMeasurePoints] = useState<{ lat: number; lng: number }[]>([]); - const [activeLayers, setActiveLayers] = useState({ + const openMeshTerminal = useCallback(() => { + setTerminalOpen(true); + setTerminalLaunchToken((prev) => prev + 1); + }, []); + + const openInfonet = useCallback(() => { + setInfonetOpen(true); + }, []); + + const openSecureTerminalLauncher = useCallback(() => { + requestSecureMeshTerminalLauncherOpen('dashboard'); + }, []); + + useEffect(() => subscribeMeshTerminalOpen(openInfonet), [openInfonet]); + + const toggleInfonet = useCallback(() => { + setInfonetOpen(prev => !prev); + }, []); + + const [activeLayers, setActiveLayers] = useState({ flights: true, private: true, jets: true, @@ -147,24 +373,105 @@ export default function Dashboard() { satellites: true, ships_military: true, ships_cargo: true, - ships_civilian: false, + ships_civilian: true, ships_passenger: true, ships_tracked_yachts: true, earthquakes: true, - cctv: false, + cctv: true, ukraine_frontline: true, global_incidents: true, day_night: true, gps_jamming: true, gibs_imagery: false, highres_satellite: false, - kiwisdr: false, - firms: false, - internet_outages: false, - datacenters: false, - military_bases: false, + kiwisdr: true, + psk_reporter: true, + satnogs: true, + tinygs: true, + scanners: true, + firms: true, + internet_outages: true, + datacenters: true, + military_bases: true, power_plants: false, + sigint_meshtastic: true, + sigint_aprs: true, + ukraine_alerts: true, + weather_alerts: true, + air_quality: true, + volcanoes: true, + fishing_activity: true, + sentinel_hub: false, + trains: true, + shodan_overlay: false, + viirs_nightlights: false, + correlations: true, }); + const [shodanResults, setShodanResults] = useState([]); + const [, setShodanQueryLabel] = useState(''); + const [shodanStyle, setShodanStyle] = useState({ shape: 'circle', color: '#16a34a', size: 'md' }); + useDataPolling(); + const backendStatus = useBackendStatus(); + const spaceWeather = useDataKey('space_weather'); + + // Notify backend of layer toggles so it can skip disabled fetchers / stop streams. + // After the POST completes, dispatch a custom event so useDataPolling immediately + // refetches slow-tier data — this makes toggled layers (power plants, GDELT, etc.) + // appear instantly instead of waiting up to 120 seconds. + const layersTimerRef = useRef | null>(null); + const initialLayerSyncRef = useRef(false); + useEffect(() => { + const syncLayers = (triggerRefetch: boolean) => + fetch(`${API_BASE}/api/layers`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ layers: activeLayers }), + }).then(() => { + if (triggerRefetch) { + window.dispatchEvent(new Event(LAYER_TOGGLE_EVENT)); + } + }).catch((e) => console.error('Failed to update backend layers:', e)); + + if (layersTimerRef.current) clearTimeout(layersTimerRef.current); + if (!initialLayerSyncRef.current) { + initialLayerSyncRef.current = true; + void syncLayers(false); + } else { + layersTimerRef.current = setTimeout(() => { + void syncLayers(true); + }, 250); + } + return () => { + if (layersTimerRef.current) clearTimeout(layersTimerRef.current); + }; + }, [activeLayers]); + + // Left panel accordion state + const [leftDataMinimized, setLeftDataMinimized] = useState(false); + const [leftMeshExpanded, setLeftMeshExpanded] = useState(true); + const [leftShodanMinimized, setLeftShodanMinimized] = useState(true); + + const launchMeshChatTab = useCallback((tab: 'infonet' | 'meshtastic' | 'dms', gate?: string) => { + setLeftOpen(true); + setLeftMeshExpanded(true); + setMeshChatLaunchRequest({ tab, gate, nonce: Date.now() }); + }, []); + + const openLiveGateFromShell = useCallback((gate: string) => { + setInfonetOpen(false); + launchMeshChatTab('infonet', gate); + }, [launchMeshChatTab]); + + // Right panel: which panel is "focused" (expanded). null = none focused, all normal. + const [rightFocusedPanel, setRightFocusedPanel] = useState(null); + + // Auto-expand Data Layers when user starts tracking an SDR/Scanner + useEffect(() => { + if (trackedSdr || trackedScanner) { + setLeftDataMinimized(false); + setLeftOpen(true); + } + }, [trackedSdr, trackedScanner]); // NASA GIBS satellite imagery state const [gibsDate, setGibsDate] = useState(() => { @@ -174,11 +481,55 @@ export default function Dashboard() { }); const [gibsOpacity, setGibsOpacity] = useState(0.6); - const [effects, setEffects] = useState({ + // Sentinel Hub satellite imagery state (user-provided Copernicus CDSE credentials) + const [sentinelDate, setSentinelDate] = useState(() => { + const d = new Date(); + d.setDate(d.getDate() - 5); // Sentinel-2 has ~5-day revisit + return d.toISOString().slice(0, 10); + }); + const [sentinelOpacity, setSentinelOpacity] = useState(0.6); + const [sentinelPreset, setSentinelPreset] = useState('TRUE-COLOR'); + const [showSentinelInfo, setShowSentinelInfo] = useState(false); + const prevSentinelRef = useRef(false); + + // Show info modal the first time sentinel_hub is toggled on + useEffect(() => { + if (activeLayers.sentinel_hub && !prevSentinelRef.current) { + if (!hasSentinelInfoBeenSeen()) { + setShowSentinelInfo(true); + markSentinelInfoSeen(); + } + if (!hasSentinelCredentials()) { + // No creds — open settings instead + setSettingsOpen(true); + } + } + prevSentinelRef.current = activeLayers.sentinel_hub; + }, [activeLayers.sentinel_hub]); + + const [effects] = useState({ bloom: true, }); const [activeStyle, setActiveStyle] = useState('DEFAULT'); + + const memoizedEffects = useMemo( + () => ({ ...effects, bloom: effects.bloom && activeStyle !== 'DEFAULT', style: activeStyle }), + [effects, activeStyle], + ); + + const handleFlyTo = useCallback( + (lat: number, lng: number) => setFlyToLocation({ lat, lng, ts: Date.now() }), + [], + ); + + const handleMeasureClick = useCallback( + (pt: { lat: number; lng: number }) => { + setMeasurePoints((prev) => (prev.length >= 3 ? prev : [...prev, pt])); + }, + [], + ); + const stylesList = ['DEFAULT', 'SATELLITE']; const cycleStyle = () => { @@ -192,320 +543,583 @@ export default function Dashboard() { }; const [activeFilters, setActiveFilters] = useState>({}); - const [flyToLocation, setFlyToLocation] = useState<{ lat: number, lng: number, ts: number } | null>(null); + const [flyToLocation, setFlyToLocation] = useState<{ + lat: number; + lng: number; + ts: number; + } | null>(null); // Eavesdrop Mode State - const [isEavesdropping, setIsEavesdropping] = useState(false); - const [eavesdropLocation, setEavesdropLocation] = useState<{ lat: number, lng: number } | null>(null); - const [cameraCenter, setCameraCenter] = useState<{ lat: number, lng: number } | null>(null); + const [isEavesdropping] = useState(false); + const [, setEavesdropLocation] = useState<{ lat: number; lng: number } | null>(null); + const [, setCameraCenter] = useState<{ lat: number; lng: number } | null>(null); // Onboarding & connection status const { showOnboarding, setShowOnboarding } = useOnboarding(); const { showChangelog, setShowChangelog } = useChangelog(); return ( - -
- - {/* MAPLIBRE WEBGL OVERLAY */} - - { - setMeasurePoints(prev => prev.length >= 3 ? prev : [...prev, pt]); - }} - measurePoints={measurePoints} - trackedSdr={trackedSdr} - setTrackedSdr={setTrackedSdr} - /> - - - {uiVisible && ( - <> - {/* WORLDVIEW HEADER */} - -
- {/* Target Reticle Icon */} -
-
-
-
+ <> +
+ {/* MAPLIBRE WEBGL OVERLAY */} + + + + + {uiVisible && ( + <> + {/* WORLDVIEW HEADER */} + +
+ {/* Target Reticle Icon */} +
+
+
+
+
+
+

+ S H A D O W B R O K E R +

+ + GLOBAL THREAT INTERCEPT + +
+ + + {/* SYSTEM METRICS TOP LEFT */} +
+ OPTIC VIS:113 SRC:180 DENS:1.42 0.8ms
-
-

- S H A D O W B R O K E R -

- GLOBAL THREAT INTERCEPT + + {/* SYSTEM METRICS TOP RIGHT */} +
+
RTX
+
VSR
- - {/* SYSTEM METRICS TOP LEFT */} -
- OPTIC VIS:113 SRC:180 DENS:1.42 0.8ms -
+ {/* LEFT HUD CONTAINER — mirrors right side: one scroll container, scrollbar on LEFT edge */} + + {/* 1. DATA LAYERS (Top) */} +
+ + setSettingsOpen(true)} + onLegendClick={() => setLegendOpen(true)} + gibsDate={gibsDate} + setGibsDate={setGibsDate} + gibsOpacity={gibsOpacity} + setGibsOpacity={setGibsOpacity} + sentinelDate={sentinelDate} + setSentinelDate={setSentinelDate} + sentinelOpacity={sentinelOpacity} + setSentinelOpacity={setSentinelOpacity} + sentinelPreset={sentinelPreset} + setSentinelPreset={setSentinelPreset} + onEntityClick={setSelectedEntity} + onFlyTo={handleFlyTo} + trackedSdr={trackedSdr} + setTrackedSdr={setTrackedSdr} + trackedScanner={trackedScanner} + setTrackedScanner={setTrackedScanner} + isMinimized={leftDataMinimized} + onMinimizedChange={setLeftDataMinimized} + /> + +
- {/* SYSTEM METRICS TOP RIGHT */} -
-
RTX
-
VSR
-
+ {/* 2. MESH CHAT (Middle) */} +
+ setSettingsOpen(true)} + onTerminalToggle={openSecureTerminalLauncher} + launchRequest={meshChatLaunchRequest} + /> +
- {/* LEFT HUD CONTAINER — slides off left edge when hidden */} - - {/* LEFT PANEL - DATA LAYERS */} - - setSettingsOpen(true)} onLegendClick={() => setLegendOpen(true)} gibsDate={gibsDate} setGibsDate={setGibsDate} gibsOpacity={gibsOpacity} setGibsOpacity={setGibsOpacity} onEntityClick={setSelectedEntity} onFlyTo={(lat, lng) => setFlyToLocation({ lat, lng, ts: Date.now() })} trackedSdr={trackedSdr} setTrackedSdr={setTrackedSdr} /> - - - - {/* LEFT SIDEBAR TOGGLE TAB */} - - - + + - {/* RIGHT SIDEBAR TOGGLE TAB */} - - - + + - {/* RIGHT HUD CONTAINER — slides off right edge when hidden */} - - + {/* RIGHT HUD CONTAINER — slides off right edge when hidden */} + + setSettingsOpen(true)} + onMeshChatNavigate={launchMeshChatTab} + dmCount={dmCount} + /> - {/* FIND / LOCATE */} -
+ {/* FIND / LOCATE */} +
{ + onLocate={(lat, lng, _entityId, _entityType) => { setFlyToLocation({ lat, lng, ts: Date.now() }); }} onFilter={(filterKey, value) => { - setActiveFilters(prev => { - const current = prev[filterKey] || []; - if (!current.includes(value)) { - return { ...prev, [filterKey]: [...current, value] }; - } - return prev; - }); - }} - /> -
+ setActiveFilters((prev) => { + const current = prev[filterKey] || []; + if (!current.includes(value)) { + return { ...prev, [filterKey]: [...current, value] }; + } + return prev; + }); + }} + /> +
- {/* TOP RIGHT - MARKETS */} -
- - - -
+ {/* GLOBAL TICKER REPLACES MARKETS PANEL - RENDERED OUTSIDE THIS DIV */} + + {/* ORACLE PREDICTIONS */} +
+ + + +
- {/* SIGINT & RADIO INTERCEPTS */} -
- - + + + +
+ + {/* BOTTOM RIGHT - NEWS FEED (fills remaining space) */} +
+ + { + if (lat !== undefined && lng !== undefined) { + setFlyToLocation({ lat, lng, ts: Date.now() }); + } + }} + /> + +
+ + + {/* BOTTOM CENTER COORDINATE / LOCATION BAR — hidden when fullscreen overlays are open */} + {!(selectedEntity?.type === 'region_dossier' && regionDossier?.sentinel2) && selectedEntity?.type !== 'cctv' && selectedEntity?.type !== 'news' && ( + + {/* LOCATE BAR — search by coordinates or place name */} + setFlyToLocation({ lat, lng, ts: Date.now() })} + onOpenChange={setLocateBarOpen} /> - -
- {/* DATA FILTERS */} -
- - - -
+
+ {/* Coordinates */} +
+
+ COORDINATES +
+
+ {mouseCoords + ? `${mouseCoords.lat.toFixed(4)}, ${mouseCoords.lng.toFixed(4)}` + : '0.0000, 0.0000'} +
+
- {/* BOTTOM RIGHT - NEWS FEED (fills remaining space) */} -
- - - -
- - - {/* BOTTOM CENTER COORDINATE / LOCATION BAR — hidden when Sentinel-2 imagery overlay is open */} - {!(selectedEntity?.type === 'region_dossier' && regionDossier?.sentinel2) && + + {/* Location name */} +
+
+ LOCATION +
+
+ {locationLabel || 'Hover over map...'} +
+
+ + {/* Divider */} +
+ + {/* Style preset (compact) */} +
+
+ STYLE +
+
+ {activeStyle} +
+
+ + {/* Divider */} +
+ + {/* Space Weather */} + {(() => { + const sw = spaceWeather as { kp_index?: number; kp_text?: string } | undefined; + return ( +
+
+ SOLAR +
+
= 5 + ? 'text-red-400' + : (sw?.kp_index ?? 0) >= 4 + ? 'text-yellow-400' + : 'text-green-400' + }`} + > + {sw?.kp_text || 'N/A'} +
+
+ ); + })()} +
+ + )} + + )} + + {/* RESTORE UI BUTTON (If Hidden) */} + {!uiVisible && ( + + )} + + {/* DYNAMIC SCALE BAR — hidden when fullscreen overlays or locate bar are open */} + {!(selectedEntity?.type === 'region_dossier' && regionDossier?.sentinel2) && selectedEntity?.type !== 'cctv' && selectedEntity?.type !== 'news' && !locateBarOpen && ( +
+ { + setMeasureMode((m) => !m); + if (measureMode) setMeasurePoints([]); + }} + onClearMeasure={() => setMeasurePoints([])} + /> +
+ )} + + {/* STATIC CRT VIGNETTE */} +
+ + {/* SCANLINES OVERLAY */} +
+ + {/* SETTINGS PANEL */} + + setSettingsOpen(false)} /> + + + {/* MAP LEGEND */} + + setLegendOpen(false)} /> + + {/* ONBOARDING MODAL */} + {showOnboarding && ( + setShowOnboarding(false)} + onOpenSettings={() => { + setShowOnboarding(false); + setSettingsOpen(true); + }} + /> + )} + + {/* v0.4 CHANGELOG MODAL — shows once per version after onboarding */} + {!showOnboarding && showChangelog && ( + setShowChangelog(false)} /> + )} + + {/* SENTINEL HUB — first-time info modal */} + {showSentinelInfo && ( +
- {/* Coordinates */} -
-
COORDINATES
-
- {mouseCoords ? `${mouseCoords.lat.toFixed(4)}, ${mouseCoords.lng.toFixed(4)}` : '0.0000, 0.0000'} + className="absolute inset-0 bg-black/90" + onClick={() => setShowSentinelInfo(false)} + /> +
+
+
+

+ SENTINEL HUB IMAGERY +

+
-
- {/* Divider */} -
+

+ You now have access to ESA Sentinel-2 satellite imagery directly on the map. + This uses the Copernicus Data Space Ecosystem with your own credentials. +

- {/* Location name */} -
-
LOCATION
-
- {locationLabel || 'Hover over map...'} +
+

AVAILABLE LAYERS

+
+ {[ + { name: 'True Color', desc: 'Natural RGB — see terrain, cities, water' }, + { name: 'False Color IR', desc: 'Near-infrared — vegetation in red' }, + { name: 'NDVI', desc: 'Vegetation health index (green = healthy)' }, + { name: 'Moisture Index', desc: 'Soil & vegetation moisture levels' }, + ].map((l) => ( +
+
{l.name}
+
{l.desc}
+
+ ))} +
-
- - {/* Divider */} -
- {/* Style preset (compact) */} -
-
STYLE
-
{activeStyle}
-
+
+

USAGE LIMITS (FREE TIER)

+
+
+ Monthly budget + 10,000 requests +
+
+ Cost per tile + 0.25 PU (256×256px) +
+
+ ~Viewport loads/month + ~500 (20 tiles each) +
+
+ Empty tiles + FREE (no data = no charge) +
+
+
- {/* Divider */} -
- - {/* Space Weather */} -
-
SOLAR
-
= 5 ? 'text-red-400' : - (data?.space_weather?.kp_index ?? 0) >= 4 ? 'text-yellow-400' : - 'text-green-400' - }`}> - {data?.space_weather?.kp_text || 'N/A'} +
+

HOW IT WORKS

+
    +
  • Sentinel-2 revisits every ~5 days — not every location has data every day
    • +
  • The date slider picks the end of a time window; zoomed out uses wider windows
    • +
  • Black patches = no satellite pass on that date range (normal)
    • +
  • Best results at zoom 8-14 — closer = sharper imagery (10m resolution)
    • +
  • Cloud filter auto-skips tiles with {'>'} 30% cloud cover
    • +
+ +
- } - - )} - - {/* RESTORE UI BUTTON (If Hidden) */} - {!uiVisible && ( - - )} +
+ )} - {/* DYNAMIC SCALE BAR */} -
- { - setMeasureMode(m => !m); - if (measureMode) setMeasurePoints([]); - }} - onClearMeasure={() => setMeasurePoints([])} + {/* MESH TERMINAL */} + setTerminalOpen(false)} + onDmCount={setDmCount} + onSettingsClick={() => setSettingsOpen(true)} /> -
- {/* STATIC CRT VIGNETTE */} -
- - {/* SCANLINES OVERLAY */} -
- - {/* SETTINGS PANEL */} - - setSettingsOpen(false)} /> - - - {/* MAP LEGEND */} - - setLegendOpen(false)} /> - - - {/* ONBOARDING MODAL */} - {showOnboarding && ( - setShowOnboarding(false)} - onOpenSettings={() => { setShowOnboarding(false); setSettingsOpen(true); }} + {/* INFONET TERMINAL */} + { + setInfonetOpen(false); + // Shut down Wormhole when the terminal closes so it doesn't stay running + fetchWormholeState(false) + .then((s) => { + if (s?.ready || s?.running) return leaveWormhole(); + }) + .catch(() => {}); + }} + onOpenLiveGate={openLiveGateFromShell} /> - )} - {/* v0.4 CHANGELOG MODAL — shows once per version after onboarding */} - {!showOnboarding && showChangelog && ( - setShowChangelog(false)} /> - )} + {/* BACKEND DISCONNECTED BANNER */} + {backendStatus === 'disconnected' && ( +
+ + BACKEND OFFLINE — Cannot reach backend server. Check that the backend container is + running and BACKEND_URL is correct. + +
+ )} + {/* BOTTOM TICKER TOGGLE TAB — moved to right to avoid Shodan overlap */} + + + - {/* BACKEND DISCONNECTED BANNER */} - {backendStatus === 'disconnected' && ( -
- - BACKEND OFFLINE — Cannot reach backend server. Check that the backend container is running and BACKEND_URL is correct. - -
- )} + {/* GLOBAL MARKETS TICKER (BOTTOM ANCHOR) */} + + + + + -
- +
+ ); } diff --git a/frontend/src/components/AdvancedFilterModal.tsx b/frontend/src/components/AdvancedFilterModal.tsx index 696095df..101fe525 100644 --- a/frontend/src/components/AdvancedFilterModal.tsx +++ b/frontend/src/components/AdvancedFilterModal.tsx @@ -1,345 +1,417 @@ -"use client"; +'use client'; import { useState, useMemo, useRef, useCallback, useEffect } from 'react'; import { motion } from 'framer-motion'; import { Search, X, Check, GripHorizontal } from 'lucide-react'; interface FilterField { - key: string; - label: string; - options: string[]; - optionLabels?: Record; + key: string; + label: string; + options: string[]; + optionLabels?: Record; } interface AdvancedFilterModalProps { - title: string; - icon: React.ReactNode; - accentColor: string; // CSS color string e.g. '#00bcd4' - accentColorName: string; // tailwind name e.g. 'cyan' - fields: FilterField[]; - activeFilters: Record; - onApply: (filters: Record) => void; - onClose: () => void; + title: string; + icon: React.ReactNode; + accentColor: string; // CSS color string e.g. '#00bcd4' + accentColorName: string; // tailwind name e.g. 'cyan' + fields: FilterField[]; + activeFilters: Record; + onApply: (filters: Record) => void; + onClose: () => void; } export default function AdvancedFilterModal({ - title, icon, accentColor, accentColorName, fields, activeFilters, onApply, onClose + title, + icon, + accentColor: _accentColor, + accentColorName, + fields, + activeFilters, + onApply, + onClose, }: AdvancedFilterModalProps) { - // Local draft state — only committed on Apply - const [draft, setDraft] = useState>>(() => { - const init: Record> = {}; - for (const field of fields) { - init[field.key] = new Set(activeFilters[field.key] || []); - } - return init; - }); - - const [searchTerms, setSearchTerms] = useState>(() => { - const init: Record = {}; - for (const field of fields) init[field.key] = ''; - return init; - }); + // Local draft state — only committed on Apply + const [draft, setDraft] = useState>>(() => { + const init: Record> = {}; + for (const field of fields) { + init[field.key] = new Set(activeFilters[field.key] || []); + } + return init; + }); - const [activeTab, setActiveTab] = useState(fields[0]?.key || ''); + const [searchTerms, setSearchTerms] = useState>(() => { + const init: Record = {}; + for (const field of fields) init[field.key] = ''; + return init; + }); - // Dragging state - const [position, setPosition] = useState({ x: 0, y: 0 }); - const [isDragging, setIsDragging] = useState(false); - const dragStartRef = useRef({ x: 0, y: 0, posX: 0, posY: 0 }); - const modalRef = useRef(null); + const [activeTab, setActiveTab] = useState(fields[0]?.key || ''); - // Center on mount - useEffect(() => { - if (modalRef.current) { - const rect = modalRef.current.getBoundingClientRect(); - setPosition({ - x: (window.innerWidth - rect.width) / 2, - y: (window.innerHeight - rect.height) / 2 - }); - } - }, []); + // Dragging state + const [position, setPosition] = useState({ x: 0, y: 0 }); + const [isDragging, setIsDragging] = useState(false); + const dragStartRef = useRef({ x: 0, y: 0, posX: 0, posY: 0 }); + const modalRef = useRef(null); - const handleMouseDown = useCallback((e: React.MouseEvent) => { - e.preventDefault(); - setIsDragging(true); - dragStartRef.current = { x: e.clientX, y: e.clientY, posX: position.x, posY: position.y }; - }, [position]); + // Center on mount, clamped so it doesn't overlap the bottom status bar (~48px) + useEffect(() => { + if (modalRef.current) { + const rect = modalRef.current.getBoundingClientRect(); + const pad = 52; // status bar + small gap + const maxY = window.innerHeight - rect.height - pad; + setPosition({ + x: Math.max(0, (window.innerWidth - rect.width) / 2), + y: Math.max(0, Math.min((window.innerHeight - rect.height) / 2, maxY)), + }); + } + }, []); - useEffect(() => { - if (!isDragging) return; - const handleMove = (e: MouseEvent) => { - const dx = e.clientX - dragStartRef.current.x; - const dy = e.clientY - dragStartRef.current.y; - setPosition({ - x: dragStartRef.current.posX + dx, - y: dragStartRef.current.posY + dy - }); - }; - const handleUp = () => setIsDragging(false); - window.addEventListener('mousemove', handleMove); - window.addEventListener('mouseup', handleUp); - return () => { - window.removeEventListener('mousemove', handleMove); - window.removeEventListener('mouseup', handleUp); - }; - }, [isDragging]); + const handleMouseDown = useCallback( + (e: React.MouseEvent) => { + e.preventDefault(); + setIsDragging(true); + dragStartRef.current = { x: e.clientX, y: e.clientY, posX: position.x, posY: position.y }; + }, + [position], + ); - const toggleItem = (fieldKey: string, value: string) => { - setDraft(prev => { - const next = { ...prev }; - const s = new Set(prev[fieldKey]); - if (s.has(value)) s.delete(value); - else s.add(value); - next[fieldKey] = s; - return next; - }); + useEffect(() => { + if (!isDragging) return; + const handleMove = (e: MouseEvent) => { + const dx = e.clientX - dragStartRef.current.x; + const dy = e.clientY - dragStartRef.current.y; + const newX = dragStartRef.current.posX + dx; + const newY = dragStartRef.current.posY + dy; + // Clamp so modal can't be dragged below the bottom status bar + const maxY = window.innerHeight - 120; // keep at least 120px visible + setPosition({ + x: Math.max(-200, newX), + y: Math.max(0, Math.min(newY, maxY)), + }); }; - - const removeChip = (fieldKey: string, value: string) => { - setDraft(prev => { - const next = { ...prev }; - const s = new Set(prev[fieldKey]); - s.delete(value); - next[fieldKey] = s; - return next; - }); + const handleUp = () => setIsDragging(false); + window.addEventListener('mousemove', handleMove); + window.addEventListener('mouseup', handleUp); + return () => { + window.removeEventListener('mousemove', handleMove); + window.removeEventListener('mouseup', handleUp); }; + }, [isDragging]); - const clearField = (fieldKey: string) => { - setDraft(prev => ({ ...prev, [fieldKey]: new Set() })); - }; + const toggleItem = (fieldKey: string, value: string) => { + setDraft((prev) => { + const next = { ...prev }; + const s = new Set(prev[fieldKey]); + if (s.has(value)) s.delete(value); + else s.add(value); + next[fieldKey] = s; + return next; + }); + }; - const clearAll = () => { - const cleared: Record> = {}; - for (const f of fields) cleared[f.key] = new Set(); - setDraft(cleared); - }; + const removeChip = (fieldKey: string, value: string) => { + setDraft((prev) => { + const next = { ...prev }; + const s = new Set(prev[fieldKey]); + s.delete(value); + next[fieldKey] = s; + return next; + }); + }; - const handleApply = () => { - const result: Record = {}; - for (const [key, set] of Object.entries(draft)) { - if (set.size > 0) result[key] = Array.from(set); - } - onApply(result); - onClose(); - }; + const clearField = (fieldKey: string) => { + setDraft((prev) => ({ ...prev, [fieldKey]: new Set() })); + }; - const totalSelected = Object.values(draft).reduce((acc, s) => acc + s.size, 0); + const clearAll = () => { + const cleared: Record> = {}; + for (const f of fields) cleared[f.key] = new Set(); + setDraft(cleared); + }; - const activeField = fields.find(f => f.key === activeTab); - const filteredOptions = useMemo(() => { - if (!activeField) return []; - const term = (searchTerms[activeTab] || '').toLowerCase(); - const opts = activeField.options; - if (!term) return opts; - return opts.filter(o => { - const displayLabel = activeField.optionLabels?.[o] || o; - return displayLabel.toLowerCase().includes(term); - }); - }, [activeField, activeTab, searchTerms]); + const handleApply = () => { + const result: Record = {}; + for (const [key, set] of Object.entries(draft)) { + if (set.size > 0) result[key] = Array.from(set); + } + onApply(result); + onClose(); + }; - const accentBorder = `border-[${accentColor}]/30`; + const totalSelected = Object.values(draft).reduce((acc, s) => acc + s.size, 0); - // Tailwind color map for dynamic classes - const colorMap: Record = { - cyan: { text: 'text-cyan-400', bg: 'bg-cyan-500/10', bgHover: 'hover:bg-cyan-500/15', border: 'border-cyan-500/30', ring: 'ring-cyan-500/50' }, - orange: { text: 'text-orange-400', bg: 'bg-orange-500/10', bgHover: 'hover:bg-orange-500/15', border: 'border-orange-500/30', ring: 'ring-orange-500/50' }, - yellow: { text: 'text-yellow-400', bg: 'bg-yellow-500/10', bgHover: 'hover:bg-yellow-500/15', border: 'border-yellow-500/30', ring: 'ring-yellow-500/50' }, - pink: { text: 'text-pink-400', bg: 'bg-pink-500/10', bgHover: 'hover:bg-pink-500/15', border: 'border-pink-500/30', ring: 'ring-pink-500/50' }, - blue: { text: 'text-blue-400', bg: 'bg-blue-500/10', bgHover: 'hover:bg-blue-500/15', border: 'border-blue-500/30', ring: 'ring-blue-500/50' }, - }; - const c = colorMap[accentColorName] || colorMap.cyan; + const activeField = fields.find((f) => f.key === activeTab); + const filteredOptions = useMemo(() => { + if (!activeField) return []; + const term = (searchTerms[activeTab] || '').toLowerCase(); + const opts = activeField.options; + if (!term) return opts; + return opts.filter((o) => { + const displayLabel = activeField.optionLabels?.[o] || o; + return displayLabel.toLowerCase().includes(term); + }); + }, [activeField, activeTab, searchTerms]); + + // Tailwind color map for dynamic classes + const colorMap: Record< + string, + { text: string; bg: string; bgHover: string; border: string; ring: string } + > = { + cyan: { + text: 'text-cyan-400', + bg: 'bg-cyan-500/10', + bgHover: 'hover:bg-cyan-500/15', + border: 'border-cyan-500/30', + ring: 'ring-cyan-500/50', + }, + orange: { + text: 'text-orange-400', + bg: 'bg-orange-500/10', + bgHover: 'hover:bg-orange-500/15', + border: 'border-orange-500/30', + ring: 'ring-orange-500/50', + }, + yellow: { + text: 'text-yellow-400', + bg: 'bg-yellow-500/10', + bgHover: 'hover:bg-yellow-500/15', + border: 'border-yellow-500/30', + ring: 'ring-yellow-500/50', + }, + pink: { + text: 'text-pink-400', + bg: 'bg-pink-500/10', + bgHover: 'hover:bg-pink-500/15', + border: 'border-pink-500/30', + ring: 'ring-pink-500/50', + }, + blue: { + text: 'text-blue-400', + bg: 'bg-blue-500/10', + bgHover: 'hover:bg-blue-500/15', + border: 'border-blue-500/30', + ring: 'ring-blue-500/50', + }, + }; + const c = colorMap[accentColorName] || colorMap.cyan; - return ( -
{ if (e.target === e.currentTarget) onClose(); }}> - {/* Backdrop */} -
+ return ( +
{ + if (e.target === e.currentTarget) onClose(); + }} + > + {/* Backdrop */} +
- {/* Modal */} -
+ + {/* ── Title Bar (Draggable) ── */} +
+
+ + {icon} + + {title} + + {totalSelected > 0 && ( + + {totalSelected} SELECTED + + )} +
+ -
+ + +
- {/* ── Tab Bar (for multi-field categories) ── */} - {fields.length > 1 && ( -
- {fields.map(field => { - const isActive = activeTab === field.key; - const count = draft[field.key]?.size || 0; - return ( - - ); - })} -
+ {/* ── Tab Bar (for multi-field categories) ── */} + {fields.length > 1 && ( +
+ {fields.map((field) => { + const isActive = activeTab === field.key; + const count = draft[field.key]?.size || 0; + return ( + + ); + })} +
+ )} - {/* ── Selected Chips ── */} - {activeField && draft[activeTab]?.size > 0 && ( -
- {Array.from(draft[activeTab]).map(val => { - const displayVal = activeField.optionLabels?.[val] || val; - return ( - - {displayVal.length > 28 ? displayVal.slice(0, 28) + '…' : displayVal} - - - ); - })} - -
- )} + {/* ── Selected Chips ── */} + {activeField && draft[activeTab]?.size > 0 && ( +
+ {Array.from(draft[activeTab]).map((val) => { + const displayVal = activeField.optionLabels?.[val] || val; + return ( + + {displayVal.length > 28 ? displayVal.slice(0, 28) + '…' : displayVal} + + + ); + })} + +
+ )} - {/* ── Search Bar ── */} -
-
- - setSearchTerms(prev => ({ ...prev, [activeTab]: e.target.value }))} - placeholder={`Search ${activeField?.label.toLowerCase() || ''}...`} - className={`w-full bg-[var(--bg-primary)]/50 border border-[var(--border-primary)]/70 rounded-lg text-[11px] text-[var(--text-secondary)] pl-8 pr-8 py-2 font-mono tracking-wide focus:outline-none focus:${c.border} focus:ring-1 ${c.ring} placeholder-[var(--text-muted)] transition-all`} - autoFocus - /> - {searchTerms[activeTab] && ( - - )} -
-
- - {filteredOptions.length} AVAILABLE - - - {draft[activeTab]?.size || 0} SELECTED - -
-
+ {/* ── Search Bar ── */} +
+
+ + + setSearchTerms((prev) => ({ ...prev, [activeTab]: e.target.value })) + } + placeholder={`Search ${activeField?.label.toLowerCase() || ''}...`} + className={`w-full bg-[var(--bg-primary)]/50 border border-[var(--border-primary)]/70 text-[11px] text-[var(--text-secondary)] pl-8 pr-8 py-2 font-mono tracking-wide focus:outline-none focus:${c.border} focus:ring-1 ${c.ring} placeholder-[var(--text-muted)] transition-all`} + autoFocus + /> + {searchTerms[activeTab] && ( + + )} +
+
+ + {filteredOptions.length} AVAILABLE + + + {draft[activeTab]?.size || 0} SELECTED + +
+
- {/* ── Scrollable Checkbox List ── */} -
- {filteredOptions.length === 0 ? ( -
- NO MATCHING RESULTS -
- ) : ( -
- {filteredOptions.map((option) => { - const isChecked = draft[activeTab]?.has(option); - return ( - - ); - })} -
- )} -
+ {/* ── Scrollable Checkbox List ── */} +
+ {filteredOptions.length === 0 ? ( +
+ NO MATCHING RESULTS +
+ ) : ( +
+ {filteredOptions.map((option) => { + const isChecked = draft[activeTab]?.has(option); + return ( + + ); + })} +
+ )} +
- {/* ── Footer ── */} -
- -
- - -
-
- + {/* ── Footer ── */} +
+ +
+ +
-
- ); +
+ +
+
+ ); } diff --git a/frontend/src/components/ChangelogModal.tsx b/frontend/src/components/ChangelogModal.tsx index e2464f1c..23bb71e7 100644 --- a/frontend/src/components/ChangelogModal.tsx +++ b/frontend/src/components/ChangelogModal.tsx @@ -1,198 +1,385 @@ -"use client"; +'use client'; -import React, { useState, useEffect } from "react"; -import { motion, AnimatePresence } from "framer-motion"; -import { X, Zap, Ship, Download, Shield, Bug, Heart } from "lucide-react"; +import React, { useState, useEffect } from 'react'; +import { motion, AnimatePresence } from 'framer-motion'; +import { + X, + Terminal, + Radio, + Camera, + Search, + TrainFront, + Globe, + Shield, + Bug, + Heart, +} from 'lucide-react'; -const CURRENT_VERSION = "0.9.5"; +const CURRENT_VERSION = '0.9.6'; const STORAGE_KEY = `shadowbroker_changelog_v${CURRENT_VERSION}`; +const RELEASE_TITLE = 'InfoNet Experimental Testnet — Decentralized Intelligence Experiment'; + +const HEADLINE_FEATURE = { + icon: , + title: 'InfoNet Experimental Testnet is Live', + subtitle: 'The first decentralized intelligence mesh built directly into an OSINT platform. This is an experimental testnet — NOT a privacy tool.', + details: [ + 'A global, obfuscated message relay running inside ShadowBroker. Anyone with the dashboard can transmit and receive on the InfoNet — no accounts, no signup, no identity required.', + 'Messages pass through a Wormhole relay layer with gate personas, canonical payload signing, and message obfuscation. Transport is obfuscated to a degree, but this is NOT private communication. Do not transmit anything you would not say in public. End-to-end encryption is being developed but is not yet implemented.', + 'Dead Drop inbox for peer-to-peer message exchange. Mesh Terminal CLI for power users. Gate persona system for pseudonymous identity. Double-ratchet DM scaffolding in progress.', + 'Nothing like this has existed in an OSINT tool before. This is an open experiment — jump on the testnet, explore the protocol, and help shape what decentralized intelligence looks like.', + ], + callToAction: 'OPEN MESH CHAT \u2192 MESH TAB \u2192 START TRANSMITTING', +}; const NEW_FEATURES = [ - { - icon: , - title: "Parallelized Boot (15s Cold Start)", - desc: "Backend startup now runs fast-tier, slow-tier, and airport data concurrently via ThreadPoolExecutor. Boot time cut from 60s+ to ~15s.", - color: "cyan", - }, - { - icon: , - title: "Adaptive Polling + ETag Caching", - desc: "Data polling engine rebuilt with adaptive retry (3s startup, 15s steady state) and ETag conditional caching. Map panning no longer interrupts data flow.", - color: "green", - }, - { - icon: , - title: "Sliding Edge Panels (LAYERS / INTEL)", - desc: "Replaced bulky Record Panel with spring-animated side tabs. LAYERS on the left, INTEL (News, Markets, Radio, Find) on the right. Premium tactical HUD feel.", - color: "blue", - }, - { - icon: , - title: "Admin Auth + Rate Limiting + Auto-Updater", - desc: "Settings and system endpoints protected by X-Admin-Key. All endpoints rate-limited via slowapi. One-click auto-update from GitHub releases with safe backup/restart.", - color: "yellow", - }, - { - icon: , - title: "Docker Swarm Secrets Support", - desc: "Production deployments can now load API keys from /run/secrets/ instead of environment variables. env_check.py enforces warning tiers for missing keys.", - color: "purple", - }, + { + icon: , + title: 'Meshtastic + APRS Radio Integration', + desc: 'Live Meshtastic mesh radio nodes plotted worldwide via MQTT. APRS amateur radio positioning via APRS-IS TCP feed. Both integrated into Mesh Chat and the SIGINT grid. Note: Mesh radio is NOT private — RF transmissions are public by nature.', + color: 'amber', + }, + { + icon: , + title: 'Mesh Terminal', + desc: 'Built-in command-line interface. Send messages, DMs, run market commands, inspect gate state. Draggable panel, minimizes to the top bar. Type "help" to see everything.', + color: 'cyan', + }, + { + icon: , + title: 'Shodan Device Search', + desc: 'Query Shodan directly from ShadowBroker. Search internet-connected devices by keyword, CVE, or port — results plotted as a live overlay on the map with configurable marker style.', + color: 'green', + }, + { + icon: , + title: 'CCTV Mesh Expanded — 12 Sources, 11,000+ Cameras', + desc: 'Massive expansion: added Spain (DGT national + Madrid city), California (12 Caltrans districts), Washington State, Georgia, Illinois, Michigan, and Windy Webcams. Now covers 6 countries. Enabled by default.', + color: 'emerald', + }, + { + icon: , + title: 'Train Tracking (Amtrak + European Rail)', + desc: 'Real-time Amtrak train positions across the US and European rail via DigiTraffic. Speed, heading, route, and status for every train on the network.', + color: 'blue', + }, + { + icon: , + title: '8 New Intelligence Layers', + desc: 'Volcanoes (Smithsonian), air quality PM2.5 (OpenAQ), severe weather alerts, fishing activity (Global Fishing Watch), military bases, 35K+ power plants, SatNOGS ground stations, TinyGS LoRa satellites, VIIRS nightlights.', + color: 'purple', + }, + { + icon: , + title: 'Sentinel Hub Imagery + Desktop Shell Scaffold', + desc: 'Copernicus CDSE satellite imagery via Sentinel Hub Process API with OAuth2 token flow. Desktop-native control routing scaffold (pre-Tauri) with session profiles and audit trail.', + color: 'yellow', + }, ]; const BUG_FIXES = [ - "Fixed start.sh: added missing `fi` after UV install block — valid bash again; setup runs whether or not uv was preinstalled (2026-03-26)", - "Stable entity IDs for GDELT & News popups — no more wrong popup after data refresh (PR #63)", - "useCallback optimization for interpolation functions — eliminates redundant React re-renders on every 1s tick", - "Restored missing GDELT and datacenter background refreshes in slow-tier loop", - "Server-side viewport bounding box filtering reduces JSON payload size by 80%+", - "Modular fetcher architecture sustained over monolithic data_fetcher.py", - "CCTV ingestors instantiated once at startup — no more fresh DB connections every 10min tick", + 'CCTV auto-seed fix — partial DB (4 of 12 sources) no longer silently skips the other 8 ingestors on startup', + 'SQLite threading fix — CCTV ingestors no longer share connections across threads', + 'CCTV layer now ON by default and participates in the All On/Off global toggle', + 'KiwiSDR, FIRMS fires, internet outages, data centers all switched to ON by default', + 'Terminal minimized tab repositioned to top-center with proper icon (no more phantom cursor)', + 'Mesh Chat defaults to MESH tab on startup instead of locked INFONET gate', ]; const CONTRIBUTORS = [ - { name: "@imqdcr", desc: "Ship toggle split into 4 categories + stable MMSI/callsign entity IDs for map markers" }, - { name: "@csysp", desc: "Dismissible threat alerts + stable entity IDs for GDELT & News popups", pr: "#48, #63" }, - { name: "@suranyami", desc: "Parallel multi-arch Docker builds (11min \u2192 3min) + runtime BACKEND_URL fix", pr: "#35, #44" }, + { + name: '@wa1id', + desc: 'CCTV ingestion fix — fresh SQLite connections per ingest, persistent DB path, startup hydration, cluster clickability', + pr: '#92', + }, + { + name: '@AlborzNazari', + desc: 'Spain DGT + Madrid CCTV sources and STIX 2.1 threat intelligence export endpoint', + pr: '#91', + }, + { + name: '@adust09', + desc: 'Power plants layer, East Asia intel coverage (JSDF bases, ICAO enrichment, Taiwan news sources, military classification)', + pr: '#71, #72, #76, #77, #87', + }, + { + name: '@Xpirix', + desc: 'LocateBar style and interaction improvements', + pr: '#78', + }, + { + name: '@imqdcr', + desc: 'Ship toggle split into 4 categories + stable MMSI/callsign entity IDs for map markers', + pr: '#52', + }, + { + name: '@csysp', + desc: 'Dismissible threat alerts + stable entity IDs for GDELT & News popups + UI declutter', + pr: '#48, #61, #63', + }, + { + name: '@suranyami', + desc: 'Parallel multi-arch Docker builds (11min \u2192 3min) + runtime BACKEND_URL fix', + pr: '#35, #44', + }, + { + name: '@chr0n1x', + desc: 'Kubernetes / Helm chart architecture for high-availability deployments', + }, + { + name: '@johan-martensson', + desc: 'COSMO-SkyMed satellite classification fix + yfinance batch download optimization', + pr: '#96, #98', + }, + { + name: '@singularfailure', + desc: 'Spanish CCTV feeds + image loading fix', + pr: '#93', + }, + { + name: '@smithbh', + desc: 'Makefile-based taskrunner with LAN/local access options', + pr: '#103', + }, + { + name: '@OrfeoTerkuci', + desc: 'UV project management setup', + pr: '#102', + }, + { + name: '@deuza', + desc: 'dos2unix fix for Mac/Linux quick start', + pr: '#101', + }, + { + name: '@tm-const', + desc: 'CI/CD workflow updates', + pr: '#108, #109', + }, + { + name: '@Elhard1', + desc: 'start.sh shell script fix', + pr: '#111', + }, + { + name: '@ttulttul', + desc: 'Podman compose support + frontend production CSS fix', + pr: '#23', + }, ]; export function useChangelog() { - const [show, setShow] = useState(false); - useEffect(() => { - const seen = localStorage.getItem(STORAGE_KEY); - if (!seen) setShow(true); - }, []); - return { showChangelog: show, setShowChangelog: setShow }; + const [show, setShow] = useState(false); + useEffect(() => { + const seen = localStorage.getItem(STORAGE_KEY); + if (!seen) setShow(true); + }, []); + return { showChangelog: show, setShowChangelog: setShow }; } interface ChangelogModalProps { - onClose: () => void; + onClose: () => void; } const ChangelogModal = React.memo(function ChangelogModal({ onClose }: ChangelogModalProps) { - const handleDismiss = () => { - localStorage.setItem(STORAGE_KEY, "true"); - onClose(); - }; - - return ( - - { + localStorage.setItem(STORAGE_KEY, 'true'); + onClose(); + }; + + return ( + + + +
e.stopPropagation()} + > + {/* Header */} +
+
+
+
+
+ v{CURRENT_VERSION} +
+

+ WHAT'S NEW +

+
+

+ {RELEASE_TITLE.toUpperCase()} +

+
+ -
-
+ className="w-8 h-8 border border-[var(--border-primary)] hover:border-red-500/50 flex items-center justify-center text-[var(--text-muted)] hover:text-red-400 transition-all hover:bg-red-950/20" + > + + +
+
+ + {/* Content */} +
+ {/* === HEADLINE: InfoNet Testnet === */} +
+
+
+ {HEADLINE_FEATURE.icon} +
+
+
+ {HEADLINE_FEATURE.title} +
+
+ {HEADLINE_FEATURE.subtitle} +
+
+
- {/* Content */} -
- {/* New Features */} -
-
-
- NEW CAPABILITIES -
-
- {NEW_FEATURES.map((f) => ( -
-
{f.icon}
-
-
{f.title}
-
{f.desc}
-
-
- ))} -
-
- - {/* Bug Fixes */} -
-
- - FIXES & IMPROVEMENTS -
-
- {BUG_FIXES.map((fix, i) => ( -
- + - {fix} -
- ))} -
-
- - {/* Contributors */} -
-
- - COMMUNITY CONTRIBUTORS -
-
- {CONTRIBUTORS.map((c, i) => ( -
- ♥ -
- {c.name} - — {c.desc} - (PR {c.pr}) -
-
- ))} -
-
+
+ {HEADLINE_FEATURE.details.map((para, i) => ( +

+ {para} +

+ ))} +
+ + {/* Testnet disclaimer */} +
+ !! +
+ + EXPERIMENTAL TESTNET — NO PRIVACY GUARANTEE + + + InfoNet messages are obfuscated but NOT encrypted end-to-end. The Mesh network + (Meshtastic/APRS) is NOT private — radio transmissions are inherently + public. Do not send anything sensitive on any channel. Privacy and E2E encryption + are actively being developed. Treat all channels as open and public for now. + +
+
+ + {/* CTA */} +
+ + {HEADLINE_FEATURE.callToAction} + +
+
+ + {/* === Other New Features === */} +
+
+
+ NEW CAPABILITIES +
+
+ {NEW_FEATURES.map((f) => ( +
+
{f.icon}
+
+
+ {f.title} +
+
+ {f.desc} +
+
+ ))} +
+
- {/* Footer */} -
- + {/* Bug Fixes */} +
+
+ + FIXES & IMPROVEMENTS +
+
+ {BUG_FIXES.map((fix, i) => ( +
+ + + + {fix} + +
+ ))} +
+
+ + {/* Contributors */} +
+
+ + COMMUNITY CONTRIBUTORS +
+
+ {CONTRIBUTORS.map((c, i) => ( +
+ + ♥ + +
+ + {c.name} + + + {' '} + — {c.desc} + + {c.pr && ( + + {' '} + (PR {c.pr}) + + )}
-
- - - ); +
+ ))} +
+
+
+ + {/* Footer */} +
+ +
+
+ + + ); }); export default ChangelogModal; diff --git a/frontend/src/components/DesktopBridgeBootstrap.tsx b/frontend/src/components/DesktopBridgeBootstrap.tsx new file mode 100644 index 00000000..ba5d286e --- /dev/null +++ b/frontend/src/components/DesktopBridgeBootstrap.tsx @@ -0,0 +1,12 @@ +'use client'; + +import { useEffect } from 'react'; +import { bootstrapDesktopControlBridge } from '@/lib/desktopBridge'; + +export default function DesktopBridgeBootstrap() { + useEffect(() => { + bootstrapDesktopControlBridge(); + }, []); + + return null; +} diff --git a/frontend/src/components/ErrorBoundary.tsx b/frontend/src/components/ErrorBoundary.tsx index 5075707e..343d6470 100644 --- a/frontend/src/components/ErrorBoundary.tsx +++ b/frontend/src/components/ErrorBoundary.tsx @@ -1,52 +1,58 @@ -"use client"; +'use client'; -import React, { Component, ErrorInfo, ReactNode } from "react"; +import React, { Component, ErrorInfo, ReactNode } from 'react'; interface Props { - children: ReactNode; - fallback?: ReactNode; - name?: string; + children: ReactNode; + fallback?: ReactNode; + name?: string; } interface State { - hasError: boolean; - error: Error | null; + hasError: boolean; + error: Error | null; } class ErrorBoundary extends Component { - constructor(props: Props) { - super(props); - this.state = { hasError: false, error: null }; - } - - static getDerivedStateFromError(error: Error): State { - return { hasError: true, error }; - } - - componentDidCatch(error: Error, errorInfo: ErrorInfo) { - console.error(`[ErrorBoundary${this.props.name ? `: ${this.props.name}` : ""}]`, error, errorInfo); - } - - render() { - if (this.state.hasError) { - if (this.props.fallback) return this.props.fallback; - return ( -
-
-
⚠ SYSTEM ERROR
-
{this.props.name || "Component"} failed to render
- -
-
- ); - } - return this.props.children; + constructor(props: Props) { + super(props); + this.state = { hasError: false, error: null }; + } + + static getDerivedStateFromError(error: Error): State { + return { hasError: true, error }; + } + + componentDidCatch(error: Error, errorInfo: ErrorInfo) { + console.error( + `[ErrorBoundary${this.props.name ? `: ${this.props.name}` : ''}]`, + error, + errorInfo, + ); + } + + render() { + if (this.state.hasError) { + if (this.props.fallback) return this.props.fallback; + return ( +
+
+
⚠ SYSTEM ERROR
+
+ {this.props.name || 'Component'} failed to render +
+ +
+
+ ); } + return this.props.children; + } } export default ErrorBoundary; diff --git a/frontend/src/components/ExternalImage.tsx b/frontend/src/components/ExternalImage.tsx new file mode 100644 index 00000000..ca68bed4 --- /dev/null +++ b/frontend/src/components/ExternalImage.tsx @@ -0,0 +1,35 @@ +'use client'; + +import Image, { type ImageLoaderProps, type ImageProps } from 'next/image'; + +const passthroughLoader = ({ src }: ImageLoaderProps) => src; + +type ExternalImageProps = Omit & { + unoptimized?: boolean; +}; + +export default function ExternalImage({ + unoptimized = true, + alt = '', + fill, + width, + height, + ...rest +}: ExternalImageProps) { + if (fill) { + return ( + {alt} + ); + } + + return ( + {alt} + ); +} diff --git a/frontend/src/components/FilterPanel.tsx b/frontend/src/components/FilterPanel.tsx index 75c2fa54..b302fa13 100644 --- a/frontend/src/components/FilterPanel.tsx +++ b/frontend/src/components/FilterPanel.tsx @@ -1,338 +1,398 @@ -"use client"; +'use client'; -import { useState, useMemo } from 'react'; +import React, { useState, useMemo } from 'react'; import { motion, AnimatePresence } from 'framer-motion'; -import { ChevronUp, Filter, Plane, Shield, Star, Ship, SlidersHorizontal } from 'lucide-react'; +import { + ChevronUp, + ChevronDown, + Filter, + Plane, + Shield, + Star, + Ship, + SlidersHorizontal, +} from 'lucide-react'; import AdvancedFilterModal from './AdvancedFilterModal'; +import { useDataKeys } from '@/hooks/useDataStore'; import { airlineNames } from '../lib/airlineCodes'; import { trackedCategories, trackedOperators } from '../lib/trackedData'; interface FilterPanelProps { - data: any; - activeFilters: Record; - setActiveFilters: (filters: Record) => void; + activeFilters: Record; + setActiveFilters: (filters: Record) => void; } type ModalConfig = { - title: string; - icon: React.ReactNode; - accentColor: string; - accentColorName: string; - fields: { key: string; label: string; options: string[]; optionLabels?: Record }[]; + title: string; + icon: React.ReactNode; + accentColor: string; + accentColorName: string; + fields: { + key: string; + label: string; + options: string[]; + optionLabels?: Record; + }[]; }; -export default function FilterPanel({ data, activeFilters, setActiveFilters }: FilterPanelProps) { - const [isMinimized, setIsMinimized] = useState(true); - const [openModal, setOpenModal] = useState(null); +const FilterPanel = React.memo(function FilterPanel({ activeFilters, setActiveFilters }: FilterPanelProps) { + const data = useDataKeys(['commercial_flights', 'private_flights', 'private_jets', 'military_flights', 'tracked_flights', 'ships'] as const); + const [isMinimized, setIsMinimized] = useState(true); + const [openModal, setOpenModal] = useState(null); - // ── Extract unique values from live data ── + // ── Extract unique values from live data ── - // Commercial: departures, arrivals, airlines - const uniqueOrigins = useMemo(() => { - const origins = new Set(); - for (const f of data?.commercial_flights || []) { - if (f.origin_name && f.origin_name !== 'UNKNOWN') origins.add(f.origin_name); - } - return Array.from(origins).sort(); - }, [data?.commercial_flights]); + // Commercial: departures, arrivals, airlines + const uniqueOrigins = useMemo(() => { + const origins = new Set(); + for (const f of data?.commercial_flights || []) { + if (f.origin_name && f.origin_name !== 'UNKNOWN') origins.add(f.origin_name); + } + return Array.from(origins).sort(); + }, [data?.commercial_flights]); - const uniqueDestinations = useMemo(() => { - const dests = new Set(); - for (const f of data?.commercial_flights || []) { - if (f.dest_name && f.dest_name !== 'UNKNOWN') dests.add(f.dest_name); - } - return Array.from(dests).sort(); - }, [data?.commercial_flights]); + const uniqueDestinations = useMemo(() => { + const dests = new Set(); + for (const f of data?.commercial_flights || []) { + if (f.dest_name && f.dest_name !== 'UNKNOWN') dests.add(f.dest_name); + } + return Array.from(dests).sort(); + }, [data?.commercial_flights]); - const uniqueAirlines = useMemo(() => { - const airlines = new Set(); - for (const f of data?.commercial_flights || []) { - if (f.airline_code && f.airline_code.trim()) airlines.add(f.airline_code.trim()); - } - return Array.from(airlines).sort(); - }, [data?.commercial_flights]); + const uniqueAirlines = useMemo(() => { + const airlines = new Set(); + for (const f of data?.commercial_flights || []) { + if (f.airline_code && f.airline_code.trim()) airlines.add(f.airline_code.trim()); + } + return Array.from(airlines).sort(); + }, [data?.commercial_flights]); - const airlineLabels = useMemo(() => { - const labels: Record = {}; - for (const code of uniqueAirlines) { - const name = airlineNames[code]; - if (name) { - labels[code] = `${code} - ${name}`; - } else { - labels[code] = code; - } - } - return labels; - }, [uniqueAirlines]); + const airlineLabels = useMemo(() => { + const labels: Record = {}; + for (const code of uniqueAirlines) { + const name = airlineNames[code]; + if (name) { + labels[code] = `${code} - ${name}`; + } else { + labels[code] = code; + } + } + return labels; + }, [uniqueAirlines]); - // Private: callsigns + aircraft types - const uniquePrivateCallsigns = useMemo(() => { - const callsigns = new Set(); - for (const f of [...(data?.private_flights || []), ...(data?.private_jets || [])]) { - if (f.callsign) callsigns.add(f.callsign); - if (f.registration) callsigns.add(f.registration); - } - return Array.from(callsigns).sort(); - }, [data?.private_flights, data?.private_jets]); + // Private: callsigns + aircraft types + const uniquePrivateCallsigns = useMemo(() => { + const callsigns = new Set(); + for (const f of [...(data?.private_flights || []), ...(data?.private_jets || [])]) { + if (f.callsign) callsigns.add(f.callsign); + if (f.registration) callsigns.add(f.registration); + } + return Array.from(callsigns).sort(); + }, [data?.private_flights, data?.private_jets]); - const uniquePrivateAircraftTypes = useMemo(() => { - const types = new Set(); - for (const f of [...(data?.private_flights || []), ...(data?.private_jets || [])]) { - if (f.model && f.model !== 'Unknown') types.add(f.model); - } - return Array.from(types).sort(); - }, [data?.private_flights, data?.private_jets]); + const uniquePrivateAircraftTypes = useMemo(() => { + const types = new Set(); + for (const f of [...(data?.private_flights || []), ...(data?.private_jets || [])]) { + if (f.model && f.model !== 'Unknown') types.add(f.model); + } + return Array.from(types).sort(); + }, [data?.private_flights, data?.private_jets]); - // Military: country + aircraft type - const uniqueMilCountries = useMemo(() => { - const countries = new Set(); - for (const f of data?.military_flights || []) { - if (f.country) countries.add(f.country); - else if (f.registration) countries.add(f.registration); - } - return Array.from(countries).sort(); - }, [data?.military_flights]); + // Military: country + aircraft type + const uniqueMilCountries = useMemo(() => { + const countries = new Set(); + for (const f of data?.military_flights || []) { + if (f.country) countries.add(f.country); + else if (f.registration) countries.add(f.registration); + } + return Array.from(countries).sort(); + }, [data?.military_flights]); - const uniqueMilAircraftTypes = useMemo(() => { - const types = new Set(); - for (const f of data?.military_flights || []) { - if (f.military_type && f.military_type !== 'default') types.add(f.military_type); - } - return Array.from(types).sort(); - }, [data?.military_flights]); + const uniqueMilAircraftTypes = useMemo(() => { + const types = new Set(); + for (const f of data?.military_flights || []) { + if (f.military_type && f.military_type !== 'default') types.add(f.military_type); + } + return Array.from(types).sort(); + }, [data?.military_flights]); - // Tracked: operators + categories - const uniqueTrackedOperators = useMemo(() => { - const ops = new Set(trackedOperators); - for (const f of data?.tracked_flights || []) { - if (f.alert_operator) ops.add(f.alert_operator); - if (f.alert_tags) ops.add(f.alert_tags); - } - return Array.from(ops).sort(); - }, [data?.tracked_flights]); + // Tracked: operators + categories + const uniqueTrackedOperators = useMemo(() => { + const ops = new Set(trackedOperators); + for (const f of data?.tracked_flights || []) { + if (f.alert_operator) ops.add(f.alert_operator); + if (f.alert_tags) for (const t of f.alert_tags) ops.add(t); + } + return Array.from(ops).sort(); + }, [data?.tracked_flights]); - const uniqueTrackedCategories = useMemo(() => { - const cats = new Set(trackedCategories); - for (const f of data?.tracked_flights || []) { - if (f.alert_category) cats.add(f.alert_category); - } - return Array.from(cats).sort(); - }, [data?.tracked_flights]); + const uniqueTrackedCategories = useMemo(() => { + const cats = new Set(trackedCategories); + for (const f of data?.tracked_flights || []) { + if (f.alert_category) cats.add(f.alert_category); + } + return Array.from(cats).sort(); + }, [data?.tracked_flights]); - // Maritime: vessel names + vessel types (using 'type' field, not 'ship_type') - const uniqueShipNames = useMemo(() => { - const names = new Set(); - for (const s of data?.ships || []) { - if (s.name && s.name !== 'UNKNOWN') names.add(s.name); - } - return Array.from(names).sort(); - }, [data?.ships]); + // Maritime: vessel names + vessel types (using 'type' field, not 'ship_type') + const uniqueShipNames = useMemo(() => { + const names = new Set(); + for (const s of data?.ships || []) { + if (s.name && s.name !== 'UNKNOWN') names.add(s.name); + } + return Array.from(names).sort(); + }, [data?.ships]); - const uniqueVesselTypes = useMemo(() => { - const types = new Set(); - for (const s of data?.ships || []) { - // Use 'type' field from AIS stream (tanker, cargo, passenger, yacht, etc.) - if (s.type && s.type !== 'unknown') types.add(s.type); - } - return Array.from(types).sort(); - }, [data?.ships]); + const uniqueVesselTypes = useMemo(() => { + const types = new Set(); + for (const s of data?.ships || []) { + // Use 'type' field from AIS stream (tanker, cargo, passenger, yacht, etc.) + if (s.type && s.type !== 'unknown') types.add(s.type); + } + return Array.from(types).sort(); + }, [data?.ships]); - // ── Modal configs ── + // ── Modal configs ── - const modalConfigs: Record = { - commercial: { - title: 'COMMERCIAL FLIGHTS', - icon: , - accentColor: '#00bcd4', - accentColorName: 'cyan', - fields: [ - { key: 'commercial_departure', label: 'DEPARTURE', options: uniqueOrigins }, - { key: 'commercial_arrival', label: 'ARRIVAL', options: uniqueDestinations }, - { key: 'commercial_airline', label: 'AIRLINE', options: uniqueAirlines, optionLabels: airlineLabels }, - ] + const modalConfigs: Record = { + commercial: { + title: 'COMMERCIAL FLIGHTS', + icon: , + accentColor: '#00bcd4', + accentColorName: 'cyan', + fields: [ + { key: 'commercial_departure', label: 'DEPARTURE', options: uniqueOrigins }, + { key: 'commercial_arrival', label: 'ARRIVAL', options: uniqueDestinations }, + { + key: 'commercial_airline', + label: 'AIRLINE', + options: uniqueAirlines, + optionLabels: airlineLabels, }, - private: { - title: 'PRIVATE / JETS', - icon: , - accentColor: '#FF8C00', - accentColorName: 'orange', - fields: [ - { key: 'private_callsign', label: 'CALLSIGN / REG', options: uniquePrivateCallsigns }, - { key: 'private_aircraft_type', label: 'AIRCRAFT TYPE', options: uniquePrivateAircraftTypes }, - ] + ], + }, + private: { + title: 'PRIVATE / JETS', + icon: , + accentColor: '#FF8C00', + accentColorName: 'orange', + fields: [ + { key: 'private_callsign', label: 'CALLSIGN / REG', options: uniquePrivateCallsigns }, + { + key: 'private_aircraft_type', + label: 'AIRCRAFT TYPE', + options: uniquePrivateAircraftTypes, }, - military: { - title: 'MILITARY', - icon: , - accentColor: '#EAB308', - accentColorName: 'yellow', - fields: [ - { key: 'military_country', label: 'COUNTRY / REG', options: uniqueMilCountries }, - { key: 'military_aircraft_type', label: 'AIRCRAFT TYPE', options: uniqueMilAircraftTypes }, - ] - }, - tracked: { - title: 'TRACKED AIRCRAFT', - icon: , - accentColor: '#EC4899', - accentColorName: 'pink', - fields: [ - { key: 'tracked_category', label: 'CATEGORY', options: uniqueTrackedCategories }, - { key: 'tracked_owner', label: 'OPERATOR / ENTITY', options: uniqueTrackedOperators }, - ] - }, - ships: { - title: 'MARITIME VESSELS', - icon: , - accentColor: '#3B82F6', - accentColorName: 'blue', - fields: [ - { key: 'ship_name', label: 'VESSEL NAME', options: uniqueShipNames }, - { key: 'ship_type', label: 'VESSEL TYPE', options: uniqueVesselTypes }, - ] - } - }; + ], + }, + military: { + title: 'MILITARY', + icon: , + accentColor: '#EAB308', + accentColorName: 'yellow', + fields: [ + { key: 'military_country', label: 'COUNTRY / REG', options: uniqueMilCountries }, + { key: 'military_aircraft_type', label: 'AIRCRAFT TYPE', options: uniqueMilAircraftTypes }, + ], + }, + tracked: { + title: 'TRACKED AIRCRAFT', + icon: , + accentColor: '#EC4899', + accentColorName: 'pink', + fields: [ + { key: 'tracked_category', label: 'CATEGORY', options: uniqueTrackedCategories }, + { key: 'tracked_owner', label: 'OPERATOR / ENTITY', options: uniqueTrackedOperators }, + ], + }, + ships: { + title: 'MARITIME VESSELS', + icon: , + accentColor: '#3B82F6', + accentColorName: 'blue', + fields: [ + { key: 'ship_name', label: 'VESSEL NAME', options: uniqueShipNames }, + { key: 'ship_type', label: 'VESSEL TYPE', options: uniqueVesselTypes }, + ], + }, + }; - const clearAll = () => setActiveFilters({}); + const clearAll = () => setActiveFilters({}); - const activeCount = Object.values(activeFilters).reduce((acc, arr) => acc + arr.length, 0); + const activeCount = Object.values(activeFilters).reduce((acc, arr) => acc + arr.length, 0); - const getCountForCategory = (category: string) => { - const config = modalConfigs[category]; - if (!config) return 0; - return config.fields.reduce((acc, f) => acc + (activeFilters[f.key]?.length || 0), 0); - }; + const getCountForCategory = (category: string) => { + const config = modalConfigs[category]; + if (!config) return 0; + return config.fields.reduce((acc, f) => acc + (activeFilters[f.key]?.length || 0), 0); + }; - const handleModalApply = (categoryKey: string, modalFilters: Record) => { - const config = modalConfigs[categoryKey]; - const next = { ...activeFilters }; - for (const field of config.fields) { - delete next[field.key]; - } - for (const [key, values] of Object.entries(modalFilters)) { - if (values.length > 0) next[key] = values; - } - setActiveFilters(next); - }; + const handleModalApply = (categoryKey: string, modalFilters: Record) => { + const config = modalConfigs[categoryKey]; + const next = { ...activeFilters }; + for (const field of config.fields) { + delete next[field.key]; + } + for (const [key, values] of Object.entries(modalFilters)) { + if (values.length > 0) next[key] = values; + } + setActiveFilters(next); + }; - const sections = [ - { key: 'commercial', title: 'COMMERCIAL FLIGHTS', icon: , color: 'cyan' }, - { key: 'private', title: 'PRIVATE / JETS', icon: , color: 'orange' }, - { key: 'military', title: 'MILITARY', icon: , color: 'yellow' }, - { key: 'tracked', title: 'TRACKED AIRCRAFT', icon: , color: 'pink' }, - { key: 'ships', title: 'MARITIME VESSELS', icon: , color: 'blue' }, - ]; + const sections = [ + { + key: 'commercial', + title: 'COMMERCIAL FLIGHTS', + icon: , + color: 'cyan', + }, + { + key: 'private', + title: 'PRIVATE / JETS', + icon: , + color: 'orange', + }, + { + key: 'military', + title: 'MILITARY', + icon: , + color: 'yellow', + }, + { + key: 'tracked', + title: 'TRACKED AIRCRAFT', + icon: , + color: 'pink', + }, + { + key: 'ships', + title: 'MARITIME VESSELS', + icon: , + color: 'blue', + }, + ]; - const borderColors: Record = { - cyan: 'border-cyan-500/20 hover:border-cyan-500/40', - orange: 'border-orange-500/20 hover:border-orange-500/40', - yellow: 'border-yellow-500/20 hover:border-yellow-500/40', - pink: 'border-pink-500/20 hover:border-pink-500/40', - blue: 'border-blue-500/20 hover:border-blue-500/40', - }; - const textColors: Record = { - cyan: 'text-cyan-400', - orange: 'text-orange-400', - yellow: 'text-yellow-400', - pink: 'text-pink-400', - blue: 'text-blue-400', - }; - const bgColors: Record = { - cyan: 'bg-cyan-500/10', - orange: 'bg-orange-500/10', - yellow: 'bg-yellow-500/10', - pink: 'bg-pink-500/10', - blue: 'bg-blue-500/10', - }; + const borderColors: Record = { + cyan: 'border-cyan-500/20 hover:border-cyan-500/40', + orange: 'border-orange-500/20 hover:border-orange-500/40', + yellow: 'border-yellow-500/20 hover:border-yellow-500/40', + pink: 'border-pink-500/20 hover:border-pink-500/40', + blue: 'border-blue-500/20 hover:border-blue-500/40', + }; + const textColors: Record = { + cyan: 'text-cyan-400', + orange: 'text-orange-400', + yellow: 'text-yellow-400', + pink: 'text-pink-400', + blue: 'text-blue-400', + }; + const bgColors: Record = { + cyan: 'bg-cyan-500/10', + orange: 'bg-orange-500/10', + yellow: 'bg-yellow-500/10', + pink: 'bg-pink-500/10', + blue: 'bg-blue-500/10', + }; - return ( - <> + return ( + <> + + {/* Header Toggle */} +
setIsMinimized(!isMinimized)} + > +
+ + + DATA FILTERS + + {activeCount > 0 && ( + + {activeCount} ACTIVE + + )} +
+ +
+ + + {!isMinimized && ( - {/* Header Toggle */} -
setIsMinimized(!isMinimized)} + {activeCount > 0 && ( + + )} + + {sections.map((section) => { + const count = getCountForCategory(section.key); + return ( +
setOpenModal(section.key)} + > +
+
+ {section.icon} + + {section.title} + + {count > 0 && ( + + {count} + )} +
+
- -
- - - {!isMinimized && ( - - {activeCount > 0 && ( - - )} - - {sections.map(section => { - const count = getCountForCategory(section.key); - return ( -
setOpenModal(section.key)} - > -
-
- {section.icon} - {section.title} - {count > 0 && ( - - {count} - - )} -
- -
-
- ); - })} - - )} - +
+ ); + })} + )} + + - {/* Render active modal */} - - {openModal && modalConfigs[openModal] && ( - handleModalApply(openModal, filters)} - onClose={() => setOpenModal(null)} - /> - )} - - - ); -} + {/* Render active modal */} + + {openModal && modalConfigs[openModal] && ( + handleModalApply(openModal, filters)} + onClose={() => setOpenModal(null)} + /> + )} + + + ); +}); + +export default FilterPanel; diff --git a/frontend/src/components/FindLocateBar.tsx b/frontend/src/components/FindLocateBar.tsx index 84d4bfb1..e8fa99f2 100644 --- a/frontend/src/components/FindLocateBar.tsx +++ b/frontend/src/components/FindLocateBar.tsx @@ -1,244 +1,264 @@ -"use client"; +'use client'; -import { useState, useMemo, useRef, useEffect } from "react"; -import { Search, Crosshair, Plane, Shield, Star, Ship, X, Database } from "lucide-react"; -import { motion, AnimatePresence } from "framer-motion"; +import React, { useState, useMemo, useRef, useEffect } from 'react'; +import { Search, Crosshair, Plane, Shield, Star, Ship, X, Database } from 'lucide-react'; +import { motion, AnimatePresence } from 'framer-motion'; import { trackedOperators } from '../lib/trackedData'; +import { useDataKeys } from '@/hooks/useDataStore'; interface FindLocateBarProps { - data: any; - onLocate: (lat: number, lng: number, entityId: string, entityType: string) => void; - onFilter?: (filterType: string, filterValue: string) => void; + onLocate: (lat: number, lng: number, entityId: string, entityType: string) => void; + onFilter?: (filterType: string, filterValue: string) => void; } interface SearchResult { - id: string; - label: string; - sublabel: string; - category: string; - categoryColor: string; - lat: number; - lng: number; - entityType: string; + id: string; + label: string; + sublabel: string; + category: string; + categoryColor: string; + lat: number; + lng: number; + entityType: string; + extra?: string; } -export default function FindLocateBar({ data, onLocate, onFilter }: FindLocateBarProps) { - const [query, setQuery] = useState(""); - const [isOpen, setIsOpen] = useState(false); - const inputRef = useRef(null); - const containerRef = useRef(null); - - // Close dropdown when clicking outside - useEffect(() => { - const handler = (e: MouseEvent) => { - if (containerRef.current && !containerRef.current.contains(e.target as Node)) { - setIsOpen(false); - } - }; - document.addEventListener("mousedown", handler); - return () => document.removeEventListener("mousedown", handler); - }, []); - - // Build searchable index from all data - const allEntities = useMemo(() => { - const results: SearchResult[] = []; - - // Commercial flights - for (const f of data?.commercial_flights || []) { - const uid = f.icao24 || f.registration || f.callsign || ''; - results.push({ - id: `flight-${uid}`, - label: f.callsign || uid, - sublabel: `${f.model || 'Unknown'} · ${f.airline_code || 'Commercial'}`, - category: "COMMERCIAL", - categoryColor: "text-cyan-400", - lat: f.lat, - lng: f.lng, - entityType: "flight", - }); - } - - // Private flights - for (const f of [...(data?.private_flights || []), ...(data?.private_jets || [])]) { - const uid = f.icao24 || f.registration || f.callsign || ''; - const type = f.type === 'private_jet' ? 'private_jet' : 'private_flight'; - results.push({ - id: `${type === 'private_jet' ? 'private-jet' : 'private-flight'}-${uid}`, - label: f.callsign || f.registration || uid, - sublabel: `${f.model || 'Unknown'} · Private`, - category: "PRIVATE", - categoryColor: "text-orange-400", - lat: f.lat, - lng: f.lng, - entityType: type, - }); - } - - // Military flights - for (const f of data?.military_flights || []) { - const uid = f.icao24 || f.registration || f.callsign || ''; - results.push({ - id: `mil-flight-${uid}`, - label: f.callsign || uid, - sublabel: `${f.model || 'Unknown'} · ${f.military_type || 'Military'}`, - category: "MILITARY", - categoryColor: "text-yellow-400", - lat: f.lat, - lng: f.lng, - entityType: "military_flight", - }); - } - - // Tracked flights — include tags/owner/name for broad search (first name, last name, etc.) - for (const f of data?.tracked_flights || []) { - const uid = f.icao24 || f.registration || f.callsign || ''; - const operator = f.alert_operator || 'Unknown Operator'; - const category = f.alert_category || 'Tracked'; - const type = f.alert_type || f.model || 'Unknown'; - const extras = [f.alert_tags, f.owner, f.name, f.callsign].filter(Boolean).join(' '); - results.push({ - id: `tracked-${uid}`, - label: operator, - sublabel: `${category} · ${type} (${f.registration || uid})`, - category: "TRACKED", - categoryColor: "text-pink-400", - lat: f.lat, - lng: f.lng, - entityType: "tracked_flight", - _extra: extras, - } as any); - } - - // Ships - for (const s of data?.ships || []) { - results.push({ - id: `ship-${s.mmsi || s.name || ''}`, - label: s.name || "UNKNOWN", - sublabel: `${s.type || 'Vessel'} · ${s.destination || 'Unknown dest'}`, - category: "MARITIME", - categoryColor: "text-blue-400", - lat: s.lat, - lng: s.lng, - entityType: "ship", - }); - } - - // Database Records - Tracked Operators - for (const op of trackedOperators) { - results.push({ - id: `tracked-db-${op}`, - label: op, - sublabel: `Database Record · Operator`, - category: "DATABASE", - categoryColor: "text-purple-400", - lat: 0, - lng: 0, - entityType: "database_operator", - }); - } - - return results; - }, [data]); - - // Filter results based on query - const filtered = useMemo(() => { - if (!query.trim()) return []; - const q = query.toLowerCase(); - return allEntities - .filter(e => { - const searchable = `${e.label} ${e.sublabel} ${e.id} ${(e as any)._extra || ''}`.toLowerCase(); - return searchable.includes(q); - }) - .slice(0, 12); - }, [query, allEntities]); - - const handleSelect = (result: SearchResult) => { - if (result.entityType === "database_operator") { - if (onFilter) onFilter("tracked_owner", result.label); - } else { - onLocate(result.lat, result.lng, result.id, result.entityType); - } - setQuery(""); +const FindLocateBar = React.memo(function FindLocateBar({ onLocate, onFilter }: FindLocateBarProps) { + const data = useDataKeys(['commercial_flights', 'private_flights', 'private_jets', 'military_flights', 'tracked_flights', 'ships'] as const); + const [query, setQuery] = useState(''); + const [isOpen, setIsOpen] = useState(false); + const inputRef = useRef(null); + const containerRef = useRef(null); + + // Close dropdown when clicking outside + useEffect(() => { + const handler = (e: MouseEvent) => { + if (containerRef.current && !containerRef.current.contains(e.target as Node)) { setIsOpen(false); + } }; + document.addEventListener('mousedown', handler); + return () => document.removeEventListener('mousedown', handler); + }, []); - const categoryIcons: Record = { - COMMERCIAL: , - PRIVATE: , - MILITARY: , - TRACKED: , - MARITIME: , - DATABASE: , - }; + // Build searchable index from all data + const allEntities = useMemo(() => { + const results: SearchResult[] = []; + + // Commercial flights + for (const f of data?.commercial_flights || []) { + const uid = f.icao24 || f.registration || f.callsign || ''; + results.push({ + id: `flight-${uid}`, + label: f.callsign || uid, + sublabel: `${f.model || 'Unknown'} · ${f.airline_code || 'Commercial'}`, + category: 'COMMERCIAL', + categoryColor: 'text-cyan-400', + lat: f.lat, + lng: f.lng, + entityType: 'flight', + }); + } + + // Private flights + for (const f of [...(data?.private_flights || []), ...(data?.private_jets || [])]) { + const uid = f.icao24 || f.registration || f.callsign || ''; + const type = f.type === 'private_jet' ? 'private_jet' : 'private_flight'; + results.push({ + id: `${type === 'private_jet' ? 'private-jet' : 'private-flight'}-${uid}`, + label: f.callsign || f.registration || uid, + sublabel: `${f.model || 'Unknown'} · Private`, + category: 'PRIVATE', + categoryColor: 'text-orange-400', + lat: f.lat, + lng: f.lng, + entityType: type, + }); + } + + // Military flights + for (const f of data?.military_flights || []) { + const uid = f.icao24 || f.registration || f.callsign || ''; + results.push({ + id: `mil-flight-${uid}`, + label: f.callsign || uid, + sublabel: `${f.model || 'Unknown'} · ${f.military_type || 'Military'}`, + category: 'MILITARY', + categoryColor: 'text-yellow-400', + lat: f.lat, + lng: f.lng, + entityType: 'military_flight', + }); + } + + // Tracked flights — include tags/owner/name for broad search (first name, last name, etc.) + for (const f of data?.tracked_flights || []) { + const uid = f.icao24 || f.registration || f.callsign || ''; + const operator = f.alert_operator || 'Unknown Operator'; + const category = f.alert_category || 'Tracked'; + const type = f.alert_type || f.model || 'Unknown'; + const extras = [f.alert_operator, f.alert_tags, f.owner, f.name, f.callsign, f.registration].filter(Boolean).join(' '); + results.push({ + id: `tracked-${uid}`, + label: operator, + sublabel: `${category} · ${type} (${f.registration || uid})`, + category: 'TRACKED', + categoryColor: 'text-pink-400', + lat: f.lat, + lng: f.lng, + entityType: 'tracked_flight', + extra: extras, + }); + } + + // Ships + for (const s of data?.ships || []) { + results.push({ + id: `ship-${s.mmsi || s.name || ''}`, + label: s.name || 'UNKNOWN', + sublabel: `${s.type || 'Vessel'} · ${s.destination || 'Unknown dest'}`, + category: 'MARITIME', + categoryColor: 'text-blue-400', + lat: s.lat, + lng: s.lng, + entityType: 'ship', + }); + } + + // Database Records - Tracked Operators + for (const op of trackedOperators) { + results.push({ + id: `tracked-db-${op}`, + label: op, + sublabel: `Database Record · Operator`, + category: 'DATABASE', + categoryColor: 'text-purple-400', + lat: 0, + lng: 0, + entityType: 'database_operator', + }); + } - return ( -
-
- - { - setQuery(e.target.value); - setIsOpen(true); - }} - onFocus={() => setIsOpen(true)} - /> - {query && ( - - )} - + return results; + }, [data]); + + // Filter results based on query + const filtered = useMemo(() => { + if (!query.trim()) return []; + const q = query.toLowerCase(); + return allEntities + .filter((e) => { + const searchable = `${e.label} ${e.sublabel} ${e.id} ${e.extra || ''}`.toLowerCase(); + return searchable.includes(q); + }) + .slice(0, 12); + }, [query, allEntities]); + + const handleSelect = (result: SearchResult) => { + if (result.entityType === 'database_operator') { + if (onFilter) onFilter('tracked_owner', result.label); + } else { + onLocate(result.lat, result.lng, result.id, result.entityType); + } + setQuery(''); + setIsOpen(false); + }; + + const categoryIcons: Record = { + COMMERCIAL: , + PRIVATE: , + MILITARY: , + TRACKED: , + MARITIME: , + DATABASE: , + }; + + return ( +
+
+ + { + setQuery(e.target.value); + setIsOpen(true); + }} + onFocus={() => setIsOpen(true)} + /> + {query && ( + + )} + +
+ + + {isOpen && filtered.length > 0 && ( + +
+ {filtered.map((r, idx) => ( + + ))} +
+
+ {filtered.length} RESULT{filtered.length !== 1 ? 'S' : ''} — CLICK TO LOCATE
+ + )} + {isOpen && query.trim() && filtered.length === 0 && ( + +
+ NO MATCHING ASSETS +
+ + )} + +
+ ); +}); - - {isOpen && filtered.length > 0 && ( - -
- {filtered.map((r, idx) => ( - - ))} -
-
- {filtered.length} RESULT{filtered.length !== 1 ? 'S' : ''} — CLICK TO LOCATE -
- - )} - {isOpen && query.trim() && filtered.length === 0 && ( - -
NO MATCHING ASSETS
- - )} - -
- ); -} +export default FindLocateBar; diff --git a/frontend/src/components/GlobalTicker.tsx b/frontend/src/components/GlobalTicker.tsx new file mode 100644 index 00000000..690590e5 --- /dev/null +++ b/frontend/src/components/GlobalTicker.tsx @@ -0,0 +1,77 @@ +'use client'; + +import { useState } from 'react'; +import { motion } from 'framer-motion'; +import { ArrowUpRight, ArrowDownRight, TrendingUp, AlertTriangle, ChevronUp } from 'lucide-react'; +import { useDataKeys } from '@/hooks/useDataStore'; + +export default function GlobalTicker() { + const { stocks, financial_source } = useDataKeys(['stocks', 'financial_source'] as const); + const entries = Object.entries(stocks || {}); + const fallback = financial_source === 'yfinance'; + + if (entries.length === 0) return null; + + // Render a single ticker item + const renderItem = ([ticker, info]: [string, any], index: number) => { + // Determine color based on price action + let colorClass = 'text-white'; + if (info.change_percent > 0) colorClass = 'text-green-400'; + if (info.change_percent < 0) colorClass = 'text-red-400'; + + const isCryptoHighlight = ticker === 'BTC' || ticker === 'ETH'; + + return ( +
+ + {isCryptoHighlight && ★} + {ticker} + + + ${(info.price ?? 0).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })} + + + {info.up ? : info.change_percent < 0 ? : } + {Math.abs(info.change_percent ?? 0).toFixed(2)}% + +
+ ); + }; + + + return ( +
+ + {fallback && ( +
+
+ + + SYS WARN: FINNHUB API KEY MISSING — YAHOO FALLBACK ACTIVE (LIMITED) + +
+
+ )} + + {/* The scrolling container */} + + {/* Render the list twice for seamless infinite scrolling */} +
+ {entries.map((item, i) => renderItem(item, i))} +
+
+ {entries.map((item, i) => renderItem(item, i + entries.length))} +
+ +
+ ); +} diff --git a/frontend/src/components/HlsVideo.tsx b/frontend/src/components/HlsVideo.tsx new file mode 100644 index 00000000..00075543 --- /dev/null +++ b/frontend/src/components/HlsVideo.tsx @@ -0,0 +1,66 @@ +'use client'; + +import { useEffect, useRef, forwardRef, useImperativeHandle } from 'react'; + +export interface HlsVideoHandle { + play(): void; + pause(): void; + get paused(): boolean; +} + +const HlsVideo = forwardRef void }>( + ({ url, className, onError }, ref) => { + const videoRef = useRef(null); + + useImperativeHandle(ref, () => ({ + play: () => videoRef.current?.play(), + pause: () => videoRef.current?.pause(), + get paused() { + return videoRef.current?.paused ?? true; + }, + })); + + useEffect(() => { + const video = videoRef.current; + if (!video || !url) return; + + let hlsInstance: { destroy(): void } | null = null; + let cancelled = false; + + (async () => { + const { default: Hls } = await import('hls.js'); + if (cancelled) return; + if (Hls.isSupported()) { + const hls = new Hls({ enableWorker: false, lowLatencyMode: true }); + hls.on(Hls.Events.ERROR, (_e: unknown, data: { fatal?: boolean }) => { + if (data.fatal) onError?.(); + }); + hls.loadSource(url); + hls.attachMedia(video); + hlsInstance = hls; + } else if (video.canPlayType('application/vnd.apple.mpegurl')) { + video.src = url; + } + })(); + + return () => { + cancelled = true; + hlsInstance?.destroy(); + }; + }, [url, onError]); + + return ( +