diff --git a/.env b/.env index 9695c8a8..13085612 100644 --- a/.env +++ b/.env @@ -128,6 +128,7 @@ REFRAG_RUNTIME=llamacpp REFRAG_ENCODER_MODEL=BAAI/bge-base-en-v1.5 REFRAG_PHI_PATH=/work/models/refrag_phi_768_to_dmodel.bin REFRAG_SENSE=heuristic +REFRAG_PSEUDO_DESCRIBE=1 GLM_API_KEY= # Llama.cpp sidecar (optional) # Use docker network hostname from containers; localhost remains ok for host-side runs if LLAMACPP_URL not exported @@ -135,6 +136,7 @@ LLAMACPP_URL=http://host.docker.internal:8081 LLAMACPP_TIMEOUT_SEC=300 DECODER_MAX_TOKENS=4000 REFRAG_DECODER_MODE=prompt # prompt|soft +REFRAG_COMMIT_DESCRIBE=1 REFRAG_SOFT_SCALE=1.0 LLAMACPP_USE_GPU=1 @@ -149,6 +151,8 @@ MAX_MICRO_CHUNKS_PER_FILE=500 QDRANT_TIMEOUT=20 MEMORY_AUTODETECT=1 MEMORY_COLLECTION_TTL_SECS=300 +SMART_SYMBOL_REINDEXING=1 +MAX_CHANGED_SYMBOLS_RATIO=0.6 # Watcher-safe defaults (recommended) @@ -196,4 +200,9 @@ INFO_REQUEST_LIMIT=10 INFO_REQUEST_CONTEXT_LINES=5 # INFO_REQUEST_EXPLAIN_DEFAULT=0 # INFO_REQUEST_RELATIONSHIPS=0 +GLM_API_BASE=https://api.z.ai/api/coding/paas/v4/ +GLM_MODEL=glm-4.6 COMMIT_VECTOR_SEARCH=0 +STRICT_MEMORY_RESTORE=1 +CTXCE_AUTH_ENABLED=0 +CTXCE_AUTH_ADMIN_TOKEN=change-me-admin-token diff --git a/.env.example b/.env.example index a53d3208..c759cff8 100644 --- a/.env.example +++ b/.env.example @@ -152,7 +152,7 @@ REFRAG_PHI_PATH=/work/models/refrag_phi_768_to_dmodel.json REFRAG_SENSE=heuristic # Enable index-time pseudo descriptions for micro-chunks (requires REFRAG_DECODER) -# REFRAG_PSEUDO_DESCRIBE=1 +REFRAG_PSEUDO_DESCRIBE=1 # Llama.cpp sidecar (optional) # Docker CPU-only (stable): http://llamacpp:8080 @@ -189,7 +189,8 @@ MEMORY_AUTODETECT=1 MEMORY_COLLECTION_TTL_SECS=300 # Smarter re-indexing for symbol cache, reuse embeddings and reduce decoder/pseudo tags to re-index -SMART_SYMBOL_REINDEXING=0 +SMART_SYMBOL_REINDEXING=1 +MAX_CHANGED_SYMBOLS_RATIO=0.6 # Watcher-safe defaults (recommended) # Applied to watcher via compose; uncomment to apply globally. @@ -226,7 +227,7 @@ SMART_SYMBOL_REINDEXING=0 REFRAG_COMMIT_DESCRIBE=1 COMMIT_VECTOR_SEARCH=0 -STRICT_MEMORY_RESTORE=0 +STRICT_MEMORY_RESTORE=1 # info_request() tool settings (simplified codebase retrieval) # Default result limit for info_request queries @@ -237,3 +238,60 @@ INFO_REQUEST_CONTEXT_LINES=5 # INFO_REQUEST_EXPLAIN_DEFAULT=0 # Enable relationship mapping by default (imports_from, calls, related_paths) # INFO_REQUEST_RELATIONSHIPS=0 + + +# --------------------------------------------------------------------------- +# Optional Auth & Bridge Configuration (OFF by default) +# --------------------------------------------------------------------------- + +# Global auth toggle for backend services (upload_service, MCP indexer/memory). +# When set to 1, services will require a valid session for protected operations. +# When unset or 0, auth is disabled and behavior matches previous versions. +# CTXCE_AUTH_ENABLED=0 + +# Shared token used by /auth/login for the bridge and other service clients. +# When CTXCE_AUTH_ENABLED=1 and this is set, /auth/login will only issue +# sessions when the provided token matches this value. +# CTXCE_AUTH_SHARED_TOKEN=change-me-dev-token + +# Create "open-dev" tokens when a shared token is unset, basically allows requesting a session with no real auth +# CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=0 + +# Optional admin token for creating additional users via /auth/users once the +# first user has been bootstrapped. If unset, only the initial user can be +# created (no additional users). +# CTXCE_AUTH_ADMIN_TOKEN=change-me-admin-token + +# Auth database location (default: sqlite file under WORK_DIR/.codebase). +# Use a SQLite URL for local/dev, or point to a different path. +# CTXCE_AUTH_DB_URL=sqlite:////work/.codebase/ctxce_auth.sqlite + +# Session TTL (seconds) for issued auth sessions. +# 0 or negative values disable expiry (sessions do not expire). When >0, +# active sessions are extended with a sliding window whenever they are used. +# CTXCE_AUTH_SESSION_TTL_SECONDS=0 + +# Collection registry & ACL (only used when CTXCE_AUTH_ENABLED=1). +# When enabled, infrastructure/health checks may populate an internal SQLite +# registry of known Qdrant collections, and ACL rules can be applied by services. + +# ACL bypass (dev/early deployments): allow all users to access all collections. +# CTXCE_ACL_ALLOW_ALL=0 + +# MCP boundary enforcement (OFF by default). +# When enabled, MCP servers will enforce collection ACLs using the auth DB. +# Requires CTXCE_AUTH_ENABLED=1. If CTXCE_ACL_ALLOW_ALL=1, enforcement is bypassed. +# This is intended for gradually rolling out collection-level permissions without +# changing existing client auth/session mechanisms. +# CTXCE_MCP_ACL_ENFORCE=0 + +# Bridge-side configuration (ctx-mcp-bridge): +# The bridge will POST to this URL for auth login and store the returned +# session id, then inject it into all MCP tool calls as the `session` field. +# CTXCE_AUTH_BACKEND_URL=http://localhost:8004 + +# Optional defaults for bridge CLI (env) auth: +# CTXCE_AUTH_TOKEN=dev-shared-token +# CTXCE_AUTH_USERNAME=you@example.com +# CTXCE_AUTH_PASSWORD=your-password + diff --git a/.github/workflows/publish-cli.yml b/.github/workflows/publish-cli.yml new file mode 100644 index 00000000..01f1aebb --- /dev/null +++ b/.github/workflows/publish-cli.yml @@ -0,0 +1,34 @@ +name: Publish ctxce CLI to npm + +on: + workflow_dispatch: + inputs: + version: + description: "Version to publish (ensure package.json is updated)" + required: false + push: + tags: + - "ctxce-cli-v*" + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: "20" + registry-url: "https://registry.npmjs.org" + + - name: Install dependencies + working-directory: ctx-mcp-bridge + run: npm install + + - name: Publish to npm + working-directory: ctx-mcp-bridge + run: npm publish --access public --provenance diff --git a/.gitignore b/.gitignore index 1e578035..f5bdce85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ # Model artifacts models/ +# Node +node_modules/ + # Qdrant snapshots qdrant/snapshots/ diff --git a/ctx-mcp-bridge/.gitignore b/ctx-mcp-bridge/.gitignore new file mode 100644 index 00000000..9e30eb9b --- /dev/null +++ b/ctx-mcp-bridge/.gitignore @@ -0,0 +1 @@ +*.tgz \ No newline at end of file diff --git a/ctx-mcp-bridge/README.md b/ctx-mcp-bridge/README.md new file mode 100644 index 00000000..95a24bec --- /dev/null +++ b/ctx-mcp-bridge/README.md @@ -0,0 +1,212 @@ +# Context Engine MCP Bridge + +`@context-engine-bridge/context-engine-mcp-bridge` provides the `ctxce` CLI, a +Model Context Protocol (MCP) bridge that speaks to the Context Engine indexer +and memory servers and exposes them as a single MCP server. + +It is primarily used by the VS Code **Context Engine Uploader** extension, +available on the Marketplace: + +- + +The bridge can also be run standalone (e.g. from a terminal, or wired into +other MCP clients) as long as the Context Engine stack is running. + +## Prerequisites + +- Node.js **>= 18** (see `engines` in `package.json`). +- A running Context Engine stack (e.g. via `docker-compose.dev-remote.yml`) with: + - MCP indexer HTTP endpoint (default: `http://localhost:8003/mcp`). + - MCP memory HTTP endpoint (optional, default: `http://localhost:8002/mcp`). +- For optional auth: + - The upload/auth services must be configured with `CTXCE_AUTH_ENABLED=1` and + a reachable auth backend URL (e.g. `http://localhost:8004`). + +## Installation + +You can install the package globally, or run it via `npx`. + +### Global install + +```bash +npm install -g @context-engine-bridge/context-engine-mcp-bridge +``` + +This installs the `ctxce` (and `ctxce-bridge`) CLI in your PATH. + +### Using npx (no global install) + +```bash +npx @context-engine-bridge/context-engine-mcp-bridge ctxce --help +``` + +The examples below assume `ctxce` is available on your PATH; if you use `npx`, +just prefix commands with `npx @context-engine-bridge/context-engine-mcp-bridge`. + +## CLI overview + +The main entrypoint is: + +```bash +ctxce [...args] +``` + +Supported commands (from `src/cli.js`): + +- `ctxce mcp-serve` – stdio MCP bridge (for stdio-based MCP clients). +- `ctxce mcp-http-serve` – HTTP MCP bridge (for HTTP-based MCP clients). +- `ctxce auth ` – auth helper commands (`login`, `status`, `logout`). + +### Environment variables + +These environment variables are respected by the bridge: + +- `CTXCE_INDEXER_URL` – MCP indexer URL (default: `http://localhost:8003/mcp`). +- `CTXCE_MEMORY_URL` – MCP memory URL, or empty/omitted to disable memory + (default: `http://localhost:8002/mcp`). +- `CTXCE_HTTP_PORT` – port for `mcp-http-serve` (default: `30810`). + +For auth (optional, shared with the upload/auth backend): + +- `CTXCE_AUTH_ENABLED` – whether auth is enabled in the backend. +- `CTXCE_AUTH_BACKEND_URL` – auth backend URL (e.g. `http://localhost:8004`). +- `CTXCE_AUTH_TOKEN` – dev/shared token for `ctxce auth login`. +- `CTXCE_AUTH_SESSION_TTL_SECONDS` – session TTL / sliding expiry (seconds). + +The CLI also stores auth sessions in `~/.ctxce/auth.json`, keyed by backend URL. + +## Running the MCP bridge (stdio) + +The stdio bridge is suitable for MCP clients that speak stdio directly (for +example, certain editors or tools that expect an MCP server on stdin/stdout). + +```bash +ctxce mcp-serve \ + --workspace /path/to/your/workspace \ + --indexer-url http://localhost:8003/mcp \ + --memory-url http://localhost:8002/mcp +``` + +Flags: + +- `--workspace` / `--path` – workspace root (default: current working directory). +- `--indexer-url` – override indexer URL (default: `CTXCE_INDEXER_URL` or + `http://localhost:8003/mcp`). +- `--memory-url` – override memory URL (default: `CTXCE_MEMORY_URL` or + disabled when empty). + +## Running the MCP bridge (HTTP) + +The HTTP bridge exposes the MCP server via an HTTP endpoint (default +`http://127.0.0.1:30810/mcp`) and is what the VS Code extension uses in its +`http` transport mode. + +```bash +ctxce mcp-http-serve \ + --workspace /path/to/your/workspace \ + --indexer-url http://localhost:8003/mcp \ + --memory-url http://localhost:8002/mcp \ + --port 30810 +``` + +Flags: + +- `--workspace` / `--path` – workspace root (default: current working directory). +- `--indexer-url` – MCP indexer URL. +- `--memory-url` – MCP memory URL (or omit/empty to disable memory). +- `--port` – HTTP port for the bridge (default: `CTXCE_HTTP_PORT` + or `30810`). + +Once running, you can point an MCP client at: + +```text +http://127.0.0.1:/mcp +``` + +## Auth helper commands (`ctxce auth ...`) + +These commands are used both by the VS Code extension and standalone flows to +log in and manage auth sessions for the backend. + +### Login (token) + +```bash +ctxce auth login \ + --backend-url http://localhost:8004 \ + --token $CTXCE_AUTH_SHARED_TOKEN +``` + +This hits the backend `/auth/login` endpoint and stores a session entry in +`~/.ctxce/auth.json` under the given backend URL. + +### Login (username/password) + +```bash +ctxce auth login \ + --backend-url http://localhost:8004 \ + --username your-user \ + --password your-password +``` + +This calls `/auth/login/password` and persists the returned session the same +way as the token flow. + +### Status + +Human-readable status: + +```bash +ctxce auth status --backend-url http://localhost:8004 +``` + +Machine-readable status (used by the VS Code extension): + +```bash +ctxce auth status --backend-url http://localhost:8004 --json +``` + +The `--json` variant prints a single JSON object to stdout, for example: + +```json +{ + "backendUrl": "http://localhost:8004", + "state": "ok", // "ok" | "missing" | "expired" | "missing_backend" + "sessionId": "...", + "userId": "user-123", + "expiresAt": 0 // 0 or a Unix timestamp +} +``` + +Exit codes: + +- `0` – `state: "ok"` (valid session present). +- `1` – `state: "missing"` or `"missing_backend"`. +- `2` – `state: "expired"`. + +### Logout + +```bash +ctxce auth logout --backend-url http://localhost:8004 +``` + +Removes the stored auth entry for the given backend URL from +`~/.ctxce/auth.json`. + +## Relationship to the VS Code extension + +The VS Code **Context Engine Uploader** extension is the recommended way to use +this bridge for day-to-day development. It: + +- Launches the standalone upload client to push code into the remote stack. +- Starts/stops the MCP HTTP bridge (`ctxce mcp-http-serve`) for the active + workspace when `autoStartMcpBridge` is enabled. +- Uses `ctxce auth status --json` and `ctxce auth login` under the hood to + manage user sessions via UI prompts. + +This package README is aimed at advanced users who want to: + +- Run the MCP bridge outside of VS Code. +- Integrate the Context Engine MCP servers with other MCP-compatible clients. + +You can safely mix both approaches: the extension and the standalone bridge +share the same auth/session storage in `~/.ctxce/auth.json`. diff --git a/ctx-mcp-bridge/bin/ctxce.js b/ctx-mcp-bridge/bin/ctxce.js new file mode 100644 index 00000000..d4fe0d56 --- /dev/null +++ b/ctx-mcp-bridge/bin/ctxce.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { runCli } from "../src/cli.js"; + +runCli().catch((err) => { + console.error("[ctxce] Fatal error:", err && err.stack ? err.stack : err); + process.exit(1); +}); diff --git a/ctx-mcp-bridge/docs/debugging.md b/ctx-mcp-bridge/docs/debugging.md new file mode 100644 index 00000000..e5904743 --- /dev/null +++ b/ctx-mcp-bridge/docs/debugging.md @@ -0,0 +1,20 @@ +{ + "mcpServers": { + "context-engine": { + "command": "node", + "args": [ + "C:/Users/Admin/Documents/GitHub/Context-Engine/ctx-mcp-bridge/bin/ctxce.js", + "mcp-serve", + "--indexer-url", + "http://192.168.100.249:30806/mcp", + "--memory-url", + "http://192.168.100.249:30804/mcp", + "--workspace", + "C:/Users/Admin/Documents/GitHub/Pirate Survivors" + ], + "env": { + "CTXCE_DEBUG_LOG": "C:/Users/Admin/ctxce-mcp.log" + } + } + } +} \ No newline at end of file diff --git a/ctx-mcp-bridge/package-lock.json b/ctx-mcp-bridge/package-lock.json new file mode 100644 index 00000000..04b28ef3 --- /dev/null +++ b/ctx-mcp-bridge/package-lock.json @@ -0,0 +1,1092 @@ +{ + "name": "@context-engine-bridge/context-engine-mcp-bridge", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@context-engine-bridge/context-engine-mcp-bridge", + "version": "0.0.1", + "dependencies": { + "@modelcontextprotocol/sdk": "^1.24.3", + "zod": "^3.25.0" + }, + "bin": { + "ctxce": "bin/ctxce.js", + "ctxce-bridge": "bin/ctxce.js" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@modelcontextprotocol/sdk": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.24.3.tgz", + "integrity": "sha512-YgSHW29fuzKKAHTGe9zjNoo+yF8KaQPzDC2W9Pv41E7/57IfY+AMGJ/aDFlgTLcVVELoggKE4syABCE75u3NCw==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "content-type": "^1.0.5", + "cors": "^2.8.5", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "express": "^5.0.1", + "express-rate-limit": "^7.5.0", + "jose": "^6.1.1", + "pkce-challenge": "^5.0.0", + "raw-body": "^3.0.0", + "zod": "^3.25 || ^4.0", + "zod-to-json-schema": "^3.25.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cfworker/json-schema": "^4.1.1", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@cfworker/json-schema": { + "optional": true + }, + "zod": { + "optional": false + } + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/body-parser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz", + "integrity": "sha512-nfDwkulwiZYQIGwxdy0RUmowMhKcFVcYXUU7m4QlKYim1rUtg83xm2yjZ40QjDuc291AJjjeSc9b++AWHSgSHw==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express-rate-limit": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.1.tgz", + "integrity": "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw==", + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/express-rate-limit" + }, + "peerDependencies": { + "express": ">= 4.11" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", + "integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.0.tgz", + "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "mime-types": "^3.0.1", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } + } + } +} diff --git a/ctx-mcp-bridge/package.json b/ctx-mcp-bridge/package.json new file mode 100644 index 00000000..0c15d48e --- /dev/null +++ b/ctx-mcp-bridge/package.json @@ -0,0 +1,23 @@ +{ + "name": "@context-engine-bridge/context-engine-mcp-bridge", + "version": "0.0.8", + "description": "Context Engine MCP bridge (http/stdio proxy combining indexer + memory servers)", + "bin": { + "ctxce": "bin/ctxce.js", + "ctxce-bridge": "bin/ctxce.js" + }, + "type": "module", + "scripts": { + "start": "node bin/ctxce.js" + }, + "dependencies": { + "@modelcontextprotocol/sdk": "^1.24.3", + "zod": "^3.25.0" + }, + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/ctx-mcp-bridge/publish.sh b/ctx-mcp-bridge/publish.sh new file mode 100755 index 00000000..7d86e455 --- /dev/null +++ b/ctx-mcp-bridge/publish.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Simple helper to login (if needed) and publish the package. +# Usage: +# ./publish.sh # publishes current version +# ./publish.sh 0.0.2 # bumps version to 0.0.2 then publishes + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +PACKAGE_NAME="@context-engine-bridge/context-engine-mcp-bridge" + +echo "[publish] Verifying npm authentication..." +if ! npm whoami >/dev/null 2>&1; then + echo "[publish] Not logged in; running npm login" + npm login +else + echo "[publish] Already authenticated as $(npm whoami)" +fi + +if [[ $# -gt 0 ]]; then + VERSION="$1" + echo "[publish] Bumping version to $VERSION" + npm version "$VERSION" --no-git-tag-version +fi + +echo "[publish] Packing $PACKAGE_NAME for verification..." +npm pack >/dev/null + +echo "[publish] Publishing $PACKAGE_NAME..." +npm publish --access public + +echo "[publish] Done!" diff --git a/ctx-mcp-bridge/src/authCli.js b/ctx-mcp-bridge/src/authCli.js new file mode 100644 index 00000000..45bc86b2 --- /dev/null +++ b/ctx-mcp-bridge/src/authCli.js @@ -0,0 +1,284 @@ +import process from "node:process"; +import { loadAuthEntry, saveAuthEntry, deleteAuthEntry, loadAnyAuthEntry } from "./authConfig.js"; + +function parseAuthArgs(args) { + let backendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; + let token = process.env.CTXCE_AUTH_TOKEN || ""; + let username = process.env.CTXCE_AUTH_USERNAME || ""; + let password = process.env.CTXCE_AUTH_PASSWORD || ""; + let outputJson = false; + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if ((a === "--backend-url" || a === "--auth-url") && i + 1 < args.length) { + backendUrl = args[i + 1]; + i += 1; + continue; + } + if ((a === "--token" || a === "--api-key") && i + 1 < args.length) { + token = args[i + 1]; + i += 1; + continue; + } + if (a === "--username" || a === "--user") { + const hasNext = i + 1 < args.length; + const next = hasNext ? String(args[i + 1]) : ""; + if (hasNext && !next.startsWith("-")) { + username = args[i + 1]; + i += 1; + } else { + console.error("[ctxce] Missing value for --username/--user; expected a username."); + process.exit(1); + } + continue; + } + if ((a === "--password" || a === "--pass") && i + 1 < args.length) { + password = args[i + 1]; + i += 1; + continue; + } + if (a === "--json" || a === "-j") { + outputJson = true; + continue; + } + } + return { backendUrl, token, username, password, outputJson }; +} + +function getBackendUrl(backendUrl) { + return (backendUrl || process.env.CTXCE_AUTH_BACKEND_URL || "").trim(); +} + +function getDefaultUploadBackend() { + // Default to upload service when nothing else is configured + return (process.env.CTXCE_UPLOAD_ENDPOINT || process.env.UPLOAD_ENDPOINT || "http://localhost:8004").trim(); +} + +function requireBackendUrl(backendUrl) { + const url = getBackendUrl(backendUrl); + if (!url) { + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + return url; +} + +function outputJsonStatus(url, state, entry, rawExpires) { + const expiresAt = typeof rawExpires === "number" + ? rawExpires + : entry && typeof entry.expiresAt === "number" + ? entry.expiresAt + : null; + console.log(JSON.stringify({ + backendUrl: url, + state, + sessionId: entry && entry.sessionId ? entry.sessionId : null, + userId: entry && entry.userId ? entry.userId : null, + expiresAt, + })); +} + +async function doLogin(args) { + const { backendUrl, token, username, password } = parseAuthArgs(args); + let url = getBackendUrl(backendUrl); + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + console.error("[ctxce] Using stored backend for login:", url); + } + } + if (!url) { + // Final fallback: default upload endpoint (extension's upload endpoint or localhost:8004) + url = getDefaultUploadBackend(); + if (url) { + console.error("[ctxce] Using default upload backend for login:", url); + } + } + if (!url) { + console.error("[ctxce] Auth backend URL not configured. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + const trimmedUser = (username || "").trim(); + const usePassword = trimmedUser && (password || "").length > 0; + + let body; + let target; + if (usePassword) { + body = { + username: trimmedUser, + password, + workspace: process.cwd(), + }; + target = url.replace(/\/+$/, "") + "/auth/login/password"; + } else { + body = { + client: "ctxce", + workspace: process.cwd(), + }; + if (token) { + body.token = token; + } + target = url.replace(/\/+$/, "") + "/auth/login"; + } + let resp; + try { + resp = await fetch(target, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + } catch (err) { + console.error("[ctxce] Auth login request failed:", String(err)); + process.exit(1); + } + if (!resp || !resp.ok) { + console.error("[ctxce] Auth login failed with status", resp ? resp.status : ""); + process.exit(1); + } + let data; + try { + data = await resp.json(); + } catch (err) { + data = {}; + } + const sessionId = data.session_id || data.sessionId || null; + const userId = data.user_id || data.userId || null; + const expiresAt = data.expires_at || data.expiresAt || null; + if (!sessionId) { + console.error("[ctxce] Auth login response missing session id."); + process.exit(1); + } + saveAuthEntry(url, { sessionId, userId, expiresAt }); + console.error("[ctxce] Auth login successful for", url); +} + +async function doStatus(args) { + const { backendUrl, outputJson } = parseAuthArgs(args); + let url = getBackendUrl(backendUrl); + let usedFallback = false; + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + usedFallback = true; + } + } + if (!url) { + // Final fallback: default upload endpoint + url = getDefaultUploadBackend(); + if (url) { + usedFallback = true; + console.error("[ctxce] Using default upload backend for status:", url); + } + } + if (!url) { + if (outputJson) { + outputJsonStatus("", "missing_backend", null, null); + process.exit(1); + } + console.error("[ctxce] Auth backend URL not configured and no stored sessions found. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + let entry; + try { + entry = loadAuthEntry(url); + } catch (err) { + entry = null; + } + const nowSecs = Math.floor(Date.now() / 1000); + const rawExpires = entry && typeof entry.expiresAt === "number" ? entry.expiresAt : null; + const hasSession = !!(entry && typeof entry.sessionId === "string" && entry.sessionId); + const expired = !!(rawExpires && rawExpires > 0 && rawExpires < nowSecs); + + if (!entry || !hasSession) { + if (outputJson) { + outputJsonStatus(url, "missing", null, rawExpires); + process.exit(1); + } + if (usedFallback) { + console.error("[ctxce] Not logged in for stored backend", url); + } else { + console.error("[ctxce] Not logged in for", url); + } + process.exit(1); + } + + if (expired) { + if (outputJson) { + outputJsonStatus(url, "expired", entry, rawExpires); + process.exit(2); + } + if (usedFallback) { + console.error("[ctxce] Stored auth session appears expired for stored backend", url); + } else { + console.error("[ctxce] Stored auth session appears expired for", url); + } + if (rawExpires) { + console.error("[ctxce] Session expired at", rawExpires); + } + process.exit(2); + } + + if (outputJson) { + outputJsonStatus(url, "ok", entry, rawExpires); + return; + } + if (usedFallback) { + console.error("[ctxce] Using stored backend for status:", url); + } + console.error("[ctxce] Logged in to", url, "as", entry.userId || ""); + if (rawExpires) { + console.error("[ctxce] Session expires at", rawExpires); + } +} + +async function doLogout(args) { + const { backendUrl } = parseAuthArgs(args); + let url = getBackendUrl(backendUrl); + if (!url) { + // Fallback: use any stored auth entry when no backend is provided + const any = loadAnyAuthEntry(); + if (any && any.backendUrl) { + url = any.backendUrl; + console.error("[ctxce] Using stored backend for logout:", url); + } + } + if (!url) { + // Final fallback: default upload endpoint + url = getDefaultUploadBackend(); + if (url) { + console.error("[ctxce] Using default upload backend for logout:", url); + } + } + if (!url) { + console.error("[ctxce] Auth backend URL not configured and no stored sessions found. Set CTXCE_AUTH_BACKEND_URL or use --backend-url."); + process.exit(1); + } + const entry = loadAuthEntry(url); + if (!entry) { + console.error("[ctxce] No stored auth session for", url); + return; + } + deleteAuthEntry(url); + console.error("[ctxce] Logged out from", url); +} + +export async function runAuthCommand(subcommand, args) { + const sub = (subcommand || "").toLowerCase(); + if (sub === "login") { + await doLogin(args || []); + return; + } + if (sub === "status") { + await doStatus(args || []); + return; + } + if (sub === "logout") { + await doLogout(args || []); + return; + } + console.error("Usage: ctxce auth [--backend-url ] [--token ]"); + process.exit(1); +} diff --git a/ctx-mcp-bridge/src/authConfig.js b/ctx-mcp-bridge/src/authConfig.js new file mode 100644 index 00000000..7cd63bb1 --- /dev/null +++ b/ctx-mcp-bridge/src/authConfig.js @@ -0,0 +1,84 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const CONFIG_DIR_NAME = ".ctxce"; +const CONFIG_BASENAME = "auth.json"; + +function getConfigPath() { + const home = os.homedir() || process.cwd(); + const dir = path.join(home, CONFIG_DIR_NAME); + return path.join(dir, CONFIG_BASENAME); +} + +function readConfig() { + try { + const cfgPath = getConfigPath(); + const raw = fs.readFileSync(cfgPath, "utf8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + } + return {}; +} + +function writeConfig(data) { + try { + const cfgPath = getConfigPath(); + const dir = path.dirname(cfgPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + fs.writeFileSync(cfgPath, JSON.stringify(data, null, 2), "utf8"); + } catch (err) { + } +} + +export function loadAuthEntry(backendUrl) { + if (!backendUrl) { + return null; + } + const all = readConfig(); + const key = String(backendUrl); + const entry = all[key]; + if (!entry || typeof entry !== "object") { + return null; + } + return entry; +} + +export function saveAuthEntry(backendUrl, entry) { + if (!backendUrl || !entry || typeof entry !== "object") { + return; + } + const all = readConfig(); + const key = String(backendUrl); + all[key] = entry; + writeConfig(all); +} + +export function deleteAuthEntry(backendUrl) { + if (!backendUrl) { + return; + } + const all = readConfig(); + const key = String(backendUrl); + if (Object.prototype.hasOwnProperty.call(all, key)) { + delete all[key]; + writeConfig(all); + } +} + +export function loadAnyAuthEntry() { + const all = readConfig(); + const keys = Object.keys(all); + for (const key of keys) { + const entry = all[key]; + if (entry && typeof entry === "object") { + return { backendUrl: key, entry }; + } + } + return null; +} diff --git a/ctx-mcp-bridge/src/cli.js b/ctx-mcp-bridge/src/cli.js new file mode 100644 index 00000000..4d995645 --- /dev/null +++ b/ctx-mcp-bridge/src/cli.js @@ -0,0 +1,123 @@ +// CLI entrypoint for ctxce + +import process from "node:process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runMcpServer, runHttpMcpServer } from "./mcpServer.js"; +import { runAuthCommand } from "./authCli.js"; + +export async function runCli() { + const argv = process.argv.slice(2); + const cmd = argv[0]; + + if (cmd === "auth") { + const sub = argv[1] || ""; + const args = argv.slice(2); + await runAuthCommand(sub, args); + return; + } + + if (cmd === "mcp-http-serve") { + const args = argv.slice(1); + let workspace = process.cwd(); + let indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + let memoryUrl = process.env.CTXCE_MEMORY_URL || null; + let port = Number.parseInt(process.env.CTXCE_HTTP_PORT || "30810", 10) || 30810; + + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if (a === "--workspace" || a === "--path") { + if (i + 1 < args.length) { + workspace = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--indexer-url") { + if (i + 1 < args.length) { + indexerUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--memory-url") { + if (i + 1 < args.length) { + memoryUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--port") { + if (i + 1 < args.length) { + const parsed = Number.parseInt(args[i + 1], 10); + if (!Number.isNaN(parsed) && parsed > 0) { + port = parsed; + } + i += 1; + continue; + } + } + } + + // eslint-disable-next-line no-console + console.error( + `[ctxce] Starting HTTP MCP bridge: workspace=${workspace}, port=${port}, indexerUrl=${indexerUrl}, memoryUrl=${memoryUrl || "disabled"}`, + ); + await runHttpMcpServer({ workspace, indexerUrl, memoryUrl, port }); + return; + } + + if (cmd === "mcp-serve") { + // Minimal flag parsing for PoC: allow passing workspace/root and indexer URL. + // Supported flags: + // --workspace / --path : workspace root (default: cwd) + // --indexer-url : override MCP indexer URL (default env CTXCE_INDEXER_URL or http://localhost:8003/mcp) + const args = argv.slice(1); + let workspace = process.cwd(); + let indexerUrl = process.env.CTXCE_INDEXER_URL || "http://localhost:8003/mcp"; + let memoryUrl = process.env.CTXCE_MEMORY_URL || null; + + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + if (a === "--workspace" || a === "--path") { + if (i + 1 < args.length) { + workspace = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--indexer-url") { + if (i + 1 < args.length) { + indexerUrl = args[i + 1]; + i += 1; + continue; + } + } + if (a === "--memory-url") { + if (i + 1 < args.length) { + memoryUrl = args[i + 1]; + i += 1; + continue; + } + } + } + + // eslint-disable-next-line no-console + console.error( + `[ctxce] Starting MCP bridge: workspace=${workspace}, indexerUrl=${indexerUrl}, memoryUrl=${memoryUrl || "disabled"}`, + ); + await runMcpServer({ workspace, indexerUrl, memoryUrl }); + return; + } + + // Default help + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const binName = "ctxce"; + + // eslint-disable-next-line no-console + console.error( + `Usage: ${binName} mcp-serve [--workspace ] [--indexer-url ] [--memory-url ] | ${binName} mcp-http-serve [--workspace ] [--indexer-url ] [--memory-url ] [--port ] | ${binName} auth [--backend-url ] [--token ] [--username --password ]`, + ); + process.exit(1); +} diff --git a/ctx-mcp-bridge/src/mcpServer.js b/ctx-mcp-bridge/src/mcpServer.js new file mode 100644 index 00000000..0cc12a53 --- /dev/null +++ b/ctx-mcp-bridge/src/mcpServer.js @@ -0,0 +1,748 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; +import { execSync } from "node:child_process"; +import { createServer } from "node:http"; +import { Server } from "@modelcontextprotocol/sdk/server/index.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; +import { loadAnyAuthEntry, loadAuthEntry } from "./authConfig.js"; +import { maybeRemapToolResult } from "./resultPathMapping.js"; + +function debugLog(message) { + try { + const text = typeof message === "string" ? message : String(message); + console.error(text); + const dest = process.env.CTXCE_DEBUG_LOG; + if (dest) { + fs.appendFileSync(dest, `${new Date().toISOString()} ${text}\n`, "utf8"); + } + } catch { + } +} + +async function sendSessionDefaults(client, payload, label) { + if (!client) { + return; + } + try { + await client.callTool({ + name: "set_session_defaults", + arguments: payload, + }); + } catch (err) { + // eslint-disable-next-line no-console + console.error(`[ctxce] Failed to call set_session_defaults on ${label}:`, err); + } +} +function dedupeTools(tools) { + const seen = new Set(); + const out = []; + for (const tool of tools) { + const key = (tool && typeof tool.name === "string" && tool.name) || ""; + if (!key || seen.has(key)) { + if (key === "" || key !== "set_session_defaults") { + continue; + } + if (seen.has(key)) { + continue; + } + } + seen.add(key); + out.push(tool); + } + return out; +} + +async function listMemoryTools(client) { + if (!client) { + return []; + } + try { + const remote = await withTimeout( + client.listTools(), + 5000, + "memory tools/list", + ); + return Array.isArray(remote?.tools) ? remote.tools.slice() : []; + } catch (err) { + debugLog("[ctxce] Error calling memory tools/list: " + String(err)); + return []; + } +} + +function withTimeout(promise, ms, label) { + return new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + const errorMessage = + label != null + ? `[ctxce] Timeout after ${ms}ms in ${label}` + : `[ctxce] Timeout after ${ms}ms`; + reject(new Error(errorMessage)); + }, ms); + promise + .then((value) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resolve(value); + }) + .catch((err) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + reject(err); + }); + }); +} + +function getBridgeToolTimeoutMs() { + try { + const raw = process.env.CTXCE_TOOL_TIMEOUT_MSEC; + if (!raw) { + return 300000; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return 300000; + } + return parsed; + } catch { + return 300000; + } +} + +function selectClientForTool(name, indexerClient, memoryClient) { + if (!name) { + return indexerClient; + } + const lowered = name.toLowerCase(); + if (memoryClient && (lowered.startsWith("memory.") || lowered.startsWith("mcp_memory_") || lowered.includes("memory"))) { + return memoryClient; + } + return indexerClient; +} + +function isSessionError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + return ( + msg.includes("No valid session ID") || + msg.includes("Mcp-Session-Id header is required") || + msg.includes("Server not initialized") || + msg.includes("Session not found") + ); + } catch { + return false; + } +} + +function getBridgeRetryAttempts() { + try { + const raw = process.env.CTXCE_TOOL_RETRY_ATTEMPTS; + if (!raw) { + return 2; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + return 1; + } + return parsed; + } catch { + return 2; + } +} + +function getBridgeRetryDelayMs() { + try { + const raw = process.env.CTXCE_TOOL_RETRY_DELAY_MSEC; + if (!raw) { + return 200; + } + const parsed = Number.parseInt(String(raw), 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return 0; + } + return parsed; + } catch { + return 200; + } +} + +function isTransientToolError(error) { + try { + const msg = + (error && typeof error.message === "string" && error.message) || + (typeof error === "string" ? error : String(error || "")); + if (!msg) { + return false; + } + const lower = msg.toLowerCase(); + + if ( + lower.includes("timed out") || + lower.includes("timeout") || + lower.includes("time-out") + ) { + return true; + } + + if ( + lower.includes("econnreset") || + lower.includes("econnrefused") || + lower.includes("etimedout") || + lower.includes("enotfound") || + lower.includes("ehostunreach") || + lower.includes("enetunreach") + ) { + return true; + } + + if ( + lower.includes("bad gateway") || + lower.includes("gateway timeout") || + lower.includes("service unavailable") || + lower.includes(" 502 ") || + lower.includes(" 503 ") || + lower.includes(" 504 ") + ) { + return true; + } + + if (lower.includes("network error")) { + return true; + } + + if (typeof error.code === "number" && error.code === -32001 && !isSessionError(error)) { + return true; + } + if ( + typeof error.code === "string" && + error.code.toLowerCase && + error.code.toLowerCase().includes("timeout") + ) { + return true; + } + + return false; + } catch { + return false; + } +} +// MCP stdio server implemented using the official MCP TypeScript SDK. +// Acts as a low-level proxy for tools, forwarding tools/list and tools/call +// to the remote qdrant-indexer MCP server while adding a local `ping` tool. + +async function createBridgeServer(options) { + const workspace = options.workspace || process.cwd(); + const indexerUrl = options.indexerUrl; + const memoryUrl = options.memoryUrl; + + const config = loadConfig(workspace); + const defaultCollection = + config && typeof config.default_collection === "string" + ? config.default_collection + : null; + const defaultMode = + config && typeof config.default_mode === "string" ? config.default_mode : null; + const defaultUnder = + config && typeof config.default_under === "string" ? config.default_under : null; + + debugLog( + `[ctxce] MCP low-level stdio bridge starting: workspace=${workspace}, indexerUrl=${indexerUrl}`, + ); + + if (defaultCollection) { + // eslint-disable-next-line no-console + console.error( + `[ctxce] Using default collection from ctx_config.json: ${defaultCollection}`, + ); + } + + let indexerClient = null; + let memoryClient = null; + + // Derive a simple session identifier for this bridge process. In the + // future this can be made user-aware (e.g. from auth), but for now we + // keep it deterministic per workspace to help the indexer reuse + // session-scoped defaults. + const explicitSession = process.env.CTXCE_SESSION_ID || ""; + const authBackendUrl = process.env.CTXCE_AUTH_BACKEND_URL || ""; + let sessionId = explicitSession; + + function resolveSessionId() { + const explicit = process.env.CTXCE_SESSION_ID || ""; + if (explicit) { + return explicit; + } + let backendToUse = authBackendUrl; + let entry = null; + if (backendToUse) { + try { + entry = loadAuthEntry(backendToUse); + } catch { + entry = null; + } + } + if (!entry) { + try { + const any = loadAnyAuthEntry(); + if (any && any.entry) { + backendToUse = any.backendUrl; + entry = any.entry; + } + } catch { + entry = null; + } + } + if (entry) { + let expired = false; + const rawExpires = entry.expiresAt; + if (typeof rawExpires === "number" && Number.isFinite(rawExpires) && rawExpires > 0) { + const nowSecs = Math.floor(Date.now() / 1000); + if (rawExpires < nowSecs) { + expired = true; + } + } + if (!expired && typeof entry.sessionId === "string" && entry.sessionId) { + return entry.sessionId; + } + if (expired) { + debugLog("[ctxce] Stored auth session appears expired; please run `ctxce auth login` again."); + } + } + return ""; + } + + if (!sessionId) { + sessionId = resolveSessionId(); + } + + if (!sessionId) { + sessionId = `ctxce-${Buffer.from(workspace).toString("hex").slice(0, 24)}`; + } + + // Best-effort: inform the indexer of default collection and session. + // If this fails we still proceed, falling back to per-call injection. + const defaultsPayload = { session: sessionId }; + if (defaultCollection) { + defaultsPayload.collection = defaultCollection; + } + if (defaultMode) { + defaultsPayload.mode = defaultMode; + } + if (defaultUnder) { + defaultsPayload.under = defaultUnder; + } + + async function initializeRemoteClients(forceRecreate = false) { + if (!forceRecreate && indexerClient) { + return; + } + + if (forceRecreate) { + try { + debugLog("[ctxce] Reinitializing remote MCP clients after session error."); + } catch { + // ignore logging failures + } + } + + let nextIndexerClient = null; + try { + const indexerTransport = new StreamableHTTPClientTransport(indexerUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-http-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await client.connect(indexerTransport); + nextIndexerClient = client; + } catch (err) { + debugLog("[ctxce] Failed to connect MCP HTTP client to indexer: " + String(err)); + nextIndexerClient = null; + } + + let nextMemoryClient = null; + if (memoryUrl) { + try { + const memoryTransport = new StreamableHTTPClientTransport(memoryUrl); + const client = new Client( + { + name: "ctx-context-engine-bridge-memory-client", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + resources: {}, + prompts: {}, + }, + }, + ); + await client.connect(memoryTransport); + debugLog(`[ctxce] Connected memory MCP client: ${memoryUrl}`); + nextMemoryClient = client; + } catch (err) { + debugLog("[ctxce] Failed to connect memory MCP client: " + String(err)); + nextMemoryClient = null; + } + } + + indexerClient = nextIndexerClient; + memoryClient = nextMemoryClient; + + if (Object.keys(defaultsPayload).length > 1 && indexerClient) { + await sendSessionDefaults(indexerClient, defaultsPayload, "indexer"); + if (memoryClient) { + await sendSessionDefaults(memoryClient, defaultsPayload, "memory"); + } + } + } + + await initializeRemoteClients(false); + + const server = new Server( // TODO: marked as depreciated + { + name: "ctx-context-engine-bridge", + version: "0.0.1", + }, + { + capabilities: { + tools: {}, + }, + }, + ); + + // tools/list → fetch tools from remote indexer + server.setRequestHandler(ListToolsRequestSchema, async () => { + let remote; + try { + debugLog("[ctxce] tools/list: fetching tools from indexer"); + await initializeRemoteClients(false); + if (!indexerClient) { + throw new Error("Indexer MCP client not initialized"); + } + remote = await withTimeout( + indexerClient.listTools(), + 10000, + "indexer tools/list", + ); + } catch (err) { + debugLog("[ctxce] Error calling remote tools/list: " + String(err)); + const memoryToolsFallback = await listMemoryTools(memoryClient); + const toolsFallback = dedupeTools([...memoryToolsFallback]); + return { tools: toolsFallback }; + } + + try { + const toolNames = + remote && Array.isArray(remote.tools) + ? remote.tools.map((t) => (t && typeof t.name === "string" ? t.name : "")) + : []; + debugLog("[ctxce] tools/list remote result tools: " + JSON.stringify(toolNames)); + } catch (err) { + debugLog("[ctxce] tools/list remote result: " + String(err)); + } + + const indexerTools = Array.isArray(remote?.tools) ? remote.tools.slice() : []; + const memoryTools = await listMemoryTools(memoryClient); + const tools = dedupeTools([...indexerTools, ...memoryTools]); + debugLog(`[ctxce] tools/list: returning ${tools.length} tools`); + return { tools }; + }); + + // tools/call → proxied to indexer or memory server + server.setRequestHandler(CallToolRequestSchema, async (request) => { + const params = request.params || {}; + const name = params.name; + let args = params.arguments; + + debugLog(`[ctxce] tools/call: ${name || ""}`); + + // Refresh session before each call; re-init clients if session changes. + const freshSession = resolveSessionId() || sessionId; + if (freshSession && freshSession !== sessionId) { + sessionId = freshSession; + try { + await initializeRemoteClients(true); + } catch (err) { + debugLog("[ctxce] Failed to reinitialize clients after session refresh: " + String(err)); + } + } + if (sessionId && (args === undefined || args === null || typeof args === "object")) { + const obj = args && typeof args === "object" ? { ...args } : {}; + if (!Object.prototype.hasOwnProperty.call(obj, "session")) { + obj.session = sessionId; + } + args = obj; + } + + if (name === "set_session_defaults") { + const indexerResult = await indexerClient.callTool({ name, arguments: args }); + if (memoryClient) { + try { + await memoryClient.callTool({ name, arguments: args }); + } catch (err) { + debugLog("[ctxce] Memory set_session_defaults failed: " + String(err)); + } + } + return indexerResult; + } + + await initializeRemoteClients(false); + + const timeoutMs = getBridgeToolTimeoutMs(); + const maxAttempts = getBridgeRetryAttempts(); + const retryDelayMs = getBridgeRetryDelayMs(); + let sessionRetried = false; + let lastError; + + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + if (attempt > 0 && retryDelayMs > 0) { + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + } + + const targetClient = selectClientForTool(name, indexerClient, memoryClient); + if (!targetClient) { + throw new Error(`Tool ${name} not available on any configured MCP server`); + } + + try { + const result = await targetClient.callTool( + { + name, + arguments: args, + }, + undefined, + { timeout: timeoutMs }, + ); + return maybeRemapToolResult(name, result, workspace); + } catch (err) { + lastError = err; + + if (isSessionError(err) && !sessionRetried) { + debugLog( + "[ctxce] tools/call: detected remote MCP session error; reinitializing clients and retrying once: " + + String(err), + ); + await initializeRemoteClients(true); + sessionRetried = true; + continue; + } + + if (!isTransientToolError(err) || attempt === maxAttempts - 1) { + throw err; + } + + debugLog( + `[ctxce] tools/call: transient error (attempt ${attempt + 1}/${maxAttempts}), retrying: ` + + String(err), + ); + // Loop will retry + } + } + + throw lastError || new Error("Unknown MCP tools/call error"); + }); + + return server; +} + +export async function runMcpServer(options) { + const server = await createBridgeServer(options); + const transport = new StdioServerTransport(); + await server.connect(transport); + + const exitOnStdinClose = process.env.CTXCE_EXIT_ON_STDIN_CLOSE !== "0"; + if (exitOnStdinClose) { + const handleStdioClosed = () => { + try { + debugLog("[ctxce] Stdio transport closed; exiting MCP bridge process."); + } catch { + // ignore + } + // Allow any in-flight logs to flush, then exit. + setTimeout(() => { + process.exit(0); + }, 10).unref(); + }; + + if (process.stdin && typeof process.stdin.on === "function") { + process.stdin.on("end", handleStdioClosed); + process.stdin.on("close", handleStdioClosed); + process.stdin.on("error", handleStdioClosed); + } + } +} + +export async function runHttpMcpServer(options) { + const server = await createBridgeServer(options); + const port = + typeof options.port === "number" + ? options.port + : Number.parseInt(process.env.CTXCE_HTTP_PORT || "30810", 10) || 30810; + + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + }); + + await server.connect(transport); + + const httpServer = createServer((req, res) => { + try { + if (!req.url || !req.url.startsWith("/mcp")) { + res.statusCode = 404; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32000, message: "Not found" }, + id: null, + }), + ); + return; + } + + if (req.method !== "POST") { + res.statusCode = 405; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32000, message: "Method not allowed" }, + id: null, + }), + ); + return; + } + + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", async () => { + let parsed; + try { + parsed = body ? JSON.parse(body) : {}; + } catch (err) { + debugLog("[ctxce] Failed to parse HTTP MCP request body: " + String(err)); + res.statusCode = 400; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32700, message: "Invalid JSON" }, + id: null, + }), + ); + return; + } + + try { + await transport.handleRequest(req, res, parsed); + } catch (err) { + debugLog("[ctxce] Error handling HTTP MCP request: " + String(err)); + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32603, message: "Internal server error" }, + id: null, + }), + ); + } + } + }); + } catch (err) { + debugLog("[ctxce] Unexpected error in HTTP MCP server: " + String(err)); + if (!res.headersSent) { + res.statusCode = 500; + res.setHeader("Content-Type", "application/json"); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + error: { code: -32603, message: "Internal server error" }, + id: null, + }), + ); + } + } + }); + + httpServer.listen(port, () => { + debugLog(`[ctxce] HTTP MCP bridge listening on port ${port}`); + }); +} + +function loadConfig(startDir) { + try { + let dir = startDir; + for (let i = 0; i < 5; i += 1) { + const cfgPath = path.join(dir, "ctx_config.json"); + if (fs.existsSync(cfgPath)) { + try { + const raw = fs.readFileSync(cfgPath, "utf8"); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object") { + return parsed; + } + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Failed to parse ctx_config.json:", err); + return null; + } + } + const parent = path.dirname(dir); + if (!parent || parent === dir) { + break; + } + dir = parent; + } + } catch (err) { + // eslint-disable-next-line no-console + console.error("[ctxce] Error while loading ctx_config.json:", err); + } + return null; +} + +function detectGitBranch(workspace) { + try { + const out = execSync("git rev-parse --abbrev-ref HEAD", { + cwd: workspace, + stdio: ["ignore", "pipe", "ignore"], + }); + const name = out.toString("utf8").trim(); + return name || null; + } catch { + return null; + } +} + diff --git a/ctx-mcp-bridge/src/resultPathMapping.js b/ctx-mcp-bridge/src/resultPathMapping.js new file mode 100644 index 00000000..97f30ce3 --- /dev/null +++ b/ctx-mcp-bridge/src/resultPathMapping.js @@ -0,0 +1,326 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; + +function envTruthy(value, defaultVal = false) { + try { + if (value === undefined || value === null) { + return defaultVal; + } + const s = String(value).trim().toLowerCase(); + if (!s) { + return defaultVal; + } + return s === "1" || s === "true" || s === "yes" || s === "on"; + } catch { + return defaultVal; + } +} + +function _posixToNative(rel) { + try { + if (!rel) { + return ""; + } + return String(rel).split("/").join(path.sep); + } catch { + return rel; + } +} + +function computeWorkspaceRelativePath(containerPath, hostPath) { + try { + const cont = typeof containerPath === "string" ? containerPath.trim() : ""; + if (cont.startsWith("/work/")) { + const rest = cont.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + return parts.slice(1).join("/"); + } + if (parts.length === 1) { + return parts[0]; + } + } + } catch { + } + try { + const hp = typeof hostPath === "string" ? hostPath.trim() : ""; + if (!hp) { + return ""; + } + // If we don't have a container path, at least try to return a basename. + return path.posix.basename(hp.replace(/\\/g, "/")); + } catch { + return ""; + } +} + +function remapRelatedPathToClient(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p : ""; + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + if (!s || !root) { + return p; + } + + const sNorm = s.replace(/\\/g, path.sep); + if (sNorm.startsWith(root + path.sep) || sNorm === root) { + return sNorm; + } + + if (s.startsWith("/work/")) { + const rest = s.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + const rel = parts.slice(1).join("/"); + const relNative = _posixToNative(rel); + return path.join(root, relNative); + } + return p; + } + + // If it's already a relative path, join it to the workspace root. + if (!s.startsWith("/") && !s.includes(":") && !s.includes("\\")) { + const relPosix = s.trim(); + if (relPosix && relPosix !== "." && !relPosix.startsWith("../") && relPosix !== "..") { + const relNative = _posixToNative(relPosix); + const joined = path.join(root, relNative); + const relCheck = path.relative(root, joined); + if (relCheck && !relCheck.startsWith(`..${path.sep}`) && relCheck !== "..") { + return joined; + } + } + } + + return p; + } catch { + return p; + } +} + +function remapHitPaths(hit, workspaceRoot) { + if (!hit || typeof hit !== "object") { + return hit; + } + const hostPath = typeof hit.host_path === "string" ? hit.host_path : ""; + const containerPath = typeof hit.container_path === "string" ? hit.container_path : ""; + const relPath = computeWorkspaceRelativePath(containerPath, hostPath); + const out = { ...hit }; + if (relPath) { + out.rel_path = relPath; + } + // Remap related_paths nested under each hit (repo_search/hybrid_search emit this per result). + try { + if (Array.isArray(out.related_paths)) { + out.related_paths = out.related_paths.map((p) => remapRelatedPathToClient(p, workspaceRoot)); + } + } catch { + // ignore + } + if (workspaceRoot && relPath) { + try { + const relNative = _posixToNative(relPath); + const candidate = path.join(workspaceRoot, relNative); + const diagnostics = envTruthy(process.env.CTXCE_BRIDGE_PATH_DIAGNOSTICS, false); + const strictClientPath = envTruthy(process.env.CTXCE_BRIDGE_CLIENT_PATH_STRICT, false); + if (strictClientPath) { + out.client_path = candidate; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "workspace_join"; + } + } else { + // Prefer a host_path that is within the current bridge workspace. + // This keeps provenance (host_path) intact while providing a user-local + // absolute path even when the bridge workspace is a parent directory. + const hp = typeof hostPath === "string" ? hostPath : ""; + const hpNorm = hp ? hp.replace(/\\/g, path.sep) : ""; + if ( + hpNorm && + hpNorm.startsWith(workspaceRoot) && + (!fs.existsSync(candidate) || fs.existsSync(hpNorm)) + ) { + out.client_path = hpNorm; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "host_path"; + } + } else { + out.client_path = candidate; + if (diagnostics) { + out.client_path_joined = candidate; + out.client_path_source = "workspace_join"; + } + } + } + } catch { + // ignore + } + } + const overridePath = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (overridePath && relPath) { + out.path = relPath; + } + return out; +} + +function remapStringPath(p, workspaceRoot) { + try { + const s = typeof p === "string" ? p : ""; + if (!s) { + return p; + } + // If this is already a path within the current client workspace, rewrite to a + // workspace-relative string when override is enabled. + try { + const root = typeof workspaceRoot === "string" ? workspaceRoot : ""; + if (root) { + const sNorm = s.replace(/\\/g, path.sep); + if (sNorm.startsWith(root + path.sep) || sNorm === root) { + const relNative = path.relative(root, sNorm); + const relPosix = String(relNative).split(path.sep).join("/"); + if (relPosix && !relPosix.startsWith("../") && relPosix !== "..") { + const override = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (override) { + return relPosix; + } + } + } + } + } catch { + // ignore + } + if (s.startsWith("/work/")) { + const rest = s.slice("/work/".length); + const parts = rest.split("/").filter(Boolean); + if (parts.length >= 2) { + const rel = parts.slice(1).join("/"); + const override = envTruthy(process.env.CTXCE_BRIDGE_OVERRIDE_PATH, true); + if (override) { + return rel; + } + return p; + } + } + return p; + } catch { + return p; + } +} + +function maybeParseToolJson(result) { + try { + if ( + result && + typeof result === "object" && + result.structuredContent && + typeof result.structuredContent === "object" + ) { + return { mode: "structured", value: result.structuredContent }; + } + } catch { + } + try { + const content = result && result.content; + if (!Array.isArray(content)) { + return null; + } + const first = content.find( + (c) => c && c.type === "text" && typeof c.text === "string", + ); + if (!first) { + return null; + } + const txt = String(first.text || "").trim(); + if (!txt || !(txt.startsWith("{") || txt.startsWith("["))) { + return null; + } + return { mode: "text", value: JSON.parse(txt) }; + } catch { + return null; + } +} + +function applyPathMappingToPayload(payload, workspaceRoot) { + if (!payload || typeof payload !== "object") { + return payload; + } + const out = Array.isArray(payload) ? payload.slice() : { ...payload }; + + const mapHitsArray = (arr) => { + if (!Array.isArray(arr)) { + return arr; + } + return arr.map((h) => remapHitPaths(h, workspaceRoot)); + }; + + // Common result shapes across tools + if (Array.isArray(out.results)) { + out.results = mapHitsArray(out.results); + } + if (Array.isArray(out.citations)) { + out.citations = mapHitsArray(out.citations); + } + if (Array.isArray(out.related_paths)) { + out.related_paths = out.related_paths.map((p) => remapRelatedPathToClient(p, workspaceRoot)); + } + + // Some tools nest under {result:{...}} + if (out.result && typeof out.result === "object") { + out.result = applyPathMappingToPayload(out.result, workspaceRoot); + } + + return out; +} + +export function maybeRemapToolResult(name, result, workspaceRoot) { + try { + if (!name || !result || !workspaceRoot) { + return result; + } + const enabled = envTruthy(process.env.CTXCE_BRIDGE_MAP_PATHS, true); + if (!enabled) { + return result; + } + const lower = String(name).toLowerCase(); + const shouldMap = ( + lower === "repo_search" || + lower === "context_search" || + lower === "context_answer" || + lower.endsWith("search_tests_for") || + lower.endsWith("search_config_for") || + lower.endsWith("search_callers_for") || + lower.endsWith("search_importers_for") + ); + if (!shouldMap) { + return result; + } + + const parsed = maybeParseToolJson(result); + if (!parsed) { + return result; + } + + const mapped = applyPathMappingToPayload(parsed.value, workspaceRoot); + if (parsed.mode === "structured") { + return { ...result, structuredContent: mapped }; + } + + // Replace text payload for clients that only read `content[].text` + try { + const content = Array.isArray(result.content) ? result.content.slice() : []; + const idx = content.findIndex( + (c) => c && c.type === "text" && typeof c.text === "string", + ); + if (idx >= 0) { + content[idx] = { ...content[idx], text: JSON.stringify(mapped) }; + return { ...result, content }; + } + } catch { + // ignore + } + return result; + } catch { + return result; + } +} diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml index 34fe12c5..ec5bbc67 100644 --- a/deploy/kubernetes/configmap.yaml +++ b/deploy/kubernetes/configmap.yaml @@ -8,8 +8,12 @@ metadata: component: configuration data: COLLECTION_NAME: codebase + COMMIT_VECTOR_SEARCH: '0' + CTXCE_AUTH_ADMIN_TOKEN: change-me-admin-token + CTXCE_AUTH_ENABLED: '0' CTX_SNIPPET_CHARS: '400' CTX_SUMMARY_CHARS: '0' + CURRENT_REPO: '' DECODER_MAX_TOKENS: '4000' EMBEDDING_MODEL: BAAI/bge-base-en-v1.5 EMBEDDING_PROVIDER: fastembed @@ -40,6 +44,11 @@ data: INDEX_UPSERT_BACKOFF: '0.5' INDEX_UPSERT_BATCH: '128' INDEX_UPSERT_RETRIES: '5' + INDEX_USE_ENHANCED_AST: '1' + INFO_REQUEST_CONTEXT_LINES: '5' + INFO_REQUEST_EXPLAIN_DEFAULT: '0' + INFO_REQUEST_LIMIT: '10' + INFO_REQUEST_RELATIONSHIPS: '0' LLAMACPP_EXTRA_ARGS: '' LLAMACPP_GPU_LAYERS: '32' LLAMACPP_GPU_SPLIT: '' @@ -70,10 +79,22 @@ data: MINI_VEC_SEED: '1337' MULTI_REPO_MODE: '1' OLLAMA_HOST: http://host.docker.internal:11434 + POST_RERANK_SYMBOL_BOOST: '1.0' PRF_ENABLED: '1' QDRANT_API_KEY: '' + QDRANT_EF_SEARCH: '128' QDRANT_TIMEOUT: '20' QDRANT_URL: http://qdrant:6333 + QUERY_OPTIMIZER_ADAPTIVE: '1' + QUERY_OPTIMIZER_COLLECTION_SIZE: '10000' + QUERY_OPTIMIZER_COMPLEX_FACTOR: '2.0' + QUERY_OPTIMIZER_COMPLEX_THRESHOLD: '0.7' + QUERY_OPTIMIZER_DENSE_THRESHOLD: '0.2' + QUERY_OPTIMIZER_MAX_EF: '512' + QUERY_OPTIMIZER_MIN_EF: '64' + QUERY_OPTIMIZER_SEMANTIC_FACTOR: '1.0' + QUERY_OPTIMIZER_SIMPLE_FACTOR: '0.5' + QUERY_OPTIMIZER_SIMPLE_THRESHOLD: '0.3' REFRAG_CANDIDATES: '200' REFRAG_COMMIT_DESCRIBE: '1' REFRAG_DECODER: '1' @@ -87,12 +108,14 @@ data: REFRAG_SENSE: heuristic REFRAG_SOFT_SCALE: '1.0' REMOTE_UPLOAD_GIT_MAX_COMMITS: '500' + REPO_AUTO_FILTER: '1' RERANKER_ENABLED: '1' - RERANKER_ONNX_PATH: /work/models/model_qint8_avx512_vnni.onnx + RERANKER_ONNX_PATH: /app/models/reranker.onnx RERANKER_RETURN_M: '20' RERANKER_TIMEOUT_MS: '3000' - RERANKER_TOKENIZER_PATH: /work/models/tokenizer.json + RERANKER_TOKENIZER_PATH: /app/models/tokenizer.json RERANKER_TOPN: '100' + RERANK_BLEND_WEIGHT: '0.6' RERANK_EXPAND: '1' RERANK_IN_PROCESS: '1' RERANK_TIMEOUT_FLOOR_MS: '1000' diff --git a/docker-compose.dev-remote.yml b/docker-compose.dev-remote.yml index 437ec4a0..7443c5b1 100644 --- a/docker-compose.dev-remote.yml +++ b/docker-compose.dev-remote.yml @@ -33,6 +33,15 @@ services: - FASTMCP_HOST=${FASTMCP_HOST} - FASTMCP_PORT=${FASTMCP_PORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - COLLECTION_NAME=${COLLECTION_NAME} - PATH_EMIT_MODE=auto - HF_HOME=/work/.cache/huggingface @@ -75,6 +84,15 @@ services: - FASTMCP_HOST=${FASTMCP_HOST} - FASTMCP_INDEXER_PORT=${FASTMCP_INDEXER_PORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - REFRAG_DECODER=${REFRAG_DECODER:-1} - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} - GLM_API_KEY=${GLM_API_KEY} @@ -120,6 +138,15 @@ services: - FASTMCP_PORT=8000 - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - COLLECTION_NAME=${COLLECTION_NAME} - PATH_EMIT_MODE=auto - HF_HOME=/work/.cache/huggingface @@ -162,6 +189,15 @@ services: - FASTMCP_INDEXER_PORT=8001 - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT} - QDRANT_URL=${QDRANT_URL} + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - REFRAG_DECODER=${REFRAG_DECODER:-1} - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp} - GLM_API_KEY=${GLM_API_KEY} @@ -288,7 +324,7 @@ services: context: . dockerfile: Dockerfile.indexer container_name: init-payload-dev-remote - user: "1000:1000" + user: "0:0" depends_on: - qdrant env_file: @@ -296,6 +332,13 @@ services: environment: - QDRANT_URL=${QDRANT_URL} - COLLECTION_NAME=${COLLECTION_NAME} + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} - HF_HOME=/work/.cache/huggingface - TRANSFORMERS_CACHE=/work/.cache/huggingface - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface @@ -309,7 +352,7 @@ services: command: [ "sh", "-c", - "mkdir -p /tmp/logs && echo 'Starting initialization sequence...' && /app/scripts/wait-for-qdrant.sh && PYTHONPATH=/app python /app/scripts/create_indexes.py && echo 'Collections and metadata created' && python /app/scripts/warm_all_collections.py && echo 'Search caches warmed for all collections' && python /app/scripts/health_check.py && echo 'Initialization completed successfully!'" + "mkdir -p /tmp/logs /work/.codebase && (chgrp -R 1000 /work/.codebase 2>/dev/null || true) && (chmod -R g+rwX /work/.codebase 2>/dev/null || true) && (find /work/.codebase -type d -exec chmod g+s {} + 2>/dev/null || true) && echo 'Starting initialization sequence...' && /app/scripts/wait-for-qdrant.sh && PYTHONPATH=/app python /app/scripts/create_indexes.py && echo 'Collections and metadata created' && python /app/scripts/warm_all_collections.py && echo 'Search caches warmed for all collections' && python /app/scripts/health_check.py && echo 'Initialization completed successfully!'" ] restart: "no" # Run once on startup networks: @@ -334,6 +377,15 @@ services: - WORKDIR=/work - MAX_BUNDLE_SIZE_MB=100 - UPLOAD_TIMEOUT_SECS=300 + # Optional auth configuration (fully opt-in via .env) + - CTXCE_AUTH_ENABLED=${CTXCE_AUTH_ENABLED:-0} + - CTXCE_MCP_ACL_ENFORCE=${CTXCE_MCP_ACL_ENFORCE:-0} + - CTXCE_ACL_ALLOW_ALL=${CTXCE_ACL_ALLOW_ALL:-0} + - CTXCE_AUTH_SHARED_TOKEN=${CTXCE_AUTH_SHARED_TOKEN} + - CTXCE_AUTH_ADMIN_TOKEN=${CTXCE_AUTH_ADMIN_TOKEN} + - CTXCE_AUTH_DB_URL=${CTXCE_AUTH_DB_URL} + - CTXCE_AUTH_SESSION_TTL_SECONDS=${CTXCE_AUTH_SESSION_TTL_SECONDS:-0} + - CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=${CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN:-0} # Indexing configuration - COLLECTION_NAME=${COLLECTION_NAME} @@ -366,6 +418,11 @@ services: - workspace_pvc:/work:rw - codebase_pvc:/work/.codebase:rw - upload_temp:/tmp/uploads + command: [ + "sh", + "-c", + "mkdir -p /work/.codebase && (chgrp -R 1000 /work/.codebase 2>/dev/null || true) && (chmod -R g+rwX /work/.codebase 2>/dev/null || true) && (find /work/.codebase -type d -exec chmod g+s {} + 2>/dev/null || true) && exec python scripts/upload_service.py" + ] healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8002/health"] interval: 30s diff --git a/requirements.txt b/requirements.txt index e901481c..3a2c0d43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ fastmcp==2.12.4 fastapi uvicorn[standard] python-multipart +jinja2 openai>=1.0 # Test-only diff --git a/scripts/admin_ui.py b/scripts/admin_ui.py new file mode 100644 index 00000000..8104f5a0 --- /dev/null +++ b/scripts/admin_ui.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional + +from fastapi import Request +from starlette.templating import Jinja2Templates +from jinja2 import select_autoescape + + +_TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates" +_templates = Jinja2Templates(directory=str(_TEMPLATES_DIR)) +_templates.env.autoescape = select_autoescape(enabled_extensions=("html", "xml"), default=True) + + +def render_admin_login( + request: Request, + error: Optional[str] = None, + status_code: int = 200, +) -> Any: + return _templates.TemplateResponse( + "admin/login.html", + {"request": request, "title": "CTXCE Admin Login", "error": error}, + status_code=status_code, + ) + + +def render_admin_bootstrap( + request: Request, + error: Optional[str] = None, + status_code: int = 200, +) -> Any: + return _templates.TemplateResponse( + "admin/bootstrap.html", + {"request": request, "title": "CTXCE Admin Bootstrap", "error": error}, + status_code=status_code, + ) + + +def render_admin_acl( + request: Request, + users: Any, + collections: Any, + grants: Any, + status_code: int = 200, +) -> Any: + return _templates.TemplateResponse( + "admin/acl.html", + { + "request": request, + "title": "CTXCE Admin ACL", + "users": users, + "collections": collections, + "grants": grants, + }, + status_code=status_code, + ) + + +def render_admin_error( + request: Request, + title: str, + message: str, + back_href: str = "/admin", + status_code: int = 400, +) -> Any: + return _templates.TemplateResponse( + "admin/error.html", + { + "request": request, + "title": title, + "message": message, + "back_href": back_href, + }, + status_code=status_code, + ) diff --git a/scripts/auth_backend.py b/scripts/auth_backend.py new file mode 100644 index 00000000..d298f6d2 --- /dev/null +++ b/scripts/auth_backend.py @@ -0,0 +1,621 @@ +"""Authentication backend for Context-Engine services. + +Provides a minimal, SQLite-backed user and session store with +password hashing and optional shared-token based session issuance. + +Design notes (PoC-friendly, forward compatible): + +- Storage schema is intentionally simple and portable (TEXT/INTEGER fields + only) so it can be migrated to a real RDBMS (Postgres/MySQL) later without + changing the logical model. +- AUTH_DB_URL accepts a SQLite-style URL today, but callers should treat it + as an abstract database URL; future versions may use Alembic-style schema + migrations and support multiple engines. +- Current focus is users + sessions only. In a fuller deployment, this module + is the natural place to grow organization and collection metadata, including + mapping users/orgs to existing Qdrant collections and enforcing collection- + level ACLs. + +Auth is fully opt-in via environment variables and can be reused by +multiple services (upload, dedicated auth service, MCP indexers, etc.). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sqlite3 +import uuid +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Dict, Optional, List + +# Configuration +WORK_DIR = os.environ.get("WORK_DIR", "/work") +AUTH_ENABLED = ( + str(os.environ.get("CTXCE_AUTH_ENABLED", "0")) + .strip() + .lower() + in {"1", "true", "yes", "on"} +) +_default_auth_db_path = os.path.join(WORK_DIR, ".codebase", "ctxce_auth.sqlite") +AUTH_DB_URL = os.environ.get("CTXCE_AUTH_DB_URL") or f"sqlite:///{_default_auth_db_path}" +AUTH_SHARED_TOKEN = os.environ.get("CTXCE_AUTH_SHARED_TOKEN") +COLLECTION_REGISTRY_ENABLED = AUTH_ENABLED +ACL_ALLOW_ALL = ( + str(os.environ.get("CTXCE_ACL_ALLOW_ALL", "0")).strip().lower() + in {"1", "true", "yes", "on"} +) +ALLOW_OPEN_TOKEN_LOGIN = ( + str(os.environ.get("CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN", "0")) + .strip() + .lower() + in {"1", "true", "yes", "on"} +) + +_SESSION_TTL_SECONDS_DEFAULT = 0 +try: + _raw_ttl = os.environ.get("CTXCE_AUTH_SESSION_TTL_SECONDS") + if _raw_ttl is not None and str(_raw_ttl).strip() != "": + AUTH_SESSION_TTL_SECONDS = int(str(_raw_ttl).strip()) + else: + AUTH_SESSION_TTL_SECONDS = _SESSION_TTL_SECONDS_DEFAULT +except Exception: + AUTH_SESSION_TTL_SECONDS = _SESSION_TTL_SECONDS_DEFAULT + + +class AuthDisabledError(Exception): + """Raised when auth is disabled via configuration.""" + + +class AuthInvalidToken(Exception): + """Raised when a shared token login attempt fails validation.""" + + +def _get_auth_db_path() -> str: + raw = AUTH_DB_URL or "" + if raw.startswith("sqlite///"): + return raw[len("sqlite///") :] + if raw.startswith("sqlite://"): + return raw[len("sqlite://") :] + return raw + + +@contextmanager +def _db_connection(): + path = _get_auth_db_path() + conn = sqlite3.connect(path) + try: + yield conn + finally: + conn.close() + + +def _ensure_db() -> None: + path = _get_auth_db_path() + if not path: + return + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + except Exception: + pass + with _db_connection() as conn: + with conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, user_id TEXT, created_at INTEGER, expires_at INTEGER, metadata_json TEXT)" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS users (id TEXT PRIMARY KEY, username TEXT UNIQUE NOT NULL, password_hash TEXT NOT NULL, created_at INTEGER NOT NULL, metadata_json TEXT, role TEXT NOT NULL DEFAULT 'user')" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS collections (id TEXT PRIMARY KEY, qdrant_collection TEXT UNIQUE NOT NULL, created_at INTEGER NOT NULL, metadata_json TEXT, is_deleted INTEGER NOT NULL DEFAULT 0)" + ) + conn.execute( + "CREATE TABLE IF NOT EXISTS collection_acl (collection_id TEXT NOT NULL, user_id TEXT NOT NULL, permission TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (collection_id, user_id))" + ) + try: + cur = conn.cursor() + cur.execute("PRAGMA table_info(users)") + cols = [r[1] for r in cur.fetchall() or []] + if "role" not in cols: + conn.execute( + "ALTER TABLE users ADD COLUMN role TEXT NOT NULL DEFAULT 'user'" + ) + except Exception: + pass + + +def _hash_password(password: str) -> str: + if not isinstance(password, str) or not password: + raise ValueError("Password is required") + salt = os.urandom(16) + iterations = 200_000 + dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return f"pbkdf2_sha256${iterations}${salt.hex()}${dk.hex()}" + + +def _verify_password(password: str, encoded: str) -> bool: + try: + scheme, iter_s, salt_hex, hash_hex = encoded.split("$", 3) + if scheme != "pbkdf2_sha256": + return False + iterations = int(iter_s) + salt = bytes.fromhex(salt_hex) + dk = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), salt, iterations) + return dk.hex() == hash_hex + except Exception: + return False + + +def create_user( + username: str, + password: str, + metadata: Optional[Dict[str, Any]] = None, + role: Optional[str] = None, +) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + path = _get_auth_db_path() + now_ts = int(datetime.now().timestamp()) + password_hash = _hash_password(password) + meta_json: Optional[str] = None + if metadata: + try: + meta_json = json.dumps(metadata) + except Exception: + meta_json = None + user_id = uuid.uuid4().hex + with _db_connection() as conn: + with conn: + desired_role = (str(role).strip().lower() if role is not None else "") + if desired_role and desired_role not in {"user", "admin"}: + raise ValueError("Invalid role") + role_val = desired_role or "user" + try: + cur = conn.cursor() + cur.execute("SELECT 1 FROM users LIMIT 1") + if not cur.fetchone(): + role_val = "admin" + except Exception: + role_val = "user" + conn.execute( + "INSERT INTO users (id, username, password_hash, created_at, metadata_json, role) VALUES (?, ?, ?, ?, ?, ?)", + (user_id, username, password_hash, now_ts, meta_json, role_val), + ) + return {"id": user_id, "user_id": user_id, "username": username, "role": role_val} + + +def _get_user_by_username(username: str) -> Optional[Dict[str, Any]]: + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT id, username, password_hash, created_at, metadata_json, role FROM users WHERE username = ?", + (username,), + ) + row = cur.fetchone() + if not row: + return None + return { + "id": row[0], + "username": row[1], + "password_hash": row[2], + "created_at": row[3], + "metadata_json": row[4], + "role": row[5], + } + + +def _get_user_role(user_id: str) -> Optional[str]: + uid = (user_id or "").strip() + if not uid: + return None + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT role FROM users WHERE id = ?", (uid,)) + row = cur.fetchone() + if not row: + return None + return str(row[0] or "").strip() or None + + +def is_admin_user(user_id: str) -> bool: + return (_get_user_role(user_id) or "").lower() == "admin" + + +def has_any_users() -> bool: + """Return True if at least one user exists. + + Used by HTTP layers to allow first-user bootstrap flows when the + database is empty. Raises AuthDisabledError when auth is disabled. + """ + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT 1 FROM users LIMIT 1") + row = cur.fetchone() + return bool(row) + + +def authenticate_user(username: str, password: str) -> Optional[Dict[str, Any]]: + user = _get_user_by_username(username) + if not user: + return None + if not _verify_password(password, user.get("password_hash") or ""): + return None + return user + + +def create_session( + user_id: str, + metadata: Optional[Dict[str, Any]] = None, + ttl_seconds: int = AUTH_SESSION_TTL_SECONDS, +) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + path = _get_auth_db_path() + now_ts = int(datetime.now().timestamp()) + ttl_val = int(ttl_seconds or 0) + if ttl_val <= 0: + expires_ts = 0 + else: + expires_ts = now_ts + ttl_val + meta_json: Optional[str] = None + if metadata: + try: + meta_json = json.dumps(metadata) + except Exception: + meta_json = None + session_id = uuid.uuid4().hex + with _db_connection() as conn: + with conn: + conn.execute( + "INSERT OR REPLACE INTO sessions (id, user_id, created_at, expires_at, metadata_json) VALUES (?, ?, ?, ?, ?)", + (session_id, user_id, now_ts, expires_ts, meta_json), + ) + return {"session_id": session_id, "user_id": user_id, "expires_at": expires_ts} + + +def create_session_for_token( + client: str, + workspace: Optional[str] = None, + token: Optional[str] = None, +) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + if AUTH_SHARED_TOKEN: + # When a shared token is configured, require it for all token-based sessions. + if not token or token != AUTH_SHARED_TOKEN: + raise AuthInvalidToken("Invalid auth token") + else: + # Harden default behavior: when auth is enabled but no shared token is configured, + # disable token-based login unless explicitly allowed via env. + if not ALLOW_OPEN_TOKEN_LOGIN: + raise AuthInvalidToken( + "Token-based login disabled (no shared token configured; set CTXCE_AUTH_SHARED_TOKEN " + "or CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=1 to enable)" + ) + user_id = client or "ctxce" + meta: Dict[str, Any] = {} + if workspace: + meta["workspace"] = workspace + return create_session(user_id=user_id, metadata=meta) + + +def validate_session(session_id: str) -> Optional[Dict[str, Any]]: + """Validate a session id and return its record if active. + + Returns a dict with keys {id, user_id, created_at, expires_at, metadata} + when valid, or None when missing/expired/unknown. Raises AuthDisabledError + when auth is disabled. + """ + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + sid = (session_id or "").strip() + if not sid: + return None + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT id, user_id, created_at, expires_at, metadata_json FROM sessions WHERE id = ?", + (sid,), + ) + row = cur.fetchone() + if not row: + return None + now_ts = int(datetime.now().timestamp()) + expires_ts = int(row[3] or 0) + if expires_ts and expires_ts < now_ts: + return None + if AUTH_SESSION_TTL_SECONDS > 0 and expires_ts: + remaining = expires_ts - now_ts + if remaining < AUTH_SESSION_TTL_SECONDS // 2: + new_expires_ts = now_ts + AUTH_SESSION_TTL_SECONDS + try: + with _db_connection() as conn2: + with conn2: + conn2.execute( + "UPDATE sessions SET expires_at = ? WHERE id = ?", + (new_expires_ts, sid), + ) + expires_ts = new_expires_ts + except Exception: + pass + meta: Optional[Dict[str, Any]] = None + raw_meta = row[4] + if isinstance(raw_meta, str) and raw_meta.strip(): + try: + obj = json.loads(raw_meta) + if isinstance(obj, dict): + meta = obj + except Exception: + meta = None + return { + "id": row[0], + "user_id": row[1], + "created_at": int(row[2] or 0), + "expires_at": expires_ts, + "metadata": meta or {}, + } + + +def ensure_collection(qdrant_collection: str, metadata: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + if not COLLECTION_REGISTRY_ENABLED: + raise AuthDisabledError("Collection registry not enabled") + name = (qdrant_collection or "").strip() + if not name: + raise ValueError("qdrant_collection is required") + _ensure_db() + now_ts = int(datetime.now().timestamp()) + meta_json: Optional[str] = None + if metadata: + try: + meta_json = json.dumps(metadata) + except Exception: + meta_json = None + with _db_connection() as conn: + with conn: + cur = conn.cursor() + cur.execute( + "SELECT id, qdrant_collection, created_at, metadata_json, is_deleted FROM collections WHERE qdrant_collection = ?", + (name,), + ) + row = cur.fetchone() + if row: + return { + "id": row[0], + "qdrant_collection": row[1], + "created_at": int(row[2] or 0), + "metadata_json": row[3], + "is_deleted": int(row[4] or 0), + } + + coll_id = uuid.uuid4().hex + conn.execute( + "INSERT INTO collections (id, qdrant_collection, created_at, metadata_json, is_deleted) VALUES (?, ?, ?, ?, 0)", + (coll_id, name, now_ts, meta_json), + ) + return { + "id": coll_id, + "qdrant_collection": name, + "created_at": now_ts, + "metadata_json": meta_json, + "is_deleted": 0, + } + + +def ensure_collections(collections: List[str]) -> int: + if not COLLECTION_REGISTRY_ENABLED: + raise AuthDisabledError("Collection registry not enabled") + names = [str(c).strip() for c in (collections or []) if str(c).strip()] + if not names: + return 0 + _ensure_db() + before_count = 0 + after_count = 0 + failures: List[str] = [] + try: + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT COUNT(1) FROM collections") + row = cur.fetchone() + if row: + before_count = int(row[0] or 0) + except Exception: + before_count = 0 + for name in names: + try: + ensure_collection(name) + except Exception as e: + failures.append(f"{name}: {e}") + continue + try: + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT COUNT(1) FROM collections") + row = cur.fetchone() + if row: + after_count = int(row[0] or 0) + except Exception: + after_count = before_count + + delta = max(0, after_count - before_count) + if failures and len(failures) >= len(names) and delta == 0: + raise RuntimeError("Failed to sync collections registry: " + "; ".join(failures[:3])) + return delta + + +def grant_collection_access(user_id: str, qdrant_collection: str, permission: str = "read") -> Dict[str, Any]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + uid = (user_id or "").strip() + perm = (permission or "read").strip() or "read" + if not uid: + raise ValueError("user_id is required") + coll = ensure_collection(qdrant_collection) + now_ts = int(datetime.now().timestamp()) + with _db_connection() as conn: + with conn: + conn.execute( + "INSERT OR REPLACE INTO collection_acl (collection_id, user_id, permission, created_at) VALUES (?, ?, ?, ?)", + (coll.get("id"), uid, perm, now_ts), + ) + return {"collection_id": coll.get("id"), "user_id": uid, "permission": perm} + + +def revoke_collection_access(user_id: str, qdrant_collection: str) -> bool: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + uid = (user_id or "").strip() + name = (qdrant_collection or "").strip() + if not uid: + raise ValueError("user_id is required") + if not name: + raise ValueError("qdrant_collection is required") + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT c.id FROM collections c WHERE c.qdrant_collection = ? AND c.is_deleted = 0", + (name,), + ) + row = cur.fetchone() + if not row: + return False + coll_id = row[0] + with conn: + cur.execute( + "DELETE FROM collection_acl WHERE collection_id = ? AND user_id = ?", + (coll_id, uid), + ) + return bool(cur.rowcount and cur.rowcount > 0) + + +def has_collection_access(user_id: str, qdrant_collection: str, permission: str = "read") -> bool: + if ACL_ALLOW_ALL: + return True + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + uid = (user_id or "").strip() + if not uid: + return False + if is_admin_user(uid): + return True + name = (qdrant_collection or "").strip() + if not name: + return False + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT c.id FROM collections c WHERE c.qdrant_collection = ? AND c.is_deleted = 0", + (name,), + ) + row = cur.fetchone() + if not row: + return False + coll_id = row[0] + cur.execute( + "SELECT permission FROM collection_acl WHERE collection_id = ? AND user_id = ?", + (coll_id, uid), + ) + perm_row = cur.fetchone() + if not perm_row: + return False + granted = str(perm_row[0] or "").strip().lower() + want = (permission or "read").strip().lower() + if granted == "admin": + return True + if want == "read": + return granted in {"read", "write"} + if want == "write": + return granted == "write" + return granted == want + + +def list_users() -> List[Dict[str, Any]]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute("SELECT id, username, created_at, role FROM users ORDER BY created_at ASC") + rows = cur.fetchall() or [] + out: List[Dict[str, Any]] = [] + for r in rows: + out.append( + { + "id": r[0], + "username": r[1], + "created_at": int(r[2] or 0), + "role": r[3], + } + ) + return out + + +def list_collections(include_deleted: bool = False) -> List[Dict[str, Any]]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + if include_deleted: + cur.execute( + "SELECT id, qdrant_collection, created_at, is_deleted FROM collections ORDER BY created_at ASC" + ) + else: + cur.execute( + "SELECT id, qdrant_collection, created_at, is_deleted FROM collections WHERE is_deleted = 0 ORDER BY created_at ASC" + ) + rows = cur.fetchall() or [] + out: List[Dict[str, Any]] = [] + for r in rows: + out.append( + { + "id": r[0], + "qdrant_collection": r[1], + "created_at": int(r[2] or 0), + "is_deleted": int(r[3] or 0), + } + ) + return out + + +def list_collection_acl() -> List[Dict[str, Any]]: + if not AUTH_ENABLED: + raise AuthDisabledError("Auth not enabled") + _ensure_db() + with _db_connection() as conn: + cur = conn.cursor() + cur.execute( + "SELECT a.collection_id, c.qdrant_collection, a.user_id, u.username, u.role, a.permission, a.created_at " + "FROM collection_acl a " + "JOIN collections c ON c.id = a.collection_id " + "LEFT JOIN users u ON u.id = a.user_id " + "WHERE c.is_deleted = 0 " + "ORDER BY c.qdrant_collection ASC, u.username ASC" + ) + rows = cur.fetchall() or [] + out: List[Dict[str, Any]] = [] + for r in rows: + out.append( + { + "collection_id": r[0], + "qdrant_collection": r[1], + "user_id": r[2], + "username": r[3], + "user_role": r[4], + "permission": r[5], + "created_at": int(r[6] or 0), + } + ) + return out diff --git a/scripts/ctx.py b/scripts/ctx.py index b1cc3c00..2c570a28 100755 --- a/scripts/ctx.py +++ b/scripts/ctx.py @@ -276,6 +276,30 @@ def parse_mcp_response(result: Dict[str, Any]) -> Optional[Dict[str, Any]]: return {"raw": text} +def _extract_tool_error(result: Dict[str, Any]) -> Optional[str]: + """Best-effort extraction of a tool-level error message from MCP response. + + Handles FastMCP-style isError + content.text as well as structuredContent.result.error. + Returns a human-readable error string when present, or None when no error detected. + """ + try: + res = result.get("result") or {} + if isinstance(res, dict) and res.get("isError") is True: + content = res.get("content") or [] + if isinstance(content, list) and content and isinstance(content[0], dict): + text = content[0].get("text") + if isinstance(text, str) and text.strip(): + return text.strip() + sc = res.get("structuredContent") or {} + rs = sc.get("result") or {} + err = rs.get("error") + if isinstance(err, str) and err.strip(): + return err.strip() + except Exception: + return None + return None + + def _compress_snippet(snippet: str, max_lines: int = 6) -> str: """Compact, high-signal subset of a code snippet. @@ -704,6 +728,40 @@ def _generate_plan(enhanced_prompt: str, context: str, note: str) -> str: "<|start_of_role|>assistant<|end_of_role|>" ) + runtime_kind = str(os.environ.get("REFRAG_RUNTIME", "llamacpp")).strip().lower() + + # GLM path mirrors rewrite_prompt behavior + if runtime_kind == "glm": + try: + from refrag_glm import GLMRefragClient # type: ignore + + client = GLMRefragClient() + response = client.client.chat.completions.create( + model=os.environ.get("GLM_MODEL", "glm-4.6"), + messages=[ + {"role": "system", "content": system_msg}, + {"role": "user", "content": user_msg}, + ], + max_tokens=200, + temperature=0.3, + stream=False, + ) + plan = ( + (response.choices[0].message.content if response and response.choices else "") + or "" + ).strip() + if not plan: + # Fall through to llama.cpp path + runtime_kind = "llamacpp" + else: + if "EXECUTION PLAN" not in plan.upper(): + plan = "EXECUTION PLAN:\n" + plan + return plan + except Exception as e: + sys.stderr.write(f"[DEBUG] Plan generation (GLM) failed, falling back to llama.cpp: {type(e).__name__}: {e}\n") + sys.stderr.flush() + runtime_kind = "llamacpp" + decoder_url = DECODER_URL # Safety: restrict to local decoder hosts parsed = urlparse(decoder_url) @@ -987,6 +1045,14 @@ def fetch_context(query: str, **filters) -> Tuple[str, str]: sys.stderr.flush() return "", f"Context retrieval failed: {error_msg}" + # Surface tool-level errors (including auth failures) explicitly instead of + # silently treating them as "no context". + tool_err = _extract_tool_error(result) + if tool_err: + sys.stderr.write(f"[DEBUG] repo_search tool error: {tool_err}\n") + sys.stderr.flush() + return "", f"Context retrieval failed: {tool_err}" + data = parse_mcp_response(result) if not data: sys.stderr.write("[DEBUG] repo_search returned no data\n") diff --git a/scripts/health_check.py b/scripts/health_check.py index 67e80e2e..574483cc 100644 --- a/scripts/health_check.py +++ b/scripts/health_check.py @@ -14,6 +14,7 @@ from scripts.utils import sanitize_vector_name +from scripts.auth_backend import ensure_collections, AuthDisabledError def assert_true(cond: bool, msg: str): @@ -44,6 +45,16 @@ def main(): collections_response = client.get_collections() collections = [c.name for c in collections_response.collections] print(f"Found collections: {collections}") + try: + created = ensure_collections(collections) + if created: + print(f"[OK] Synced collections registry (new entries: {created})") + else: + print("[OK] Synced collections registry") + except AuthDisabledError: + pass + except Exception as e: + print(f"[WARN] Failed to sync collections registry: {e}") except Exception as e: print(f"Error getting collections: {e}") sys.exit(1) diff --git a/scripts/mcp_auth.py b/scripts/mcp_auth.py new file mode 100644 index 00000000..2b13c791 --- /dev/null +++ b/scripts/mcp_auth.py @@ -0,0 +1,74 @@ +import os +from typing import Any, Dict, Optional + +try: + from scripts.logger import ValidationError +except Exception: + + class ValidationError(Exception): + pass + + +try: + from scripts.auth_backend import ( + AUTH_ENABLED as AUTH_ENABLED_AUTH, + ACL_ALLOW_ALL as ACL_ALLOW_ALL_AUTH, + validate_session as _auth_validate_session, + has_collection_access as _has_collection_access, + ) +except Exception as _auth_backend_import_exc: + _AUTH_BACKEND_IMPORT_ERROR = repr(_auth_backend_import_exc) + AUTH_ENABLED_AUTH = ( + str(os.environ.get("CTXCE_AUTH_ENABLED", "0")).strip().lower() in {"1", "true", "yes", "on"} + ) + ACL_ALLOW_ALL_AUTH = ( + str(os.environ.get("CTXCE_ACL_ALLOW_ALL", "0")).strip().lower() in {"1", "true", "yes", "on"} + ) + + def _auth_validate_session(session_id: str): # type: ignore[no-redef] + if AUTH_ENABLED_AUTH: + raise ValidationError( + f"Auth backend unavailable (import failed): {_AUTH_BACKEND_IMPORT_ERROR}" + ) + return None + + def _has_collection_access( + user_id: str, qdrant_collection: str, permission: str = "read" + ) -> bool: # type: ignore[no-redef] + if AUTH_ENABLED_AUTH: + raise ValidationError( + f"Auth backend unavailable (import failed): {_AUTH_BACKEND_IMPORT_ERROR}" + ) + return True + + +ACL_ENFORCE = ( + str(os.environ.get("CTXCE_MCP_ACL_ENFORCE", "0")).strip().lower() + in {"1", "true", "yes", "on"} +) + + +def require_auth_session(session: Optional[str]) -> Optional[Dict[str, Any]]: + if not AUTH_ENABLED_AUTH: + return None + sid = (session or "").strip() + if not sid: + raise ValidationError("Missing session for authorized operation") + info = _auth_validate_session(sid) + if not info: + raise ValidationError("Invalid or expired session") + return info + + +def require_collection_access(user_id: Optional[str], collection: str, perm: str) -> None: + if not ACL_ENFORCE or not AUTH_ENABLED_AUTH: + return + if ACL_ALLOW_ALL_AUTH: + return + uid = (user_id or "").strip() + if not uid: + raise ValidationError("Not authorized: missing user id") + if not _has_collection_access(uid, collection, perm): + raise ValidationError( + f"Forbidden: {perm} access to collection '{collection}' denied" + ) diff --git a/scripts/mcp_indexer_server.py b/scripts/mcp_indexer_server.py index c8833ccb..c3da499f 100644 --- a/scripts/mcp_indexer_server.py +++ b/scripts/mcp_indexer_server.py @@ -123,6 +123,12 @@ def safe_bool(value, default=False, logger=None, context=""): return default +from scripts.mcp_auth import ( + require_auth_session as _require_auth_session, + require_collection_access as _require_collection_access, +) + + # Global lock to guard temporary env toggles used during ReFRAG retrieval/decoding _ENV_LOCK = threading.Lock() @@ -952,7 +958,7 @@ def _detect_current_repo() -> str | None: @mcp.tool() async def qdrant_index_root( - recreate: Optional[bool] = None, collection: Optional[str] = None + recreate: Optional[bool] = None, collection: Optional[str] = None, session: Optional[str] = None ) -> Dict[str, Any]: """Initialize or refresh the vector index for the workspace root (/work). @@ -970,6 +976,8 @@ async def qdrant_index_root( - Omit fields instead of sending null values. - Safe to call repeatedly; unchanged files are skipped by the indexer. """ + sess = _require_auth_session(session) + # Leniency: if clients embed JSON in 'collection' (and include 'recreate'), parse it try: if _looks_jsonish_string(collection): @@ -1003,6 +1011,8 @@ async def qdrant_index_root( except Exception: coll = _default_collection() + _require_collection_access((sess or {}).get("user_id") if sess else None, coll, "write") + env = os.environ.copy() env["QDRANT_URL"] = QDRANT_URL env["COLLECTION_NAME"] = coll @@ -1565,6 +1575,7 @@ async def qdrant_index( subdir: Optional[str] = None, recreate: Optional[bool] = None, collection: Optional[str] = None, + session: Optional[str] = None, ) -> Dict[str, Any]: """Index the workspace (/work) or a specific subdirectory. @@ -1581,6 +1592,8 @@ async def qdrant_index( - Paths are sandboxed to /work; attempts to escape will be rejected. - Omit fields rather than sending null values. """ + sess = _require_auth_session(session) + # Leniency: parse JSON-ish payloads mistakenly sent in 'collection' or 'subdir' try: if _looks_jsonish_string(collection): @@ -1631,6 +1644,8 @@ async def qdrant_index( except Exception: coll = _default_collection() + _require_collection_access((sess or {}).get("user_id") if sess else None, coll, "write") + env = os.environ.copy() env["QDRANT_URL"] = QDRANT_URL env["COLLECTION_NAME"] = coll @@ -1658,11 +1673,14 @@ async def qdrant_index( @mcp.tool() async def set_session_defaults( collection: Any = None, + mode: Any = None, + under: Any = None, + language: Any = None, session: Any = None, ctx: Context = None, **kwargs, ) -> Dict[str, Any]: - """Set defaults (e.g., collection) for subsequent calls. + """Set defaults (e.g., collection, mode, under) for subsequent calls. Behavior: - If request Context is available, persist defaults per-connection so later calls on @@ -1674,20 +1692,34 @@ async def set_session_defaults( if _extra: if (collection is None or (isinstance(collection, str) and collection.strip() == "")) and _extra.get("collection") is not None: collection = _extra.get("collection") + if (mode is None or (isinstance(mode, str) and str(mode).strip() == "")) and _extra.get("mode") is not None: + mode = _extra.get("mode") + if (under is None or (isinstance(under, str) and str(under).strip() == "")) and _extra.get("under") is not None: + under = _extra.get("under") + if (language is None or (isinstance(language, str) and str(language).strip() == "")) and _extra.get("language") is not None: + language = _extra.get("language") if (session is None or (isinstance(session, str) and str(session).strip() == "")) and _extra.get("session") is not None: session = _extra.get("session") except Exception: pass defaults: Dict[str, Any] = {} - if isinstance(collection, str) and collection.strip(): - defaults["collection"] = str(collection).strip() + unset_keys: set[str] = set() + for _key, _val in (("collection", collection), ("mode", mode), ("under", under), ("language", language)): + if isinstance(_val, str): + _s = _val.strip() + if _s: + defaults[_key] = _s + else: + unset_keys.add(_key) # Per-connection storage (preferred) try: - if ctx is not None and getattr(ctx, "session", None) is not None and defaults: + if ctx is not None and getattr(ctx, "session", None) is not None and (defaults or unset_keys): with _SESSION_CTX_LOCK: existing2 = SESSION_DEFAULTS_BY_SESSION.get(ctx.session) or {} + for _k in unset_keys: + existing2.pop(_k, None) existing2.update(defaults) SESSION_DEFAULTS_BY_SESSION[ctx.session] = existing2 except Exception: @@ -1698,9 +1730,11 @@ async def set_session_defaults( if not sid: sid = uuid.uuid4().hex[:12] try: - if defaults: + if defaults or unset_keys: with _SESSION_LOCK: existing = SESSION_DEFAULTS.get(sid) or {} + for _k in unset_keys: + existing.pop(_k, None) existing.update(defaults) SESSION_DEFAULTS[sid] = existing except Exception: @@ -1797,6 +1831,8 @@ async def repo_search( - path_glob=["scripts/**","**/*.py"], language="python" - symbol="context_answer", under="scripts" """ + sess = _require_auth_session(session) + # Handle queries alias (explicit parameter) if queries is not None and (query is None or (isinstance(query, str) and str(query).strip() == "")): query = queries @@ -1973,35 +2009,61 @@ def _to_str(x, default=""): ) highlight_snippet = _to_bool(highlight_snippet, True) - # Optional mode knob: "code_first" (default for IDE), "docs_first", "balanced" - mode_str = _to_str(mode, "").strip().lower() - - # Resolve collection precedence: explicit > per-connection defaults > token defaults > env default + # Resolve collection and related hints: explicit > per-connection defaults > token defaults > env coll_hint = _to_str(collection, "").strip() + mode_hint = _to_str(mode, "").strip() + under_hint = _to_str(under, "").strip() + lang_hint = _to_str(language, "").strip() # 1) Per-connection defaults via ctx (no token required) - if (not coll_hint) and ctx is not None and getattr(ctx, "session", None) is not None: + if ctx is not None and getattr(ctx, "session", None) is not None: try: with _SESSION_CTX_LOCK: _d2 = SESSION_DEFAULTS_BY_SESSION.get(ctx.session) or {} - _sc2 = str((_d2.get("collection") or "")).strip() - if _sc2: - coll_hint = _sc2 + if not coll_hint: + _sc2 = str((_d2.get("collection") or "")).strip() + if _sc2: + coll_hint = _sc2 + if not mode_hint: + _sm2 = str((_d2.get("mode") or "")).strip() + if _sm2: + mode_hint = _sm2 + if not under_hint: + _su2 = str((_d2.get("under") or "")).strip() + if _su2: + under_hint = _su2 + if not lang_hint: + _sl2 = str((_d2.get("language") or "")).strip() + if _sl2: + lang_hint = _sl2 except Exception: pass # 2) Legacy token-based defaults - if (not coll_hint) and sid: + if sid: try: with _SESSION_LOCK: _d = SESSION_DEFAULTS.get(sid) or {} - _sc = str((_d.get("collection") or "")).strip() - if _sc: - coll_hint = _sc + if not coll_hint: + _sc = str((_d.get("collection") or "")).strip() + if _sc: + coll_hint = _sc + if not mode_hint: + _sm = str((_d.get("mode") or "")).strip() + if _sm: + mode_hint = _sm + if not under_hint: + _su = str((_d.get("under") or "")).strip() + if _su: + under_hint = _su + if not lang_hint: + _sl = str((_d.get("language") or "")).strip() + if _sl: + lang_hint = _sl except Exception: pass - # 3) Environment default + # 3) Environment default (collection only for now) env_coll = (os.environ.get("DEFAULT_COLLECTION") or os.environ.get("COLLECTION_NAME") or "").strip() if (not coll_hint) and env_coll: coll_hint = env_coll @@ -2010,6 +2072,19 @@ def _to_str(x, default=""): env_fallback = (os.environ.get("DEFAULT_COLLECTION") or os.environ.get("COLLECTION_NAME") or "my-collection").strip() collection = coll_hint or env_fallback + _require_collection_access((sess or {}).get("user_id") if sess else None, collection, "read") + + # Optional mode knob: "code_first" (default for IDE), "docs_first", "balanced" + if not mode: + mode = mode_hint + mode_str = _to_str(mode, "").strip().lower() + + # Apply defaults for language / under when explicit args are empty + if not language: + language = lang_hint + if not under: + under = under_hint + language = _to_str(language, "").strip() under = _to_str(under, "").strip() kind = _to_str(kind, "").strip() @@ -2440,6 +2515,7 @@ def _doc_for(obj: dict) -> str: language=language or None, under=under or None, model=model, + collection=collection, ) if items: results = items @@ -2460,6 +2536,8 @@ def _doc_for(obj: dict) -> str: "--limit", str(int(rerank_return_m)), ] + if collection: + rcmd += ["--collection", str(collection)] if language: rcmd += ["--language", language] if under: @@ -2942,6 +3020,7 @@ async def search_tests_for( context_lines: Any = None, under: Any = None, language: Any = None, + session: Any = None, compact: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: @@ -2980,6 +3059,7 @@ async def search_tests_for( under=under, language=language, path_glob=globs, + session=session, compact=compact, kwargs={k: v for k, v in _kwargs.items() if k not in {"path_glob"}}, ) @@ -2992,6 +3072,7 @@ async def search_config_for( include_snippet: Any = None, context_lines: Any = None, under: Any = None, + session: Any = None, compact: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: @@ -3033,6 +3114,7 @@ async def search_config_for( include_snippet=include_snippet, context_lines=context_lines, under=under, + session=session, path_glob=globs, compact=compact, kwargs={k: v for k, v in _kwargs.items() if k not in {"path_glob"}}, @@ -3044,6 +3126,7 @@ async def search_callers_for( query: Any = None, limit: Any = None, language: Any = None, + session: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: """Heuristic search for callers/usages of a symbol. @@ -3059,6 +3142,7 @@ async def search_callers_for( query=query, limit=limit, language=language, + session=session, kwargs=kwargs, ) @@ -3068,6 +3152,7 @@ async def search_importers_for( query: Any = None, limit: Any = None, language: Any = None, + session: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: """Find files likely importing or referencing a module/symbol. @@ -3111,6 +3196,7 @@ async def search_importers_for( limit=limit, language=language, path_glob=globs, + session=session, kwargs={k: v for k, v in _kwargs.items() if k not in {"path_glob"}}, ) @@ -3581,6 +3667,7 @@ async def context_search( ext: Any = None, not_: Any = None, case: Any = None, + session: Any = None, compact: Any = None, # Repo scoping (cross-codebase isolation) repo: Any = None, # str, list[str], or "*" to search all repos @@ -4117,6 +4204,7 @@ def _maybe_dict(val: Any) -> Dict[str, Any]: case=case, compact=False, repo=repo, # Cross-codebase isolation + session=session, ) # Optional debug @@ -7850,6 +7938,7 @@ async def code_search( ext: Any = None, not_: Any = None, case: Any = None, + session: Any = None, compact: Any = None, kwargs: Any = None, ) -> Dict[str, Any]: @@ -7880,6 +7969,7 @@ async def code_search( ext=ext, not_=not_, case=case, + session=session, compact=compact, kwargs=kwargs, ) @@ -8032,6 +8122,8 @@ async def info_request( include_explanation: bool = None, # Relationship mapping include_relationships: bool = None, + # Auth/session (passed through to repo_search) + session: str = None, # Optional filters (pass-through to repo_search) limit: int = None, language: str = None, @@ -8109,6 +8201,7 @@ async def info_request( query=query, limit=eff_limit, per_path=3, # Better default for info requests + session=session, include_snippet=eff_snippet, context_lines=eff_context, language=language, diff --git a/scripts/mcp_memory_server.py b/scripts/mcp_memory_server.py index 6644fe07..9fbcd404 100644 --- a/scripts/mcp_memory_server.py +++ b/scripts/mcp_memory_server.py @@ -4,6 +4,17 @@ import threading from weakref import WeakKeyDictionary +# Ensure repo roots are importable so 'scripts' resolves inside container +import sys as _sys +_roots_env = os.environ.get("WORK_ROOTS", "") +_roots = [p.strip() for p in _roots_env.split(",") if p.strip()] or ["/work", "/app"] +try: + for _root in _roots: + if _root and _root not in _sys.path: + _sys.path.insert(0, _root) +except Exception: + pass + # FastMCP server and request Context (ctx) for per-connection state try: @@ -13,6 +24,11 @@ from mcp.server.fastmcp import FastMCP # type: ignore Context = Any # type: ignore +from scripts.mcp_auth import ( + require_auth_session as _require_auth_session, + require_collection_access as _require_collection_access, +) + from qdrant_client import QdrantClient, models # Env @@ -27,41 +43,8 @@ EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5") # Minimal embedding via fastembed (CPU) -from fastembed import TextEmbedding # Single-process embedding model cache (avoid re-initializing fastembed on each call) -_EMBED_MODEL = None -_EMBED_LOCK = threading.Lock() - -def _get_embedding_model(): - global _EMBED_MODEL - m = _EMBED_MODEL - if m is None: - with _EMBED_LOCK: - m = _EMBED_MODEL - if m is None: - m = TextEmbedding(model_name=EMBEDDING_MODEL) - # Best-effort warmup to load weights once - try: - _ = list(m.embed(["memory", "search"])) - except Exception: - pass - _EMBED_MODEL = m - return m - -# Ensure repo roots are importable so 'scripts' resolves inside container -import sys as _sys -_roots_env = os.environ.get("WORK_ROOTS", "") -_roots = [p.strip() for p in _roots_env.split(",") if p.strip()] or ["/work", "/app"] -try: - for _root in _roots: - if _root and _root not in _sys.path: - _sys.path.insert(0, _root) -except Exception: - pass - -# Map model to named vector used in indexer - # Use shared utils for consistent vector naming and lexical hashing from scripts.utils import sanitize_vector_name as _sanitize_vector_name @@ -96,6 +79,10 @@ def _get_embedding_model(): m = _EMBED_MODEL_CACHE.get(EMBEDDING_MODEL) if m is None: m = TextEmbedding(model_name=EMBEDDING_MODEL) + try: + _ = next(m.embed(["warmup"])) + except Exception: + pass _EMBED_MODEL_CACHE[EMBEDDING_MODEL] = m return m @@ -153,6 +140,8 @@ def _inner(fn): SESSION_DEFAULTS_BY_SESSION: "WeakKeyDictionary[Any, Dict[str, Any]]" = WeakKeyDictionary() + + def _start_readyz_server(): try: from http.server import BaseHTTPRequestHandler, HTTPServer @@ -218,6 +207,9 @@ def _ensure_collection(name: str): # Choose dense dimension based on config: probe (default) vs env-configured if MEMORY_PROBE_EMBED_DIM: try: + # Probe dimension without populating the shared model cache. + # This preserves the "cache loads on first tool call" behavior and + # keeps MEMORY_COLD_SKIP_DENSE semantics unchanged. from fastembed import TextEmbedding _model_probe = TextEmbedding(model_name=EMBEDDING_MODEL) _dense_vec = next(_model_probe.embed(["probe"])) @@ -256,7 +248,11 @@ def _ensure_collection(name: str): except Exception: pass - client.create_collection(collection_name=name, vectors_config=vectors_cfg) + client.create_collection( + collection_name=name, + vectors_config=vectors_cfg, + hnsw_config=models.HnswConfigDiff(m=16, ef_construct=256), + ) vector_names = list(vectors_cfg.keys()) print(f"[MEMORY_SERVER] Created collection '{name}' with vectors: {vector_names}") return True @@ -354,7 +350,9 @@ def store( First call may be slower because the embedding model loads lazily. """ + sess = _require_auth_session(session) coll = _resolve_collection(collection, session=session, ctx=ctx, extra_kwargs=kwargs) + _require_collection_access((sess or {}).get("user_id"), coll, "write") _ensure_once(coll) model = _get_embedding_model() dense = next(model.embed([str(information)])).tolist() @@ -388,7 +386,9 @@ def find( Cold-start option: set MEMORY_COLD_SKIP_DENSE=1 to skip dense embedding until the model is cached (useful on slow storage). """ + sess = _require_auth_session(session) coll = _resolve_collection(collection, session=session, ctx=ctx, extra_kwargs=kwargs) + _require_collection_access((sess or {}).get("user_id") if sess else None, coll, "read") _ensure_once(coll) use_dense = True diff --git a/scripts/mcp_router.py b/scripts/mcp_router.py index 319a4420..e016ad15 100644 --- a/scripts/mcp_router.py +++ b/scripts/mcp_router.py @@ -564,9 +564,11 @@ def _mcp_handshake(base_url: str, timeout: float = 30.0) -> Dict[str, str]: sid = j.get("sessionId") except Exception: sid = None - if not sid: - raise RuntimeError("MCP handshake failed: no session id") - headers["Mcp-Session-Id"] = sid + # Tolerate servers (e.g., streamable-http bridge) that do not emit a session id header. + # In that case, proceed without attaching Mcp-Session-Id; downstream calls will still work + # for bridges that manage their own session lifecycle. + if sid: + headers["Mcp-Session-Id"] = sid # Send initialized notification (no id required) try: _post_raw_retry(base_url, {"jsonrpc": "2.0", "method": "notifications/initialized"}, headers, timeout=timeout) diff --git a/scripts/memory_restore.py b/scripts/memory_restore.py index c2f4c01e..c85e925e 100644 --- a/scripts/memory_restore.py +++ b/scripts/memory_restore.py @@ -27,7 +27,7 @@ try: from qdrant_client import QdrantClient - from qdrant_client.models import VectorParams, Distance + from qdrant_client.models import VectorParams, Distance, HnswConfigDiff from fastembed import TextEmbedding except ImportError as e: print(f"ERROR: Missing required dependency: {e}") @@ -84,9 +84,10 @@ def ensure_collection_exists( size=vector_dimension, distance=Distance.COSINE ) - } + }, + hnsw_config=HnswConfigDiff(m=16, ef_construct=256), ) - print(f"✅ Created collection '{collection_name}' with {vector_dimension}-dim vectors") + print(f"Created collection '{collection_name}' with {vector_dimension}-dim vectors") except Exception as e: raise RuntimeError(f"Failed to create collection '{collection_name}': {e}") @@ -270,7 +271,7 @@ def restore_memories( error_count += len(batch_points) # Final statistics - print(f"\n✅ Memory restore completed!") + print(f"\n Memory restore completed!") print(f" Total memories in backup: {len(memories)}") print(f" Successfully restored: {restored_count}") print(f" Skipped (already exists): {skipped_count}") @@ -394,13 +395,13 @@ def main(): ) if result["success"]: - print(f"\n🎉 Memory restoration completed successfully!") + print(f"\n Memory restoration completed successfully!") else: - print(f"\n❌ Memory restoration failed!") + print(f"\n Memory restoration failed!") sys.exit(1) except Exception as e: - print(f"\n❌ Error during restoration: {e}") + print(f"\n Error during restoration: {e}") sys.exit(1) diff --git a/scripts/remote_upload_client.py b/scripts/remote_upload_client.py index b6bd54dd..82348799 100644 --- a/scripts/remote_upload_client.py +++ b/scripts/remote_upload_client.py @@ -29,6 +29,7 @@ import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +from scripts.upload_auth_utils import get_auth_session # Configure logging logging.basicConfig(level=logging.INFO) @@ -492,6 +493,7 @@ def _get_temp_bundle_dir(self) -> Path: if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(prefix="delta_bundle_") return Path(self.temp_dir) + # CLI is stateless - sequence tracking is handled by server def detect_file_changes(self, changed_paths: List[Path]) -> Dict[str, List]: @@ -819,7 +821,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, "workspace_path": self.workspace_path, "collection_name": self.collection_name, "created_at": created_at, - # CLI is stateless - server will assign sequence numbers + # CLI is stateless - server handles sequence numbers "sequence_number": None, # Server will assign "parent_sequence": None, # Server will determine "operations": { @@ -898,76 +900,75 @@ def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, # Check bundle size (server-side enforcement) bundle_size = os.path.getsize(bundle_path) - with open(bundle_path, 'rb') as bundle_file: - files = { - 'bundle': (f"{manifest['bundle_id']}.tar.gz", bundle_file, 'application/gzip') - } + files = { + "bundle": open(bundle_path, "rb"), + } + data = { + "workspace_path": self._translate_to_container_path(self.workspace_path), + "collection_name": self.collection_name, + "sequence_number": manifest.get("sequence_number"), + "force": False, + "source_path": self.workspace_path, + "logical_repo_id": _compute_logical_repo_id(self.workspace_path), + } - data = { - 'workspace_path': self._translate_to_container_path(self.workspace_path), - 'collection_name': self.collection_name, - # CLI is stateless - server handles sequence numbers - 'force': 'false', - 'source_path': self.workspace_path, - } - if getattr(self, "logical_repo_id", None): - data['logical_repo_id'] = self.logical_repo_id + sess = get_auth_session(self.upload_endpoint) + if sess: + data["session"] = sess - logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") + if getattr(self, "logical_repo_id", None): + data['logical_repo_id'] = self.logical_repo_id - response = self.session.post( - f"{self.upload_endpoint}/api/v1/delta/upload", - files=files, - data=data, - timeout=(10, self.timeout) - ) + logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") + + response = self.session.post( + f"{self.upload_endpoint}/api/v1/delta/upload", + files=files, + data=data, + timeout=(10, self.timeout) + ) - if response.status_code == 200: - result = response.json() - logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + result = None + try: + result = response.json() + except Exception: + result = None - seq = None + if response.status_code == 200 and isinstance(result, dict) and result.get("success", False): + logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + seq = result.get("sequence_number") + if seq is not None: try: - seq = result.get("sequence_number") + manifest["sequence"] = seq except Exception: - seq = None - if seq is not None: - try: - manifest["sequence"] = seq - except Exception: - pass - - poll_result = self._poll_after_timeout(manifest) - if poll_result.get("success"): - combined = dict(result) - for k, v in poll_result.items(): - if k in ("success", "error"): - continue - if k not in combined: - combined[k] = v - return combined - - logger.warning("[remote_upload] Upload accepted but polling did not confirm processing; returning original result") - return result - - # Handle error - error_msg = f"Upload failed with status {response.status_code}" - try: - error_detail = response.json() - error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') - error_msg += f": {error_detail_msg}" - error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') - except: - error_msg += f": {response.text[:200]}" - error_code = "HTTP_ERROR" + pass + return result + + # Handle error + error_msg = f"Upload failed with status {response.status_code}" + try: + error_detail = result if isinstance(result, dict) else response.json() + error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') + error_msg += f": {error_detail_msg}" + error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') + except Exception: + error_msg += f": {response.text[:200]}" + error_code = "HTTP_ERROR" + + # Special-case 401 to make auth issues obvious to users + if response.status_code == 401: + if error_code in {None, "HTTP_ERROR"}: + error_code = "UNAUTHORIZED" + # Always append a clear hint for auth failures + error_msg += " (unauthorized; please log in with `ctxce auth login` and retry)" - last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} + last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} - # Don't retry on client errors (except 429) - if 400 <= response.status_code < 500 and response.status_code != 429: - return last_error + # Don't retry on client errors (except 429) + if 400 <= response.status_code < 500 and response.status_code != 429: + return last_error - logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") + logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") except requests.exceptions.ConnectTimeout as e: last_error = {"success": False, "error": {"code": "TIMEOUT_ERROR", "message": f"Upload timeout: {str(e)}"}} diff --git a/scripts/rerank_local.py b/scripts/rerank_local.py index 035d9467..4a0f8790 100644 --- a/scripts/rerank_local.py +++ b/scripts/rerank_local.py @@ -14,7 +14,6 @@ ort = None Tokenizer = None -COLLECTION = os.environ.get("COLLECTION_NAME", "codebase") MODEL_NAME = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-base-en-v1.5") QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333") API_KEY = os.environ.get("QDRANT_API_KEY") @@ -175,11 +174,12 @@ def dense_results( query: str, flt, topk: int, + collection_name: str, ) -> List[Any]: vec = next(model.embed([query])).tolist() try: qp = client.query_points( - collection_name=COLLECTION, + collection_name=collection_name, query=vec, using=vec_name, query_filter=flt, @@ -190,7 +190,7 @@ def dense_results( return getattr(qp, "points", qp) except Exception: res = client.search( - collection_name=COLLECTION, + collection_name=collection_name, query_vector={"name": vec_name, "vector": vec}, limit=topk, with_payload=True, @@ -275,11 +275,17 @@ def rerank_in_process( language: str | None = None, under: str | None = None, model: TextEmbedding | None = None, + collection: str | None = None, ) -> List[Dict[str, Any]]: + eff_collection = ( + str(collection).strip() + if isinstance(collection, str) and collection.strip() + else (os.environ.get("COLLECTION_NAME") or "codebase") + ) client = QdrantClient(url=QDRANT_URL, api_key=API_KEY or None) _model = model or TextEmbedding(model_name=MODEL_NAME) dim = len(next(_model.embed(["dimension probe"]))) - vec_name = _select_dense_vector_name(client, COLLECTION, _model, dim) + vec_name = _select_dense_vector_name(client, eff_collection, _model, dim) must = [] if language: @@ -297,9 +303,9 @@ def rerank_in_process( ) flt = models.Filter(must=must) if must else None - pts = dense_results(client, _model, vec_name, query, flt, topk) + pts = dense_results(client, _model, vec_name, query, flt, topk, eff_collection) if not pts and flt is not None: - pts = dense_results(client, _model, vec_name, query, None, topk) + pts = dense_results(client, _model, vec_name, query, None, topk, eff_collection) if not pts: return [] @@ -330,13 +336,19 @@ def main(): ap.add_argument("--limit", type=int, default=12) ap.add_argument("--language", type=str, default=None) ap.add_argument("--under", type=str, default=None) + ap.add_argument("--collection", type=str, default=None) args = ap.parse_args() client = QdrantClient(url=QDRANT_URL, api_key=API_KEY or None) model = TextEmbedding(model_name=MODEL_NAME) dim = len(next(model.embed(["dimension probe"]))) - vec_name = _select_dense_vector_name(client, COLLECTION, model, dim) + eff_collection = ( + str(args.collection).strip() + if isinstance(args.collection, str) and args.collection.strip() + else (os.environ.get("COLLECTION_NAME") or "codebase") + ) + vec_name = _select_dense_vector_name(client, eff_collection, model, dim) must = [] if args.language: @@ -354,10 +366,10 @@ def main(): ) flt = models.Filter(must=must) if must else None - pts = dense_results(client, model, vec_name, args.query, flt, args.topk) + pts = dense_results(client, model, vec_name, args.query, flt, args.topk, eff_collection) # Fallback: if filtered search yields nothing, retry without filters to avoid empty rerank if not pts and flt is not None: - pts = dense_results(client, model, vec_name, args.query, None, args.topk) + pts = dense_results(client, model, vec_name, args.query, None, args.topk, eff_collection) if not pts: return pairs = prepare_pairs(args.query, pts) diff --git a/scripts/standalone_upload_client.py b/scripts/standalone_upload_client.py index 958d2ae1..a975d68a 100644 --- a/scripts/standalone_upload_client.py +++ b/scripts/standalone_upload_client.py @@ -28,6 +28,12 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry +try: + from upload_auth_utils import get_auth_session # type: ignore[import] +except ImportError: + def get_auth_session(upload_endpoint: str) -> str: + return "" + # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -656,6 +662,7 @@ def _get_temp_bundle_dir(self) -> Path: if not self.temp_dir: self.temp_dir = tempfile.mkdtemp(prefix="delta_bundle_") return Path(self.temp_dir) + # CLI is stateless - sequence tracking is handled by server def detect_file_changes(self, changed_paths: List[Path]) -> Dict[str, List]: @@ -981,7 +988,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, "workspace_path": self.workspace_path, "collection_name": self.collection_name, "created_at": created_at, - # CLI is stateless - server will assign sequence numbers + # CLI is stateless - server handles sequence numbers "sequence_number": None, # Server will assign "parent_sequence": None, # Server will determine "operations": { @@ -1031,8 +1038,7 @@ def create_delta_bundle(self, changes: Dict[str, List]) -> Tuple[str, Dict[str, return str(bundle_path), manifest def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, Any]: - """ - Upload delta bundle to remote server with exponential backoff retry. + """Upload delta bundle to remote server with exponential backoff retry. Args: bundle_path: Path to the bundle tarball @@ -1058,76 +1064,75 @@ def upload_bundle(self, bundle_path: str, manifest: Dict[str, Any]) -> Dict[str, # Check bundle size (server-side enforcement) bundle_size = os.path.getsize(bundle_path) - with open(bundle_path, 'rb') as bundle_file: - files = { - 'bundle': (f"{manifest['bundle_id']}.tar.gz", bundle_file, 'application/gzip') - } + files = { + "bundle": open(bundle_path, "rb"), + } + data = { + "workspace_path": self._translate_to_container_path(self.workspace_path), + "collection_name": self.collection_name, + "sequence_number": manifest.get("sequence_number"), + "force": False, + "source_path": self.workspace_path, + "logical_repo_id": _compute_logical_repo_id(self.workspace_path), + } - data = { - 'workspace_path': self._translate_to_container_path(self.workspace_path), - 'collection_name': self.collection_name, - # CLI is stateless - server handles sequence numbers - 'force': 'false', - 'source_path': self.workspace_path, - } + sess = get_auth_session(self.upload_endpoint) + if sess: + data["session"] = sess - if getattr(self, "logical_repo_id", None): - data['logical_repo_id'] = self.logical_repo_id + if getattr(self, "logical_repo_id", None): + data['logical_repo_id'] = self.logical_repo_id - logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") + logger.info(f"[remote_upload] Uploading bundle {manifest['bundle_id']} (size: {bundle_size} bytes)") - response = self.session.post( - f"{self.upload_endpoint}/api/v1/delta/upload", - files=files, - data=data, - timeout=(10, self.timeout) - ) + response = self.session.post( + f"{self.upload_endpoint}/api/v1/delta/upload", + files=files, + data=data, + timeout=(10, self.timeout) + ) - if response.status_code == 200: - result = response.json() - logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + result = None + try: + result = response.json() + except Exception: + result = None - seq = None + if response.status_code == 200 and isinstance(result, dict) and result.get("success", False): + logger.info(f"[remote_upload] Successfully uploaded bundle {manifest['bundle_id']}") + seq = result.get("sequence_number") + if seq is not None: try: - seq = result.get("sequence_number") + manifest["sequence"] = seq except Exception: - seq = None - if seq is not None: - try: - manifest["sequence"] = seq - except Exception: - pass - - poll_result = self._poll_after_timeout(manifest) - if poll_result.get("success"): - combined = dict(result) - for k, v in poll_result.items(): - if k in ("success", "error"): - continue - if k not in combined: - combined[k] = v - return combined - - logger.warning("[remote_upload] Upload accepted but polling did not confirm processing; returning original result") - return result - # Handle error - error_msg = f"Upload failed with status {response.status_code}" - try: - error_detail = response.json() - error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') - error_msg += f": {error_detail_msg}" - error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') - except: - error_msg += f": {response.text[:200]}" - error_code = "HTTP_ERROR" + pass + return result + + # Handle error + error_msg = f"Upload failed with status {response.status_code}" + try: + error_detail = result if isinstance(result, dict) else response.json() + error_detail_msg = error_detail.get('error', {}).get('message', 'Unknown error') + error_msg += f": {error_detail_msg}" + error_code = error_detail.get('error', {}).get('code', 'HTTP_ERROR') + except Exception: + error_msg += f": {response.text[:200]}" + error_code = "HTTP_ERROR" + + # Special-case 401 to make auth issues obvious to users + if response.status_code == 401: + if error_code in {None, "HTTP_ERROR"}: + error_code = "UNAUTHORIZED" + # Always append a clear hint for auth failures + error_msg += " (unauthorized; please log in with `ctxce auth login` and retry)" - last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} + last_error = {"success": False, "error": {"code": error_code, "message": error_msg, "status_code": response.status_code}} - # Don't retry on client errors (except 429) - if 400 <= response.status_code < 500 and response.status_code != 429: - return last_error + # Don't retry on client errors (except 429) + if 400 <= response.status_code < 500 and response.status_code != 429: + return last_error - logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") + logger.warning(f"[remote_upload] Upload attempt {attempt + 1} failed: {error_msg}") except requests.exceptions.ConnectTimeout as e: last_error = {"success": False, "error": {"code": "TIMEOUT_ERROR", "message": f"Upload timeout: {str(e)}"}} diff --git a/scripts/subprocess_manager.py b/scripts/subprocess_manager.py index 0bf89f3b..aaf2904f 100644 --- a/scripts/subprocess_manager.py +++ b/scripts/subprocess_manager.py @@ -9,6 +9,7 @@ from typing import Optional, Dict, Any, List, Union import logging import os +import contextlib logger = logging.getLogger(__name__) diff --git a/scripts/sync_env_to_k8s.py b/scripts/sync_env_to_k8s.py old mode 100644 new mode 100755 diff --git a/scripts/upload_auth_utils.py b/scripts/upload_auth_utils.py new file mode 100644 index 00000000..76b15e80 --- /dev/null +++ b/scripts/upload_auth_utils.py @@ -0,0 +1,47 @@ +import os +import json +import time +from typing import Any + + +def get_auth_session(upload_endpoint: str) -> str: + """Resolve auth session from environment or ~/.ctxce/auth.json. + + This mirrors the existing behavior used by the upload clients: + - Prefer CTXCE_UPLOAD_SESSION_ID / CTXCE_SESSION_ID from the environment. + - Fall back to ~/.ctxce/auth.json keyed by the upload endpoint (with and without + a trailing slash), honoring an optional numeric expiresAt/expires_at field. + - Treat expiresAt <= 0 or missing as non-expiring. + + Returns an empty string when no usable session is found. + """ + try: + sess = (os.environ.get("CTXCE_UPLOAD_SESSION_ID") or os.environ.get("CTXCE_SESSION_ID") or "").strip() + except Exception: + sess = "" + if sess: + return sess + + try: + home = os.path.expanduser("~") + cfg_path = os.path.join(home, ".ctxce", "auth.json") + if not os.path.exists(cfg_path): + return "" + with open(cfg_path, "r", encoding="utf-8") as f: + raw: Any = json.load(f) + if not isinstance(raw, dict): + return "" + key = upload_endpoint.rstrip("/") + entry = raw.get(key) or raw.get(upload_endpoint) + if not isinstance(entry, dict): + return "" + sid = entry.get("sessionId") or entry.get("session_id") + exp = entry.get("expiresAt") or entry.get("expires_at") + now_secs = int(time.time()) + if isinstance(exp, (int, float)) and exp > 0: + if exp >= now_secs: + return (sid or "").strip() + return "" + return (sid or "").strip() + except Exception: + return "" diff --git a/scripts/upload_delta_bundle.py b/scripts/upload_delta_bundle.py new file mode 100644 index 00000000..16415aac --- /dev/null +++ b/scripts/upload_delta_bundle.py @@ -0,0 +1,256 @@ +import os +import json +import tarfile +import hashlib +import logging +from pathlib import Path +from typing import Dict, Any + + +try: + from scripts.workspace_state import _extract_repo_name_from_path +except ImportError: + _extract_repo_name_from_path = None + + +logger = logging.getLogger(__name__) + +WORK_DIR = os.environ.get("WORK_DIR", "/work") + + +def get_workspace_key(workspace_path: str) -> str: + """Generate 16-char hash for collision avoidance in remote uploads. + + Remote uploads may have identical folder names from different users, + so uses longer hash than local indexing (8-chars) to ensure uniqueness. + + Both host paths (/home/user/project/repo) and container paths (/work/repo) + should generate the same key for the same repository. + """ + repo_name = Path(workspace_path).name + return hashlib.sha256(repo_name.encode("utf-8")).hexdigest()[:16] + + +def _cleanup_empty_dirs(path: Path, stop_at: Path) -> None: + """Recursively remove empty directories up to stop_at (exclusive).""" + try: + path = path.resolve() + stop_at = stop_at.resolve() + except Exception: + pass + while True: + try: + if path == stop_at or not path.exists() or not path.is_dir(): + break + if any(path.iterdir()): + break + path.rmdir() + path = path.parent + except Exception: + break + + +def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[str, Any]) -> Dict[str, int]: + """Process delta bundle and return operation counts.""" + operations_count = { + "created": 0, + "updated": 0, + "deleted": 0, + "moved": 0, + "skipped": 0, + "failed": 0, + } + + try: + # CRITICAL: Always materialize writes under WORK_DIR using a slugged repo directory. + # Do NOT write directly into the client-supplied workspace_path, since that may be a host + # path (e.g. /home/user/repo) that is not mounted/visible to the watcher/indexer. + if _extract_repo_name_from_path: + repo_name = _extract_repo_name_from_path(workspace_path) + if not repo_name: + repo_name = Path(workspace_path).name + else: + repo_name = Path(workspace_path).name + + # Workspace slug: -<16charhash>. This ensures uniqueness across users/workspaces + # that may share the same leaf folder name. + workspace_key = get_workspace_key(workspace_path) + workspace = Path(WORK_DIR) / f"{repo_name}-{workspace_key}" + workspace.mkdir(parents=True, exist_ok=True) + slug_repo_name = f"{repo_name}-{workspace_key}" + + workspace_root = workspace.resolve() + + def _safe_join(base: Path, rel: str) -> Path: + # SECURITY: Prevent path traversal / absolute-path writes by ensuring the resolved + # candidate path stays within the intended workspace root. + rp = Path(str(rel)) + if str(rp) in {".", ""}: + raise ValueError("Invalid operation path") + if rp.is_absolute(): + raise ValueError(f"Absolute paths are not allowed: {rel}") + base_resolved = base.resolve() + candidate = (base_resolved / rp).resolve() + try: + ok = candidate.is_relative_to(base_resolved) + except Exception: + ok = os.path.commonpath([str(base_resolved), str(candidate)]) == str(base_resolved) + if not ok: + raise ValueError(f"Path escapes workspace: {rel}") + return candidate + + with tarfile.open(bundle_path, "r:gz") as tar: + ops_member = None + for member in tar.getnames(): + if member.endswith("metadata/operations.json"): + ops_member = member + break + + if not ops_member: + raise ValueError("operations.json not found in bundle") + + ops_file = tar.extractfile(ops_member) + if not ops_file: + raise ValueError("Cannot extract operations.json") + + operations_data = json.loads(ops_file.read().decode("utf-8")) + operations = operations_data.get("operations", []) + + # Best-effort: extract git history metadata for watcher to ingest + try: + git_member = None + for member in tar.getnames(): + if member.endswith("metadata/git_history.json"): + git_member = member + break + if git_member: + git_file = tar.extractfile(git_member) + if git_file: + history_bytes = git_file.read() + history_dir = workspace / ".remote-git" + history_dir.mkdir(parents=True, exist_ok=True) + bundle_id = manifest.get("bundle_id") or "unknown" + history_path = history_dir / f"git_history_{bundle_id}.json" + try: + history_path.write_bytes(history_bytes) + except Exception as write_err: + logger.debug( + f"[upload_service] Failed to write git history manifest: {write_err}", + ) + except Exception as git_err: + logger.debug(f"[upload_service] Error extracting git history metadata: {git_err}") + + for operation in operations: + op_type = operation.get("operation") + rel_path = operation.get("path") + + if not rel_path: + operations_count["skipped"] += 1 + continue + + # Defensive guard: if the operation path already includes the slugged repo name + # ("-/..."), then writing it under workspace_root would create + # a nested slug directory ("slug/slug/..."), which is almost always client misuse. + if rel_path == slug_repo_name or rel_path.startswith(slug_repo_name + "/"): + msg = ( + f"[upload_service] Refusing to apply operation {op_type} for suspicious path {rel_path} " + f"which already contains workspace slug {slug_repo_name}" + ) + logger.error(msg) + raise ValueError(msg) + + target_path = _safe_join(workspace_root, rel_path) + + safe_source_path = None + source_rel_path = None + if op_type == "moved": + source_rel_path = operation.get("source_path") or operation.get("source_relative_path") + if source_rel_path: + safe_source_path = _safe_join(workspace_root, source_rel_path) + + try: + if op_type == "created": + file_member = None + for member in tar.getnames(): + if member.endswith(f"files/created/{rel_path}"): + file_member = member + break + + if file_member: + file_content = tar.extractfile(file_member) + if file_content: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(file_content.read()) + operations_count["created"] += 1 + else: + operations_count["failed"] += 1 + else: + operations_count["failed"] += 1 + + elif op_type == "updated": + file_member = None + for member in tar.getnames(): + if member.endswith(f"files/updated/{rel_path}"): + file_member = member + break + + if file_member: + file_content = tar.extractfile(file_member) + if file_content: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(file_content.read()) + operations_count["updated"] += 1 + else: + operations_count["failed"] += 1 + else: + operations_count["failed"] += 1 + + elif op_type == "moved": + file_member = None + for member in tar.getnames(): + if member.endswith(f"files/moved/{rel_path}"): + file_member = member + break + + if file_member: + file_content = tar.extractfile(file_member) + if file_content: + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(file_content.read()) + operations_count["moved"] += 1 + else: + operations_count["failed"] += 1 + else: + operations_count["failed"] += 1 + + if safe_source_path is not None and source_rel_path: + if safe_source_path.exists(): + try: + safe_source_path.unlink() + operations_count["deleted"] += 1 + _cleanup_empty_dirs(safe_source_path.parent, workspace) + except Exception as del_err: + logger.error( + f"Error deleting source file for move {source_rel_path}: {del_err}", + ) + + elif op_type == "deleted": + if target_path.exists(): + target_path.unlink() + _cleanup_empty_dirs(target_path.parent, workspace) + operations_count["deleted"] += 1 + else: + operations_count["skipped"] += 1 + + else: + operations_count["skipped"] += 1 + + except Exception as e: + logger.error(f"Error processing operation {op_type} for {rel_path}: {e}") + operations_count["failed"] += 1 + + return operations_count + + except Exception as e: + logger.error(f"Error processing delta bundle: {e}") + raise diff --git a/scripts/upload_service.py b/scripts/upload_service.py index 4807ef0f..0e179138 100644 --- a/scripts/upload_service.py +++ b/scripts/upload_service.py @@ -10,7 +10,6 @@ import json import tarfile import tempfile -import hashlib import asyncio import logging from pathlib import Path @@ -19,9 +18,47 @@ import uvicorn from fastapi import FastAPI, File, UploadFile, Form, HTTPException, Request, status -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, RedirectResponse from fastapi.middleware.cors import CORSMiddleware + +from scripts.upload_delta_bundle import get_workspace_key, process_delta_bundle + from pydantic import BaseModel, Field +from scripts.auth_backend import ( + AuthDisabledError, + AuthInvalidToken, + authenticate_user, + create_session, + create_session_for_token, + create_user, + has_any_users, + has_collection_access, + validate_session, + AUTH_ENABLED, + AUTH_SESSION_TTL_SECONDS, + is_admin_user, + list_users, + list_collections, + list_collection_acl, + grant_collection_access, + revoke_collection_access, +) +try: + from scripts.admin_ui import ( + render_admin_acl, + render_admin_bootstrap, + render_admin_error, + render_admin_login, + ) +except Exception: + + def _admin_ui_unavailable(*args, **kwargs): + raise HTTPException(status_code=500, detail="Admin UI unavailable") + + render_admin_acl = _admin_ui_unavailable + render_admin_bootstrap = _admin_ui_unavailable + render_admin_error = _admin_ui_unavailable + render_admin_login = _admin_ui_unavailable # Import existing workspace state and indexing functions try: @@ -66,6 +103,10 @@ def logical_repo_reuse_enabled() -> bool: # type: ignore[no-redef] WORK_DIR = os.environ.get("WORK_DIR", "/work") MAX_BUNDLE_SIZE_MB = int(os.environ.get("MAX_BUNDLE_SIZE_MB", "100")) UPLOAD_TIMEOUT_SECS = int(os.environ.get("UPLOAD_TIMEOUT_SECS", "300")) +CTXCE_MCP_ACL_ENFORCE = ( + str(os.environ.get("CTXCE_MCP_ACL_ENFORCE", "0")).strip().lower() + in {"1", "true", "yes", "on"} +) # FastAPI app app = FastAPI( @@ -111,17 +152,108 @@ class HealthResponse(BaseModel): qdrant_url: str work_dir: str -def get_workspace_key(workspace_path: str) -> str: - """Generate 16-char hash for collision avoidance in remote uploads. - Remote uploads may have identical folder names from different users, - so uses longer hash than local indexing (8-chars) to ensure uniqueness. +class AuthLoginRequest(BaseModel): + client: str + workspace: Optional[str] = None + token: Optional[str] = None + + +class AuthLoginResponse(BaseModel): + session_id: str + user_id: Optional[str] = None + expires_at: Optional[int] = None + + +class AuthStatusResponse(BaseModel): + enabled: bool + has_users: Optional[bool] = None + session_ttl_seconds: int - Both host paths (/home/user/project/repo) and container paths (/work/repo) - should generate the same key for the same repository. - """ - repo_name = Path(workspace_path).name - return hashlib.sha256(repo_name.encode('utf-8')).hexdigest()[:16] + +class AuthUserCreateRequest(BaseModel): + username: str + password: str + + +class AuthUserCreateResponse(BaseModel): + user_id: str + username: str + + +class PasswordLoginRequest(BaseModel): + username: str + password: str + workspace: Optional[str] = None + + +ADMIN_SESSION_COOKIE_NAME = "ctxce_session" + + +def _get_session_candidate_from_request(request: Request) -> Dict[str, Any]: + sid = (request.cookies.get(ADMIN_SESSION_COOKIE_NAME) or "").strip() + if sid: + return {"session_id": sid, "source": "cookie"} + try: + qp = request.query_params + sid = (qp.get("session") or qp.get("session_id") or qp.get("sessionId") or "").strip() + except Exception: + sid = "" + if sid: + return {"session_id": sid, "source": "query"} + sid = ( + (request.headers.get("X-Session-Id") or "").strip() + or (request.headers.get("X-Auth-Session") or "").strip() + ) + if sid: + return {"session_id": sid, "source": "header"} + return {"session_id": "", "source": ""} + + +def _set_admin_session_cookie(resp: Any, session_id: str) -> Any: + sid = (session_id or "").strip() + if not sid: + return resp + try: + kwargs: Dict[str, Any] = { + "key": ADMIN_SESSION_COOKIE_NAME, + "value": sid, + "httponly": True, + "samesite": "lax", + "path": "/", + } + ttl = int(AUTH_SESSION_TTL_SECONDS or 0) + if ttl > 0: + kwargs["max_age"] = ttl + resp.set_cookie(**kwargs) + except Exception: + pass + return resp + + +def _get_valid_session_record(request: Request) -> Optional[Dict[str, Any]]: + sid = (_get_session_candidate_from_request(request).get("session_id") or "").strip() + if not sid: + return None + try: + return validate_session(sid) + except AuthDisabledError: + return None + except Exception as e: + logger.error(f"[upload_service] Failed to validate session cookie: {e}") + raise HTTPException(status_code=500, detail="Failed to validate auth session") + + +def _require_admin_session(request: Request) -> Dict[str, Any]: + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + record = _get_valid_session_record(request) + if record is None: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Not authenticated") + user_id = str(record.get("user_id") or "").strip() + if not user_id or not is_admin_user(user_id): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin required") + return record def get_next_sequence(workspace_path: str) -> int: """Get next sequence number for workspace.""" @@ -175,197 +307,6 @@ def validate_bundle_format(bundle_path: Path) -> Dict[str, Any]: except Exception as e: raise ValueError(f"Invalid bundle format: {str(e)}") -def _cleanup_empty_dirs(path: Path, stop_at: Path) -> None: - """Recursively remove empty directories up to stop_at (exclusive).""" - try: - path = path.resolve() - stop_at = stop_at.resolve() - except Exception: - pass - while True: - try: - if path == stop_at or not path.exists() or not path.is_dir(): - break - if any(path.iterdir()): - break - path.rmdir() - path = path.parent - except Exception: - break - - -def process_delta_bundle(workspace_path: str, bundle_path: Path, manifest: Dict[str, Any]) -> Dict[str, int]: - """Process delta bundle and return operation counts.""" - operations_count = { - "created": 0, - "updated": 0, - "deleted": 0, - "moved": 0, - "skipped": 0, - "failed": 0 - } - - try: - # CRITICAL FIX: Extract repo name and create workspace under WORK_DIR - # Previous bug: used source workspace_path directly, extracting files outside /work - # This caused watcher service to never see uploaded files - if _extract_repo_name_from_path: - repo_name = _extract_repo_name_from_path(workspace_path) - # Fallback to directory name if repo detection fails - if not repo_name: - repo_name = Path(workspace_path).name - else: - # Fallback: use directory name - repo_name = Path(workspace_path).name - - # Generate workspace under WORK_DIR using repo name hash - workspace_key = get_workspace_key(workspace_path) - workspace = Path(WORK_DIR) / f"{repo_name}-{workspace_key}" - workspace.mkdir(parents=True, exist_ok=True) - - with tarfile.open(bundle_path, "r:gz") as tar: - # Extract operations metadata - ops_member = None - for member in tar.getnames(): - if member.endswith("metadata/operations.json"): - ops_member = member - break - - if not ops_member: - raise ValueError("operations.json not found in bundle") - - ops_file = tar.extractfile(ops_member) - if not ops_file: - raise ValueError("Cannot extract operations.json") - - operations_data = json.loads(ops_file.read().decode('utf-8')) - operations = operations_data.get("operations", []) - - # Best-effort: extract git history metadata for watcher to ingest - try: - git_member = None - for member in tar.getnames(): - if member.endswith("metadata/git_history.json"): - git_member = member - break - if git_member: - git_file = tar.extractfile(git_member) - if git_file: - history_bytes = git_file.read() - history_dir = workspace / ".remote-git" - history_dir.mkdir(parents=True, exist_ok=True) - bundle_id = manifest.get("bundle_id") or "unknown" - history_path = history_dir / f"git_history_{bundle_id}.json" - try: - history_path.write_bytes(history_bytes) - except Exception as write_err: - logger.debug(f"[upload_service] Failed to write git history manifest: {write_err}") - except Exception as git_err: - logger.debug(f"[upload_service] Error extracting git history metadata: {git_err}") - - # Process each operation - for operation in operations: - op_type = operation.get("operation") - rel_path = operation.get("path") - - if not rel_path: - operations_count["skipped"] += 1 - continue - - target_path = workspace / rel_path - - try: - if op_type == "created": - # Extract file from bundle - file_member = None - for member in tar.getnames(): - if member.endswith(f"files/created/{rel_path}"): - file_member = member - break - - if file_member: - file_content = tar.extractfile(file_member) - if file_content: - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_bytes(file_content.read()) - operations_count["created"] += 1 - else: - operations_count["failed"] += 1 - else: - operations_count["failed"] += 1 - - elif op_type == "updated": - # Extract updated file - file_member = None - for member in tar.getnames(): - if member.endswith(f"files/updated/{rel_path}"): - file_member = member - break - - if file_member: - file_content = tar.extractfile(file_member) - if file_content: - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_bytes(file_content.read()) - operations_count["updated"] += 1 - else: - operations_count["failed"] += 1 - else: - operations_count["failed"] += 1 - - elif op_type == "moved": - # Extract moved file to destination - file_member = None - for member in tar.getnames(): - if member.endswith(f"files/moved/{rel_path}"): - file_member = member - break - - if file_member: - file_content = tar.extractfile(file_member) - if file_content: - target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_bytes(file_content.read()) - operations_count["moved"] += 1 - else: - operations_count["failed"] += 1 - else: - operations_count["failed"] += 1 - - # Remove original source file if provided - source_rel_path = operation.get("source_path") or operation.get("source_relative_path") - if source_rel_path: - source_path = workspace / source_rel_path - if source_path.exists(): - try: - source_path.unlink() - operations_count["deleted"] += 1 - _cleanup_empty_dirs(source_path.parent, workspace) - except Exception as del_err: - logger.error(f"Error deleting source file for move {source_rel_path}: {del_err}") - - elif op_type == "deleted": - # Delete file - if target_path.exists(): - target_path.unlink() - _cleanup_empty_dirs(target_path.parent, workspace) - operations_count["deleted"] += 1 - else: - operations_count["skipped"] += 1 - - else: - operations_count["skipped"] += 1 - - except Exception as e: - logger.error(f"Error processing operation {op_type} for {rel_path}: {e}") - operations_count["failed"] += 1 - - return operations_count - - except Exception as e: - logger.error(f"Error processing delta bundle: {e}") - raise - async def _process_bundle_background( workspace_path: str, @@ -410,6 +351,331 @@ async def _process_bundle_background( pass +@app.get("/auth/status", response_model=AuthStatusResponse) +async def auth_status(): + try: + if not AUTH_ENABLED: + return AuthStatusResponse( + enabled=False, + has_users=None, + session_ttl_seconds=AUTH_SESSION_TTL_SECONDS, + ) + try: + users_exist = has_any_users() + except AuthDisabledError: + return AuthStatusResponse( + enabled=False, + has_users=None, + session_ttl_seconds=AUTH_SESSION_TTL_SECONDS, + ) + return AuthStatusResponse( + enabled=True, + has_users=users_exist, + session_ttl_seconds=AUTH_SESSION_TTL_SECONDS, + ) + except Exception as e: + logger.error(f"[upload_service] Failed to report auth status: {e}") + raise HTTPException(status_code=500, detail="Failed to read auth status") + + +@app.post("/auth/login", response_model=AuthLoginResponse) +async def auth_login(payload: AuthLoginRequest): + try: + session = create_session_for_token( + client=payload.client, + workspace=payload.workspace, + token=payload.token, + ) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except AuthInvalidToken: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid auth token") + except Exception as e: + logger.error(f"[upload_service] Failed to create auth session: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + return AuthLoginResponse( + session_id=session.get("session_id"), + user_id=session.get("user_id"), + expires_at=session.get("expires_at"), + ) + + +@app.get("/admin") +async def admin_root(request: Request): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + users_exist = has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to inspect user state for admin UI: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + + if not users_exist: + return RedirectResponse(url="/admin/bootstrap", status_code=302) + + candidate = _get_session_candidate_from_request(request) + record = _get_valid_session_record(request) + if record is None: + return RedirectResponse(url="/admin/login", status_code=302) + + user_id = str(record.get("user_id") or "").strip() + if user_id and is_admin_user(user_id): + resp = RedirectResponse(url="/admin/acl", status_code=302) + if candidate.get("source") and candidate.get("source") != "cookie": + _set_admin_session_cookie(resp, str(candidate.get("session_id") or "")) + return resp + return RedirectResponse(url="/admin/login", status_code=302) + + +@app.get("/admin/bootstrap") +async def admin_bootstrap_form(request: Request): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + users_exist = has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to inspect user state for bootstrap: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + if users_exist: + return RedirectResponse(url="/admin/login", status_code=302) + return render_admin_bootstrap(request) + + +@app.post("/admin/bootstrap") +async def admin_bootstrap_submit( + request: Request, + username: str = Form(...), + password: str = Form(...), +): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + users_exist = has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to inspect user state for bootstrap submit: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + if users_exist: + return RedirectResponse(url="/admin/login", status_code=302) + + try: + user = create_user(username, password) + except Exception as e: + return render_admin_error( + request=request, + title="Bootstrap Failed", + message=str(e), + back_href="/admin/bootstrap", + status_code=400, + ) + + try: + session = create_session(user_id=user.get("user_id"), metadata={"client": "admin_ui"}) + except Exception as e: + logger.error(f"[upload_service] Failed to create session after bootstrap: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + + resp = RedirectResponse(url="/admin/acl", status_code=302) + _set_admin_session_cookie(resp, str(session.get("session_id") or "")) + return resp + + +@app.get("/admin/login") +async def admin_login_form(request: Request): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + return render_admin_login(request) + + +@app.post("/admin/login") +async def admin_login_submit( + request: Request, + username: str = Form(...), + password: str = Form(...), +): + if not AUTH_ENABLED: + raise HTTPException(status_code=404, detail="Auth disabled") + try: + user = authenticate_user(username, password) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Error authenticating user for admin UI: {e}") + raise HTTPException(status_code=500, detail="Authentication error") + + if not user: + return render_admin_login( + request=request, + error="Invalid credentials", + status_code=401, + ) + + try: + session = create_session(user_id=user.get("id"), metadata={"client": "admin_ui"}) + except Exception as e: + logger.error(f"[upload_service] Failed to create session for admin UI: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + + resp = RedirectResponse(url="/admin/acl", status_code=302) + _set_admin_session_cookie(resp, str(session.get("session_id") or "")) + return resp + + +@app.post("/admin/logout") +async def admin_logout(): + resp = RedirectResponse(url="/admin/login", status_code=302) + resp.delete_cookie(key=ADMIN_SESSION_COOKIE_NAME, path="/") + return resp + + +@app.get("/admin/acl") +async def admin_acl_page(request: Request): + _require_admin_session(request) + try: + users = list_users() + collections = list_collections(include_deleted=False) + grants = list_collection_acl() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to load admin UI data: {e}") + raise HTTPException(status_code=500, detail="Failed to load admin data") + + resp = render_admin_acl(request, users=users, collections=collections, grants=grants) + candidate = _get_session_candidate_from_request(request) + if candidate.get("source") and candidate.get("source") != "cookie": + _set_admin_session_cookie(resp, str(candidate.get("session_id") or "")) + return resp + + +@app.post("/admin/acl/grant") +async def admin_acl_grant( + request: Request, + user_id: str = Form(...), + collection: str = Form(...), + permission: str = Form("read"), +): + _require_admin_session(request) + try: + grant_collection_access(user_id=user_id, qdrant_collection=collection, permission=permission) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + return render_admin_error(request, title="Grant Failed", message=str(e), back_href="/admin/acl") + return RedirectResponse(url="/admin/acl", status_code=302) + + +@app.post("/admin/users") +async def admin_create_user( + request: Request, + username: str = Form(...), + password: str = Form(...), + role: str = Form("user"), +): + _require_admin_session(request) + try: + create_user(username, password, role=role) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + return render_admin_error(request, title="Create User Failed", message=str(e), back_href="/admin/acl") + return RedirectResponse(url="/admin/acl", status_code=302) + + +@app.post("/admin/acl/revoke") +async def admin_acl_revoke( + request: Request, + user_id: str = Form(...), + collection: str = Form(...), +): + _require_admin_session(request) + try: + revoke_collection_access(user_id=user_id, qdrant_collection=collection) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + return render_admin_error(request, title="Revoke Failed", message=str(e), back_href="/admin/acl") + return RedirectResponse(url="/admin/acl", status_code=302) + + +@app.post("/auth/users", response_model=AuthUserCreateResponse) +async def auth_create_user(payload: AuthUserCreateRequest, request: Request): + try: + first_user = not has_any_users() + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to check user state: {e}") + raise HTTPException(status_code=500, detail="Failed to inspect user state") + + admin_token = os.environ.get("CTXCE_AUTH_ADMIN_TOKEN") or os.environ.get("CTXCE_AUTH_SHARED_TOKEN") + if not first_user: + if not admin_token: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin token not configured", + ) + header = request.headers.get("X-Admin-Token") + if not header or header != admin_token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid admin token", + ) + + try: + user = create_user(payload.username, payload.password) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to create user: {e}") + msg = str(e) + if "UNIQUE" in msg or "unique" in msg: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Username already exists", + ) + raise HTTPException(status_code=500, detail="Failed to create user") + + return AuthUserCreateResponse(user_id=user.get("user_id"), username=user.get("username")) + + +@app.post("/auth/login/password", response_model=AuthLoginResponse) +async def auth_login_password(payload: PasswordLoginRequest): + try: + user = authenticate_user(payload.username, payload.password) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Error authenticating user: {e}") + raise HTTPException(status_code=500, detail="Authentication error") + + if not user: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials") + + meta: Optional[Dict[str, Any]] = None + if payload.workspace: + meta = {"workspace": payload.workspace} + + try: + session = create_session(user_id=user.get("id"), metadata=meta) + except AuthDisabledError: + raise HTTPException(status_code=404, detail="Auth disabled") + except Exception as e: + logger.error(f"[upload_service] Failed to create session for user: {e}") + raise HTTPException(status_code=500, detail="Failed to create auth session") + + return AuthLoginResponse( + session_id=session.get("session_id"), + user_id=session.get("user_id"), + expires_at=session.get("expires_at"), + ) + + @app.get("/health", response_model=HealthResponse) async def health_check(): """Health check endpoint.""" @@ -465,13 +731,35 @@ async def upload_delta_bundle( force: Optional[bool] = Form(False), source_path: Optional[str] = Form(None), logical_repo_id: Optional[str] = Form(None), + session: Optional[str] = Form(None), ): """Upload and process delta bundle.""" start_time = datetime.now() client_host = request.client.host if hasattr(request, 'client') and request.client else 'unknown' + record: Optional[Dict[str, Any]] = None + try: logger.info(f"[upload_service] Begin processing upload for workspace={workspace_path} from {client_host}") + + if AUTH_ENABLED: + session_value = (session or "").strip() + try: + record = validate_session(session_value) + except AuthDisabledError: + record = None + except Exception as e: + logger.error(f"[upload_service] Failed to validate auth session for upload: {e}") + raise HTTPException( + status_code=500, + detail="Failed to validate auth session", + ) + if record is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired session", + ) + # Validate workspace path workspace = Path(workspace_path) if not workspace.is_absolute(): @@ -534,6 +822,28 @@ async def upload_delta_bundle( else: collection_name = DEFAULT_COLLECTION + # Enforce collection write access for uploads when auth is enabled. + # Semantics: "write" is sufficient for uploading/indexing content. + if AUTH_ENABLED and CTXCE_MCP_ACL_ENFORCE: + uid = str((record or {}).get("user_id") or "").strip() + if not uid: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired session", + ) + try: + allowed = has_collection_access(uid, str(collection_name), "write") + except AuthDisabledError: + allowed = True + except Exception as e: + logger.error(f"[upload_service] Failed to check collection access for upload: {e}") + raise HTTPException(status_code=500, detail="Failed to check collection access") + if not allowed: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Forbidden: write access to collection '{collection_name}' denied", + ) + # Persist origin metadata for remote lookups (including client source_path) # Use slugged repo name (repo+16) for state so it matches ingest/watch_index usage try: diff --git a/scripts/workspace_state.py b/scripts/workspace_state.py index da7d6b73..d31bafb3 100644 --- a/scripts/workspace_state.py +++ b/scripts/workspace_state.py @@ -626,7 +626,6 @@ def _detect_repo_name_from_path(path: Path) -> str: os.environ.get("WORKSPACE_PATH"), "/work", os.environ.get("HOST_ROOT"), - "/home/coder/project/Context-Engine/dev-workspace", ): if not root_str: continue diff --git a/templates/admin/acl.html b/templates/admin/acl.html new file mode 100644 index 00000000..e3c752f7 --- /dev/null +++ b/templates/admin/acl.html @@ -0,0 +1,110 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

Users

+ + + + {% if users and users|length > 0 %} + {% for u in users %} + + + + + + {% endfor %} + {% else %} + + {% endif %} +
idusernamerole
{{ u.id }}{{ u.username }}{{ u.role }}
(none)
+ +

Create User

+
+

+

+ +

+ +
+
+ +
+

Collections

+ + + + {% if collections and collections|length > 0 %} + {% for c in collections %} + + + + + {% endfor %} + {% else %} + + {% endif %} +
idqdrant_collection
{{ c.id }}{{ c.qdrant_collection }}
(none)
+
+ +
+

Grants

+ + + + {% if grants and grants|length > 0 %} + {% for g in grants %} + + + + + + + + {% endfor %} + {% else %} + + {% endif %} +
collectionusernameuser_idpermission
{{ g.qdrant_collection }}{{ g.username }}{{ g.user_id }}{{ g.permission }} +
+ + + +
+
(none)
+ +

Grant Collection Access

+
+

+ +

+ +

+ +
+ +

Collection permissions are enforced by MCP servers when CTXCE_MCP_ACL_ENFORCE=1 (and CTXCE_ACL_ALLOW_ALL=0).

+
+{% endblock %} diff --git a/templates/admin/base.html b/templates/admin/base.html new file mode 100644 index 00000000..f3aa77d9 --- /dev/null +++ b/templates/admin/base.html @@ -0,0 +1,39 @@ + + + + + + {{ title }} + + + +
+

{{ title }}

+ +
+ + {% block content %}{% endblock %} + +

CTXCE Admin UI

+ + diff --git a/templates/admin/bootstrap.html b/templates/admin/bootstrap.html new file mode 100644 index 00000000..34b05382 --- /dev/null +++ b/templates/admin/bootstrap.html @@ -0,0 +1,27 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

Bootstrap Admin User

+ + {% if error %} +

{{ error }}

+ {% endif %} + +
+ +

+ +

+ +
+ +

Only available when no users exist yet.

+
+{% endblock %} diff --git a/templates/admin/error.html b/templates/admin/error.html new file mode 100644 index 00000000..850f623b --- /dev/null +++ b/templates/admin/error.html @@ -0,0 +1,9 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

{{ title }}

+

{{ message }}

+

Back

+
+{% endblock %} diff --git a/templates/admin/login.html b/templates/admin/login.html new file mode 100644 index 00000000..1b94be36 --- /dev/null +++ b/templates/admin/login.html @@ -0,0 +1,25 @@ +{% extends "admin/base.html" %} + +{% block content %} +
+

Login

+ + {% if error %} +

{{ error }}

+ {% endif %} + +
+ +

+ +

+ +
+
+{% endblock %} diff --git a/tests/test_reranker_verification.py b/tests/test_reranker_verification.py index 2642c65f..e7a05245 100644 --- a/tests/test_reranker_verification.py +++ b/tests/test_reranker_verification.py @@ -108,6 +108,45 @@ def fake_rerank_local(pairs): assert [r["path"] for r in rr["results"]] == ["/work/b.py", "/work/a.py"] +@pytest.mark.service +@pytest.mark.anyio +async def test_rerank_inproc_dense_respects_collection_argument(monkeypatch): + # Drive the in-process dense rerank fallback path by returning no hybrid candidates. + monkeypatch.setenv("HYBRID_IN_PROCESS", "1") + monkeypatch.setenv("RERANK_IN_PROCESS", "1") + + def fake_run_hybrid_search(**kwargs): + return [] + + monkeypatch.setitem(sys.modules, "scripts.hybrid_search", _make_hybrid_stub(fake_run_hybrid_search)) + monkeypatch.delitem(sys.modules, "scripts.mcp_indexer_server", raising=False) + server = importlib.import_module("scripts.mcp_indexer_server") + monkeypatch.setattr(server, "_get_embedding_model", _fake_embedding_model) + + captured = {} + + def fake_rerank_in_process(**kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr( + importlib.import_module("scripts.rerank_local"), + "rerank_in_process", + fake_rerank_in_process, + ) + + await server.repo_search( + query="q", + limit=2, + per_path=2, + rerank_enabled=True, + compact=True, + collection="other-collection", + ) + + assert captured.get("collection") == "other-collection" + + @pytest.mark.service @pytest.mark.anyio async def test_rerank_subprocess_timeout_fallback(monkeypatch): @@ -122,6 +161,10 @@ def fake_run_hybrid_search(**kwargs): ] async def fake_run_async(cmd, env=None, timeout=None): + # Ensure explicit collection is forwarded to subprocess reranker + assert "--collection" in cmd + idx = cmd.index("--collection") + assert cmd[idx + 1] == "test-coll" # Simulate subprocess reranker timing out return {"ok": False, "code": -1, "stdout": "", "stderr": f"Command timed out after {timeout}s"} @@ -137,7 +180,14 @@ async def fake_run_async(cmd, env=None, timeout=None): monkeypatch.setattr(server, "_get_embedding_model", _fake_embedding_model) monkeypatch.setattr(server, "_run_async", fake_run_async) - rr = await server.repo_search(query="q", limit=2, per_path=2, rerank_enabled=True, compact=True) + rr = await server.repo_search( + query="q", + limit=2, + per_path=2, + rerank_enabled=True, + compact=True, + collection="test-coll", + ) # Fallback should keep original order from hybrid; timeout counter incremented assert rr.get("used_rerank") is False assert rr.get("rerank_counters", {}).get("timeout", 0) >= 1 diff --git a/tests/test_upload_service_path_traversal.py b/tests/test_upload_service_path_traversal.py new file mode 100644 index 00000000..2c9ba889 --- /dev/null +++ b/tests/test_upload_service_path_traversal.py @@ -0,0 +1,84 @@ +import io +import json +import tarfile +from pathlib import Path + +import pytest + + +def _write_bundle(tmp_path: Path, operations: list[dict]) -> Path: + bundle_path = tmp_path / "bundle.tar.gz" + payload = json.dumps({"operations": operations}).encode("utf-8") + + with tarfile.open(bundle_path, "w:gz") as tar: + info = tarfile.TarInfo(name="metadata/operations.json") + info.size = len(payload) + tar.addfile(info, io.BytesIO(payload)) + + return bundle_path + + +def test_process_delta_bundle_rejects_traversal_created(tmp_path, monkeypatch): + import scripts.upload_delta_bundle as us + + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(us, "WORK_DIR", str(work_dir)) + + bundle = _write_bundle( + tmp_path, + [{"operation": "created", "path": "../../evil.txt"}], + ) + + with pytest.raises(ValueError, match="escapes workspace"): + us.process_delta_bundle( + workspace_path="/home/user/repo", + bundle_path=bundle, + manifest={"bundle_id": "b1"}, + ) + + +def test_process_delta_bundle_rejects_absolute_paths(tmp_path, monkeypatch): + import scripts.upload_delta_bundle as us + + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(us, "WORK_DIR", str(work_dir)) + + bundle = _write_bundle( + tmp_path, + [{"operation": "created", "path": "/etc/passwd"}], + ) + + with pytest.raises(ValueError, match="Absolute paths"): + us.process_delta_bundle( + workspace_path="/home/user/repo", + bundle_path=bundle, + manifest={"bundle_id": "b1"}, + ) + + +def test_process_delta_bundle_rejects_traversal_moved_source(tmp_path, monkeypatch): + import scripts.upload_delta_bundle as us + + work_dir = tmp_path / "work" + work_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(us, "WORK_DIR", str(work_dir)) + + bundle = _write_bundle( + tmp_path, + [ + { + "operation": "moved", + "path": "dst.txt", + "source_path": "../../escape.txt", + } + ], + ) + + with pytest.raises(ValueError, match="escapes workspace"): + us.process_delta_bundle( + workspace_path="/home/user/repo", + bundle_path=bundle, + manifest={"bundle_id": "b1"}, + ) diff --git a/vscode-extension/build/build.sh b/vscode-extension/build/build.sh index 9bb2519b..856ea157 100755 --- a/vscode-extension/build/build.sh +++ b/vscode-extension/build/build.sh @@ -14,6 +14,7 @@ CTX_SRC="$SCRIPT_DIR/../../scripts/ctx.py" ROUTER_SRC="$SCRIPT_DIR/../../scripts/mcp_router.py" REFRAG_SRC="$SCRIPT_DIR/../../scripts/refrag_glm.py" ENV_EXAMPLE_SRC="$SCRIPT_DIR/../../.env.example" +AUTH_SRC="$SCRIPT_DIR/../../scripts/upload_auth_utils.py" cleanup() { rm -rf "$STAGE_DIR" @@ -54,6 +55,11 @@ if [[ -f "$REFRAG_SRC" ]]; then cp "$REFRAG_SRC" "$STAGE_DIR/refrag_glm.py" fi +# Bundle auth helper used by standalone_upload_client.py +if [[ -f "$AUTH_SRC" ]]; then + cp "$AUTH_SRC" "$STAGE_DIR/upload_auth_utils.py" +fi + if [[ -f "$ENV_EXAMPLE_SRC" ]]; then cp "$ENV_EXAMPLE_SRC" "$STAGE_DIR/env.example" fi @@ -64,9 +70,9 @@ if [[ "$BUNDLE_DEPS" == "--bundle-deps" ]]; then # On macOS, urllib3 v2 + system LibreSSL emits NotOpenSSLWarning; pin <2 there. if [[ "$(uname -s)" == "Darwin" ]]; then echo "Detected macOS; pinning urllib3<2 to avoid LibreSSL/OpenSSL warning." - "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" "urllib3<2" requests charset_normalizer + "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" "urllib3<2" requests charset_normalizer "openai>=1.0" else - "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" requests urllib3 charset_normalizer + "$PYTHON_BIN" -m pip install -t "$STAGE_DIR/python_libs" requests urllib3 charset_normalizer "openai>=1.0" fi fi diff --git a/vscode-extension/context-engine-uploader/README.md b/vscode-extension/context-engine-uploader/README.md index 50df124a..75494645 100644 --- a/vscode-extension/context-engine-uploader/README.md +++ b/vscode-extension/context-engine-uploader/README.md @@ -1,6 +1,10 @@ Context Engine Uploader ======================= +Install +------- +- Install from the VS Code Marketplace (search for "Context Engine Uploader" or publisher `context-engine`). You can also install it directly from the Extensions view in VS Code. + Features -------- - Runs a force sync (`Index Codebase`) followed by watch mode to keep a remote Context Engine instance in sync with your workspace. @@ -21,9 +25,16 @@ Configuration - **Prompt+ decoder:** set `Context Engine Uploader: Decoder Url` (default `http://localhost:8081`, auto-appends `/completion`) to point at your local llama.cpp decoder. For Ollama, set it to `http://localhost:11434/api/chat`. Turn on `Use Gpu Decoder` to set `USE_GPU_DECODER=1` so ctx.py prefers the GPU llama.cpp sidecar. Prompt+ automatically runs the bundled `scripts/ctx.py` when an embedded copy is available, falling back to the workspace version if not. - **Claude/Windsurf MCP config:** - `MCP Indexer Url` and `MCP Memory Url` control the URLs written into the project-local `.mcp.json` (Claude) and Windsurf `mcp_config.json` when you run the `Write MCP Config` command. These URLs are used **literally** (e.g. `http://localhost:8001/sse` or `http://localhost:8003/mcp`). - - `MCP Transport Mode` (`contextEngineUploader.mcpTransportMode`) chooses how those URLs are wrapped: - - `sse-remote` (default): emit stdio configs that call `npx mcp-remote --transport sse-only`. - - `http`: emit direct HTTP MCP entries of the form `{ "type": "http", "url": "" }` for Claude/Windsurf. Use this when pointing at HTTP `/mcp` endpoints exposed by the Context-Engine MCP services. + - `MCP Server Mode` (`contextEngineUploader.mcpServerMode`) controls *what* servers are written: + - `bridge`: write a single `context-engine` server that talks to the `ctxce` MCP bridge. + - `direct`: write two servers, `qdrant-indexer` and `memory`, that talk directly to the configured URLs. + - `MCP Transport Mode` (`contextEngineUploader.mcpTransportMode`) controls *how* those servers talk: + - `sse-remote` (default): use stdio MCP processes behind an SSE tunnel (`bridge-stdio` / `direct-sse`). + - `http`: use HTTP MCP endpoints directly (`bridge-http` / `direct-http`). + - Common combinations: + - `bridge` + `sse-remote` → **bridge-stdio**: write a single `context-engine` stdio server that runs `ctxce mcp-serve` behind an SSE tunnel. + - `bridge` + `http` → **bridge-http**: write a single `context-engine` HTTP server that points at the local `ctxce mcp-http-serve` URL (e.g. `http://127.0.0.1:30810/mcp`). + - `direct` + `http` → **direct-http**: write separate HTTP servers for the indexer and memory MCP backends. - **MCP config on startup:** - `contextEngineUploader.autoWriteMcpConfigOnStartup` (default `false`) controls whether the extension automatically runs the same logic as `Write MCP Config` on activation. When enabled, it refreshes `.mcp.json`, Windsurf `mcp_config.json`, and the Claude hook (`.claude/settings.local.json`) to match your current settings and the installed extension version. If `scaffoldCtxConfig` is also `true`, this startup path will additionally scaffold/update `ctx_config.json` and `.env` as described below. - **CTX + GLM settings:** @@ -37,6 +48,110 @@ Configuration - The scaffolder also enforces CTX defaults (e.g., `MULTI_REPO_MODE=1`, `REFRAG_RUNTIME=glm`, `REFRAG_DECODER=1`) so the embedded `ctx.py` is ready for remote uploads, regardless of the “Use GLM Decoder” toggle. - `contextEngineUploader.surfaceQdrantCollectionHint` gates whether the Claude hook adds a hint line with the Qdrant collection ID when ctx is enhancing prompts. This setting is also respected when the extension writes `.claude/settings.local.json`. +MCP bridge (ctx-mcp-bridge) & MCP config lifecycle +--------------------------------------------------- +- The MCP bridge (`@context-engine-bridge/context-engine-mcp-bridge`, CLI `ctxce`) is a small local MCP server that fans out to two upstream MCP services: the Qdrant indexer and the memory/search backend. The VS Code extension can drive it in two ways: + - **Bridge stdio (`bridge-stdio`)** – a stdio MCP server (`ctxce mcp-serve`) wrapped behind an SSE tunnel. + - **Bridge HTTP (`bridge-http`)** – an HTTP MCP server (`ctxce mcp-http-serve`) listening on `http://127.0.0.1:/mcp`. +- Why use the bridge instead of two direct MCP entries? + - **Single server entry:** IDEs only need to register one MCP server (`context-engine`) instead of juggling separate `qdrant-indexer` and `memory` entries, avoiding coordination mistakes. + - **Shared session defaults:** the bridge loads `ctx_config.json` and injects collection name, repo metadata, and any other ctx defaults so every IDE window talks to the right collection without hand-editing `.mcp.json`. + - **Per-user credential isolation:** each IDE maintains its own MCP session while the bridge multiplexes upstream calls. When you enable backend auth (via `CTXCE_AUTH_ENABLED` and `ctxce auth ...` sessions), uploads and MCP calls are gated by per-user sessions, so multiple IDEs can share the same stack while still having isolated access control and preferences. See **Optional auth with the MCP bridge (PoC)** below for details. + - **Flexible transport:** stdio mode works everywhere (even when HTTP ports aren’t reachable), while HTTP mode keeps Claude/Windsurf happy when they want direct URLs; the extension automatically writes the right flavor. + - **Centralized logging & health:** when the bridge process runs once per workspace you get a single stream of logs (`Context Engine Upload` output) and a single port to probe for health checks instead of multiple MCP child processes per IDE. +- When you run **`Write MCP Config`**, the extension: + - Writes `.mcp.json` in the workspace for Claude Code. + - Optionally writes Windsurf’s `mcp_config.json` (when `mcpWindsurfEnabled=true`). + - Optionally scaffolds `ctx_config.json` + `.env` (when `scaffoldCtxConfig=true`). +- The effective wiring mode is determined by the two MCP settings: + - `mcpServerMode = bridge`, `mcpTransportMode = sse-remote` → **bridge-stdio**. + - `mcpServerMode = bridge`, `mcpTransportMode = http` → **bridge-http**. + - `mcpServerMode = direct`, `mcpTransportMode = sse-remote` → **direct-sse** (two stdio `mcp-remote` servers). + - `mcpServerMode = direct`, `mcpTransportMode = http` → **direct-http** (two HTTP servers, no bridge). +- In **bridge-stdio**, the configs run the `ctxce mcp-serve` CLI via `npx` (for example, + `npx @context-engine-bridge/context-engine-mcp-bridge ctxce mcp-serve`), passing the + workspace path (auto-detected from the uploader target path) plus `--indexer-url` + and `--memory-url` derived from the MCP settings. +- In **bridge-http**, the extension can also **manage the bridge process**: + - `autoStartMcpBridge=true` and `mcpServerMode='bridge'` with `mcpTransportMode='http'` → the extension starts `ctxce mcp-http-serve` in the background for the active workspace using `mcpBridgePort`. + - The resulting HTTP URL (`http://127.0.0.1:/mcp`) is written into `.mcp.json` and Windsurf’s `mcp_config.json` as the `context-engine` server URL. + - In **stdio or direct modes**, the HTTP bridge is **not** auto-started; only the explicit `Start MCP HTTP Bridge` command will launch it. +- Bridge settings are **workspace-scoped**, so different workspaces can choose different modes and ports (e.g., one workspace using stdio bridge, another using HTTP bridge on a different port). + +Optional auth with the MCP bridge (PoC) +-------------------------------------- + +Auth is **off by default** and fully opt-in. When enabled, the MCP indexer and +memory servers expect a valid `session` id (issued by the backend) on protected +tools. The bridge CLI (`ctxce auth ...`) is the primary way to obtain and cache +that session. + +High-level steps: + +- Enable auth on the remote stack (e.g. dev-remote compose): + - Set `CTXCE_AUTH_ENABLED=1` in the upload/indexer environment. + - Optionally set `CTXCE_AUTH_SHARED_TOKEN` for token-based login. + - Optional: set `CTXCE_AUTH_ADMIN_TOKEN` for creating additional users via `/auth/users`. + - Optional (dev-only): set `CTXCE_AUTH_ALLOW_OPEN_TOKEN_LOGIN=1` **only** if you want `/auth/login` + to issue sessions even when `CTXCE_AUTH_SHARED_TOKEN` is unset. By default (`0`/unset), + token-based login is disabled when no shared token is configured. +- Point the bridge at the auth backend: + - In your local shell (where you run the `ctxce` CLI via `npx`), you can either: + - Explicitly set `CTXCE_AUTH_BACKEND_URL` to the upload service URL (e.g. `http://localhost:8004`), or + - Let the CLI discover the backend URL in this order when you run `ctxce auth login`: + `--backend-url` / `--auth-url` → `CTXCE_AUTH_BACKEND_URL` → any stored auth entry → + the uploader's `upload_endpoint` (`CTXCE_UPLOAD_ENDPOINT` / `UPLOAD_ENDPOINT`) → + `http://localhost:8004`. + +Token-based login: + +```bash +export CTXCE_AUTH_BACKEND_URL=http://localhost:8004 # optional when using this extension +export CTXCE_AUTH_TOKEN=change-me-dev-token # must match CTXCE_AUTH_SHARED_TOKEN in the stack + +# Obtain a session and cache it under ~/.ctxce/auth.json +npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth login + +# Check status (optional) +npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth status +``` + +Username/password login (when you have real users): + +- First, create the initial user (once) via `/auth/users` while the auth DB is + empty (no admin token required). This is typically done with a small script + or curl call against the upload service, for example: + + ```bash + curl -X POST http://localhost:8004/auth/users \ + -H "Content-Type: application/json" \ + -d '{"username":"you@example.com","password":"your-password"}' + ``` + +- Then login via the bridge: + +```bash +npx @context-engine-bridge/context-engine-mcp-bridge ctxce auth login \ + --username you@example.com \ + --password 'your-password' +``` + +In both modes, the bridge stores the returned `session_id` keyed by +`CTXCE_AUTH_BACKEND_URL` and automatically injects it into all MCP tool calls +as the `session` field. Once you have at least one entry in `~/.ctxce/auth.json`, +the MCP bridge used by the extension will discover and reuse that session +automatically; MCP configs do not need to set `CTXCE_AUTH_BACKEND_URL` in their +`env` blocks. If `CTXCE_AUTH_ENABLED` is off on the backend, these auth settings +are ignored and the bridge behaves exactly as before. + +Session lifetime: + +- By default, issued sessions do **not** expire (`CTXCE_AUTH_SESSION_TTL_SECONDS` + defaults to `0`). +- Operators who want expiry can set `CTXCE_AUTH_SESSION_TTL_SECONDS` (in seconds) + on the backend services. Values `> 0` enable a sliding window: active sessions + are refreshed when validated; values `<= 0` disable expiry. + Workspace-level ctx integration ------------------------------- - The VSIX bundles an `env.example` template plus the ctx hook/CLI so you can dogfood the workflow without copying files manually. @@ -51,9 +166,11 @@ Commands -------- - Command Palette → “Context Engine Uploader” exposes Start/Stop/Restart/Index Codebase and Prompt+ (unicorn) rewrite commands. - Status-bar button (`Index Codebase`) mirrors Start/Stop/Restart/Index status, while the `Prompt+` status button runs the ctx rewrite command on the current selection. -- `Context Engine Uploader: Write MCP Config (.mcp.json)` writes or updates a project-local `.mcp.json` with MCP server entries for the Qdrant indexer and memory/search endpoints, using the configured MCP URLs. +- `Context Engine Uploader: Write MCP Config (.mcp.json)` writes or updates a project-local `.mcp.json` (plus Windsurf `mcp_config.json` when enabled) using the currently selected bridge/direct + transport modes. If bridge-http is required and not yet running, the extension starts `ctxce mcp-http-serve` before writing configs. - `Context Engine Uploader: Write CTX Config (ctx_config.json/.env)` scaffolds the ctx config + env files as described above. This command runs automatically after `Write MCP Config` if scaffolding is enabled, but it is also exposed in the Command Palette for manual use. - `Context Engine Uploader: Upload Git History (force sync bundle)` triggers a one-off force sync using the configured git history settings, producing a bundle that includes a `metadata/git_history.json` manifest for remote lineage ingestion. +- `Context Engine Uploader: Start MCP HTTP Bridge` launches `ctxce mcp-http-serve` using the workspace’s resolved target path, MCP URLs, and configured `mcpBridgePort`. Use this when you want to run the HTTP bridge manually (e.g., testing unpublished builds or sharing a port across IDEs). +- `Context Engine Uploader: Stop MCP HTTP Bridge` gracefully terminates a running HTTP bridge process. Logs ---- diff --git a/vscode-extension/context-engine-uploader/auth_utils.js b/vscode-extension/context-engine-uploader/auth_utils.js new file mode 100644 index 00000000..319825ed --- /dev/null +++ b/vscode-extension/context-engine-uploader/auth_utils.js @@ -0,0 +1,234 @@ +const process = require('process'); + +const _skippedAuthCombos = new Set(); + +function getFetch(deps) { + if (deps && typeof deps.fetchGlobal === 'function') { + return deps.fetchGlobal; + } + try { + if (typeof fetch === 'function') { + return fetch; + } + } catch (_) { + } + return null; +} + +async function ensureAuthIfRequired(endpoint, deps) { + try { + if (!deps || !deps.vscode || !deps.spawnSync || !deps.resolveBridgeCliInvocation || !deps.getWorkspaceFolderPath || !deps.log) { + return; + } + const { vscode, spawnSync, resolveBridgeCliInvocation, getWorkspaceFolderPath, log } = deps; + const fetchFn = getFetch(deps); + const raw = (endpoint || '').trim(); + if (!raw) { + return; + } + if (!fetchFn) { + log('Auth status probe skipped: fetch is not available in this runtime.'); + return; + } + + let baseUrl = raw; + try { + const u = new URL(raw); + baseUrl = `${u.protocol}//${u.host}`; + } catch (_) { + baseUrl = raw.replace(/\/+$/, ''); + } + const statusUrl = `${baseUrl.replace(/\/+$/, '')}/auth/status`; + + let res; + try { + res = await fetchFn(statusUrl, { method: 'GET' }); + } catch (error) { + log(`Auth status probe failed: ${error instanceof Error ? error.message : String(error)}`); + return; + } + if (!res || !res.ok) { + return; + } + + let json; + try { + json = await res.json(); + } catch (_) { + return; + } + if (!json || !json.enabled) { + return; + } + + const invocation = resolveBridgeCliInvocation(); + if (!invocation) { + log('Context Engine Uploader: ctxce CLI not found; skipping auth status check.'); + return; + } + + const backendUrl = baseUrl; + const workspacePath = (typeof getWorkspaceFolderPath === 'function' && getWorkspaceFolderPath()) || ''; + const skipKey = `${backendUrl}::${workspacePath}`; + if (_skippedAuthCombos.has(skipKey)) { + return; + } + + const args = [...invocation.args, 'auth', 'status', '--json', '--backend-url', backendUrl]; + let result; + try { + result = spawnSync(invocation.command, args, { + cwd: getWorkspaceFolderPath() || process.cwd(), + env: { + ...process.env, + CTXCE_AUTH_BACKEND_URL: backendUrl, + }, + encoding: 'utf8', + }); + } catch (error) { + log(`Auth status check failed to run: ${error instanceof Error ? error.message : String(error)}`); + return; + } + + const stdout = (result && result.stdout) || ''; + let parsed; + try { + parsed = stdout ? JSON.parse(stdout) : null; + } catch (_) { + parsed = null; + } + const state = parsed && typeof parsed.state === 'string' ? parsed.state : undefined; + const exitCode = result && typeof result.status === 'number' ? result.status : undefined; + log(`Context Engine Uploader: auth status JSON state=${state || ''} exitCode=${exitCode !== undefined ? exitCode : ''}`); + if (state === 'ok' || result.status === 0) { + return; + } + + const choice = await vscode.window.showInformationMessage( + 'Context Engine: authentication is enabled on the backend but no valid session is available.', + 'Sign In', + 'Skip for now', + ); + if (choice !== 'Sign In') { + _skippedAuthCombos.add(skipKey); + return; + } + + await runAuthLoginFlow(backendUrl, deps); + } catch (error) { + if (deps && typeof deps.log === 'function') { + deps.log(`ensureAuthIfRequired error: ${error instanceof Error ? error.message : String(error)}`); + } + } +} + +async function runAuthLoginFlow(explicitBackendUrl, deps) { + if (!deps || !deps.vscode || !deps.spawn || !deps.resolveBridgeCliInvocation || !deps.getWorkspaceFolderPath || !deps.attachOutput || !deps.log) { + return; + } + const { vscode, spawn, resolveBridgeCliInvocation, getWorkspaceFolderPath, attachOutput, log } = deps; + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + let endpoint = (settings.get('endpoint') || '').trim(); + let backendUrl = explicitBackendUrl || endpoint; + if (!backendUrl) { + vscode.window.showErrorMessage('Context Engine Uploader: backend endpoint is not configured (contextEngineUploader.endpoint).'); + return; + } + + try { + const u = new URL(backendUrl); + backendUrl = `${u.protocol}//${u.host}`; + } catch (_) { + backendUrl = backendUrl.replace(/\/+$/, ''); + } + + const mode = await vscode.window.showQuickPick( + ['Token (shared dev token)', 'Username / password'], + { placeHolder: 'Select Context Engine auth method' }, + ); + if (!mode) { + return; + } + + const invocation = resolveBridgeCliInvocation(); + if (!invocation) { + vscode.window.showErrorMessage('Context Engine Uploader: unable to locate ctxce CLI for auth.'); + return; + } + const cwd = getWorkspaceFolderPath() || process.cwd(); + + if (mode.startsWith('Token')) { + const token = await vscode.window.showInputBox({ + prompt: 'Enter Context Engine shared auth token', + password: true, + ignoreFocusOut: true, + }); + if (!token) { + return; + } + const args = [...invocation.args, 'auth', 'login']; + const env = { + ...process.env, + CTXCE_AUTH_BACKEND_URL: backendUrl, + CTXCE_AUTH_TOKEN: token, + }; + await new Promise(resolve => { + const child = spawn(invocation.command, args, { cwd, env }); + attachOutput(child, 'auth'); + child.on('error', error => { + log(`ctxce auth login (token) failed to start: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: auth login failed to start. See output for details.'); + resolve(); + }); + child.on('close', code => { + if (code === 0) { + vscode.window.showInformationMessage('Context Engine Uploader: auth login successful.'); + } else { + vscode.window.showErrorMessage(`Context Engine Uploader: auth login failed with exit code ${code}. See output for details.`); + } + resolve(); + }); + }); + return; + } + + const username = await vscode.window.showInputBox({ + prompt: 'Enter Context Engine username', + ignoreFocusOut: true, + }); + if (!username) { + return; + } + const password = await vscode.window.showInputBox({ + prompt: 'Enter Context Engine password', + password: true, + ignoreFocusOut: true, + }); + if (!password) { + return; + } + + const args = [...invocation.args, 'auth', 'login', '--backend-url', backendUrl, '--username', username, '--password', password]; + await new Promise(resolve => { + const child = spawn(invocation.command, args, { cwd, env: process.env }); + attachOutput(child, 'auth'); + child.on('error', error => { + log(`ctxce auth login failed to start: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: auth login failed to start. See output for details.'); + resolve(); + }); + child.on('close', code => { + if (code === 0) { + vscode.window.showInformationMessage('Context Engine Uploader: auth login successful.'); + } else { + vscode.window.showErrorMessage(`Context Engine Uploader: auth login failed with exit code ${code}. See output for details.`); + } + resolve(); + }); + }); +} + +module.exports = { + ensureAuthIfRequired, + runAuthLoginFlow, +}; diff --git a/vscode-extension/context-engine-uploader/extension.js b/vscode-extension/context-engine-uploader/extension.js index df469fa4..4ef1e562 100644 --- a/vscode-extension/context-engine-uploader/extension.js +++ b/vscode-extension/context-engine-uploader/extension.js @@ -3,6 +3,7 @@ const { spawn, spawnSync } = require('child_process'); const path = require('path'); const fs = require('fs'); const os = require('os'); +const { ensureAuthIfRequired, runAuthLoginFlow } = require('./auth_utils'); let outputChannel; let watchProcess; let forceProcess; @@ -17,6 +18,10 @@ let watchedTargetPath; let indexedWatchDisposables = []; let globalStoragePath; let pythonOverridePath; +let httpBridgeProcess; +let httpBridgePort; +let httpBridgeWorkspace; +let pendingBridgeConfigTimer; const REQUIRED_PYTHON_MODULES = ['requests', 'urllib3', 'charset_normalizer']; const DEFAULT_CONTAINER_ROOT = '/work'; // const CLAUDE_HOOK_COMMAND = '/home/coder/project/Context-Engine/ctx-hook-simple.sh'; @@ -83,12 +88,29 @@ function activate(context) { const showLogsDisposable = vscode.commands.registerCommand('contextEngineUploader.showUploadServiceLogs', () => { try { openUploadServiceLogsTerminal(); } catch (e) { log(`Show logs failed: ${e && e.message ? e.message : String(e)}`); } }); + const startBridgeDisposable = vscode.commands.registerCommand('contextEngineUploader.startMcpHttpBridge', () => { + startHttpBridgeProcess().catch(error => { + log(`HTTP MCP bridge start failed: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: failed to start HTTP MCP bridge. Check Output for details.'); + }); + }); + const stopBridgeDisposable = vscode.commands.registerCommand('contextEngineUploader.stopMcpHttpBridge', () => { + stopHttpBridgeProcess().catch(error => { + log(`HTTP MCP bridge stop failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }); const promptEnhanceDisposable = vscode.commands.registerCommand('contextEngineUploader.promptEnhance', () => { enhanceSelectionWithUnicorn().catch(error => { log(`Prompt+ failed: ${error instanceof Error ? error.message : String(error)}`); vscode.window.showErrorMessage('Prompt+ failed. See Context Engine Upload output.'); }); }); + const authLoginDisposable = vscode.commands.registerCommand('contextEngineUploader.authLogin', () => { + runAuthLoginFlow(undefined, buildAuthDeps()).catch(error => { + log(`Auth login failed: ${error instanceof Error ? error.message : String(error)}`); + vscode.window.showErrorMessage('Context Engine Uploader: auth login failed. See output for details.'); + }); + }); const configDisposable = vscode.workspace.onDidChangeConfiguration(event => { if (event.affectsConfiguration('contextEngineUploader') && watchProcess) { runSequence('auto').catch(error => log(`Auto-restart failed: ${error instanceof Error ? error.message : String(error)}`)); @@ -109,6 +131,18 @@ function activate(context) { // Best-effort auto-update of MCP + hook configurations when settings change writeMcpConfig().catch(error => log(`Auto MCP config write failed: ${error instanceof Error ? error.message : String(error)}`)); } + if ( + event.affectsConfiguration('contextEngineUploader.autoStartMcpBridge') || + event.affectsConfiguration('contextEngineUploader.mcpBridgePort') || + event.affectsConfiguration('contextEngineUploader.mcpBridgeBinPath') || + event.affectsConfiguration('contextEngineUploader.mcpBridgeLocalOnly') || + event.affectsConfiguration('contextEngineUploader.mcpIndexerUrl') || + event.affectsConfiguration('contextEngineUploader.mcpMemoryUrl') || + event.affectsConfiguration('contextEngineUploader.mcpServerMode') || + event.affectsConfiguration('contextEngineUploader.mcpTransportMode') + ) { + handleHttpBridgeSettingsChanged().catch(error => log(`HTTP MCP bridge restart failed: ${error instanceof Error ? error.message : String(error)}`)); + } }); const workspaceDisposable = vscode.workspace.onDidChangeWorkspaceFolders(() => { ensureTargetPathConfigured(); @@ -127,6 +161,9 @@ function activate(context) { uploadGitHistoryDisposable, showLogsDisposable, promptEnhanceDisposable, + authLoginDisposable, + startBridgeDisposable, + stopBridgeDisposable, mcpConfigDisposable, ctxConfigDisposable, configDisposable, @@ -146,18 +183,48 @@ function activate(context) { // Legacy behavior: scaffold ctx_config.json/.env directly when MCP auto-write is disabled writeCtxConfig().catch(error => log(`CTX config auto-scaffold on activation failed: ${error instanceof Error ? error.message : String(error)}`)); } + if (config.get('autoStartMcpBridge', false)) { + const transportModeRaw = config.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = config.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + if (requiresHttpBridge(serverMode, transportMode)) { + startHttpBridgeProcess().catch(error => log(`Auto-start HTTP MCP bridge failed: ${error instanceof Error ? error.message : String(error)}`)); + } else { + log('Context Engine Uploader: autoStartMcpBridge is enabled, but current MCP wiring does not use the HTTP bridge; skipping auto-start.'); + } + } +} +function buildAuthDeps() { + return { + vscode, + spawn, + spawnSync, + resolveBridgeCliInvocation, + getWorkspaceFolderPath, + attachOutput, + log, + fetchGlobal: (typeof fetch === 'function' ? fetch : undefined), + }; } async function runSequence(mode = 'auto') { const options = resolveOptions(); if (!options) { return; } + + try { + await ensureAuthIfRequired(options.endpoint, buildAuthDeps()); + } catch (error) { + log(`Auth preflight check failed: ${error instanceof Error ? error.message : String(error)}`); + } + const depsSatisfied = await ensurePythonDependencies(options.pythonPath); if (!depsSatisfied) { setStatusBarState('idle'); return; } - // Re-resolve options in case ensurePythonDependencies switched to a private venv interpreter + // Re-resolve options in case ensurePythonDependencies switched to a better interpreter const reoptions = resolveOptions(); if (reoptions) { Object.assign(options, reoptions); @@ -255,7 +322,7 @@ function resolveOptions() { startWatchAfterForce }; } -function getTargetPath(config) { +function resolveTargetPathFromConfig(config) { let inspected; try { if (typeof config.inspect === 'function') { @@ -265,6 +332,22 @@ function getTargetPath(config) { inspected = undefined; } let targetPath = (config.get('targetPath') || '').trim(); + const metadata = inspected || {}; + if (targetPath) { + return { path: targetPath, inspected: metadata }; + } + const folderPath = getWorkspaceFolderPath(); + if (!folderPath) { + return { path: undefined, inspected: metadata }; + } + const autoTarget = detectDefaultTargetPath(folderPath); + return { path: autoTarget, inspected: metadata, inferred: true }; +} + +function getTargetPath(config) { + const result = resolveTargetPathFromConfig(config); + const targetPath = result.path; + const inspected = result.inspected; if (inspected && targetPath) { let sourceLabel = 'default'; if (inspected.workspaceFolderValue !== undefined) { @@ -288,15 +371,9 @@ function getTargetPath(config) { updateStatusBarTooltip(targetPath); return targetPath; } - const folderPath = getWorkspaceFolderPath(); - if (!folderPath) { - vscode.window.showErrorMessage('Context Engine Uploader: open a folder or set contextEngineUploader.targetPath.'); - updateStatusBarTooltip(); - return undefined; - } - const autoTarget = detectDefaultTargetPath(folderPath); - updateStatusBarTooltip(autoTarget); - return autoTarget; + vscode.window.showErrorMessage('Context Engine Uploader: open a folder or set contextEngineUploader.targetPath.'); + updateStatusBarTooltip(); + return undefined; } function saveTargetPath(config, targetPath) { const hasWorkspace = vscode.workspace.workspaceFolders && vscode.workspace.workspaceFolders.length; @@ -374,6 +451,108 @@ function detectDefaultTargetPath(workspaceFolderPath) { return workspaceFolderPath; } } + +function resolveBridgeWorkspacePath() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const target = getTargetPath(settings); + if (target) { + return path.resolve(target); + } + } catch (error) { + log(`Context Engine Uploader: failed to resolve bridge workspace path via getTargetPath: ${error instanceof Error ? error.message : String(error)}`); + } + const fallbackFolder = getWorkspaceFolderPath(); + if (!fallbackFolder) { + return undefined; + } + try { + const autoTarget = detectDefaultTargetPath(fallbackFolder); + return autoTarget ? path.resolve(autoTarget) : path.resolve(fallbackFolder); + } catch (error) { + log(`Context Engine Uploader: failed fallback bridge workspace path detection: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } +} + +function scheduleMcpConfigRefreshAfterBridge(delayMs = 1500) { + try { + if (pendingBridgeConfigTimer) { + clearTimeout(pendingBridgeConfigTimer); + pendingBridgeConfigTimer = undefined; + } + // For bridge-http mode started by the extension, Windsurf needs the + // "context-engine" MCP server entry removed and then re-added once the + // HTTP bridge is ready. Best-effort removal happens immediately here; + // writeMcpConfig() below will re-write configs after the bridge comes up. + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const windsurfEnabled = settings.get('mcpWindsurfEnabled', false); + const transportModeRaw = settings.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = settings.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + if (windsurfEnabled && serverMode === 'bridge' && transportMode === 'http') { + removeContextEngineFromWindsurfConfig().catch(error => { + log(`Context Engine Uploader: failed to remove context-engine from Windsurf MCP config before HTTP bridge restart: ${error instanceof Error ? error.message : String(error)}`); + }); + } + } catch (error) { + log(`Context Engine Uploader: failed to prepare Windsurf MCP removal before HTTP bridge restart: ${error instanceof Error ? error.message : String(error)}`); + } + pendingBridgeConfigTimer = setTimeout(() => { + pendingBridgeConfigTimer = undefined; + log('Context Engine Uploader: HTTP bridge ready; refreshing MCP configs.'); + writeMcpConfig().catch(error => { + log(`Context Engine Uploader: MCP config refresh after bridge start failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }, delayMs); + } catch (error) { + log(`Context Engine Uploader: failed to schedule MCP config refresh: ${error instanceof Error ? error.message : String(error)}`); + } +} + +async function removeContextEngineFromWindsurfConfig() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const customPath = (settings.get('windsurfMcpPath') || '').trim(); + const configPath = customPath || getDefaultWindsurfMcpPath(); + if (!configPath) { + return; + } + if (!fs.existsSync(configPath)) { + // Nothing to remove yet. + return; + } + let config = { mcpServers: {} }; + try { + const raw = fs.readFileSync(configPath, 'utf8'); + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + config = parsed; + } + } catch (error) { + log(`Context Engine Uploader: failed to parse Windsurf mcp_config.json when removing context-engine: ${error instanceof Error ? error.message : String(error)}`); + return; + } + if (!config.mcpServers || typeof config.mcpServers !== 'object') { + return; + } + if (!config.mcpServers['context-engine']) { + return; + } + delete config.mcpServers['context-engine']; + try { + const json = JSON.stringify(config, null, 2) + '\n'; + fs.writeFileSync(configPath, json, 'utf8'); + log(`Context Engine Uploader: removed context-engine server from Windsurf MCP config at ${configPath} before HTTP bridge restart.`); + } catch (error) { + log(`Context Engine Uploader: failed to write Windsurf mcp_config.json when removing context-engine: ${error instanceof Error ? error.message : String(error)}`); + } + } catch (error) { + log(`Context Engine Uploader: error while removing context-engine from Windsurf MCP config: ${error instanceof Error ? error.message : String(error)}`); + } +} function ensureTargetPathConfigured() { const config = vscode.workspace.getConfiguration('contextEngineUploader'); const current = (config.get('targetPath') || '').trim(); @@ -581,6 +760,23 @@ async function detectSystemPython() { } return undefined; } + +function requiresHttpBridge(serverMode, transportMode) { + return serverMode === 'bridge' && transportMode === 'http'; +} + +async function ensureHttpBridgeReadyForConfigs() { + try { + if (httpBridgeProcess) { + return true; + } + await startHttpBridgeProcess(); + return !!httpBridgeProcess; + } catch (error) { + log(`Failed to ensure HTTP bridge is ready: ${error instanceof Error ? error.message : String(error)}`); + return false; + } +} function setStatusBarState(mode) { if (!statusBarItem) { return; @@ -781,6 +977,25 @@ async function enhanceSelectionWithUnicorn() { if (decoderUrl) { env.DECODER_URL = decoderUrl; } + try { + const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); + const transportModeRaw = cfg.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = cfg.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + let idxUrlRaw = (cfg.get('ctxIndexerUrl') || cfg.get('mcpIndexerUrl') || '').trim(); + if (serverMode === 'bridge' && transportMode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + idxUrlRaw = bridgeUrl; + } + } + if (idxUrlRaw) { + env.MCP_INDEXER_URL = idxUrlRaw; + } + } catch (_) { + // ignore config read failures; fall back to defaults + } try { const cfg = vscode.workspace.getConfiguration('contextEngineUploader'); const useGpuDecoder = cfg.get('useGpuDecoder', false); @@ -914,7 +1129,7 @@ async function stopProcesses() { setStatusBarState('idle'); } } -function terminateProcess(proc, label) { +function terminateProcess(proc, label, afterStop) { if (!proc) { return Promise.resolve(); } @@ -922,14 +1137,17 @@ function terminateProcess(proc, label) { let finished = false; let termTimer; let killTimer; - const cleanup = () => { + const clearTimers = () => { if (termTimer) clearTimeout(termTimer); if (killTimer) clearTimeout(killTimer); }; const finalize = (reason) => { if (finished) return; finished = true; - cleanup(); + clearTimers(); + if (typeof afterStop === 'function') { + afterStop(); + } if (proc === forceProcess) { forceProcess = undefined; } @@ -1027,6 +1245,51 @@ function buildChildEnv(options) { } return env; } +function normalizeBridgeUrl(url) { + if (!url || typeof url !== 'string') { + return ''; + } + const trimmed = url.trim(); + if (!trimmed) { + return ''; + } + return trimmed; +} + +function normalizeWorkspaceForBridge(workspacePath) { + if (!workspacePath || typeof workspacePath !== 'string') { + return ''; + } + try { + const resolved = path.resolve(workspacePath); + if (process.platform === 'win32') { + return resolved.replace(/\//g, '\\'); + } + return resolved; + } catch (_) { + return workspacePath; + } +} + +function buildBridgeServerConfig(workspacePath, indexerUrl, memoryUrl) { + const invocation = resolveBridgeCliInvocation(); + const args = [...invocation.args, 'mcp-serve']; + if (workspacePath) { + args.push('--workspace', normalizeWorkspaceForBridge(workspacePath)); + } + if (indexerUrl) { + args.push('--indexer-url', indexerUrl); + } + if (memoryUrl) { + args.push('--memory-url', memoryUrl); + } + return { + command: invocation.command, + args, + env: {} + }; +} + async function writeMcpConfig() { const settings = vscode.workspace.getConfiguration('contextEngineUploader'); const claudeEnabled = settings.get('mcpClaudeEnabled', true); @@ -1039,9 +1302,39 @@ async function writeMcpConfig() { } const transportModeRaw = (settings.get('mcpTransportMode') || 'sse-remote'); const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverModeRaw = (settings.get('mcpServerMode') || 'bridge'); + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + const needsHttpBridge = requiresHttpBridge(serverMode, transportMode); + const bridgeWasRunning = !!httpBridgeProcess; + if (needsHttpBridge) { + const ready = await ensureHttpBridgeReadyForConfigs(); + if (!ready) { + vscode.window.showErrorMessage('Context Engine Uploader: HTTP MCP bridge failed to start; MCP config not updated.'); + return; + } + if (!bridgeWasRunning && httpBridgeProcess) { + log('Context Engine Uploader: HTTP MCP bridge launching; delaying MCP config write until bridge signals ready.'); + return; + } + } + const effectiveMode = + serverMode === 'bridge' + ? (transportMode === 'http' ? 'bridge-http' : 'bridge-stdio') + : (transportMode === 'http' ? 'direct-http' : 'direct-sse'); + log(`Context Engine Uploader: MCP wiring mode=${effectiveMode} (serverMode=${serverMode}, transportMode=${transportMode}).`); + if (effectiveMode === 'bridge-http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + log(`Context Engine Uploader: bridge HTTP endpoint ${bridgeUrl}`); + } + } - let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8001/sse').trim(); - let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8000/sse').trim(); + let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8003/mcp').trim(); + let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8002/mcp').trim(); + if (serverMode === 'bridge') { + indexerUrl = normalizeBridgeUrl(indexerUrl); + memoryUrl = normalizeBridgeUrl(memoryUrl); + } let wroteAny = false; let hookWrote = false; if (claudeEnabled) { @@ -1049,14 +1342,15 @@ async function writeMcpConfig() { if (!root) { vscode.window.showErrorMessage('Context Engine Uploader: open a folder before writing .mcp.json.'); } else { - const result = await writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode); + const result = await writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, serverMode); wroteAny = wroteAny || result; } } if (windsurfEnabled) { const customPath = (settings.get('windsurfMcpPath') || '').trim(); const windsPath = customPath || getDefaultWindsurfMcpPath(); - const result = await writeWindsurfMcpServers(windsPath, indexerUrl, memoryUrl, transportMode); + const workspaceHint = getWorkspaceFolderPath(); + const result = await writeWindsurfMcpServers(windsPath, indexerUrl, memoryUrl, transportMode, serverMode, workspaceHint); wroteAny = wroteAny || result; } if (claudeHookEnabled) { @@ -1076,11 +1370,6 @@ async function writeMcpConfig() { } } } - if (!wroteAny && !hookWrote) { - log('Context Engine Uploader: MCP config write skipped (no targets succeeded).'); - } - - // Optionally scaffold ctx_config.json and .env using the inferred collection if (settings.get('scaffoldCtxConfig', true)) { try { await writeCtxConfig(); @@ -1277,6 +1566,12 @@ async function scaffoldCtxConfigFiles(workspaceDir, collectionName) { ctxChanged = true; } } + if (decoderRuntime === 'llamacpp') { + if (ctxConfig.llamacpp_model === undefined) { + ctxConfig.llamacpp_model = 'llamacpp-4.6'; + ctxChanged = true; + } + } if (ctxChanged) { fs.writeFileSync(ctxConfigPath, JSON.stringify(ctxConfig, null, 2) + '\n', 'utf8'); if (notifiedDefault) { @@ -1442,9 +1737,19 @@ async function scaffoldCtxConfigFiles(workspaceDir, collectionName) { // Ensure MCP_INDEXER_URL is present based on extension setting (for ctx.py) if (uploaderSettings) { try { - const ctxIndexerUrl = (uploaderSettings.get('ctxIndexerUrl') || 'http://localhost:8003/mcp').trim(); - if (ctxIndexerUrl) { - upsertEnv('MCP_INDEXER_URL', ctxIndexerUrl, { treatEmptyAsUnset: true }); + const transportModeRaw = uploaderSettings.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = uploaderSettings.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + let targetUrl = (uploaderSettings.get('ctxIndexerUrl') || 'http://localhost:8003/mcp').trim(); + if (serverMode === 'bridge' && transportMode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + targetUrl = bridgeUrl; + } + } + if (targetUrl) { + upsertEnv('MCP_INDEXER_URL', targetUrl, { treatEmptyAsUnset: true }); } } catch (error) { log(`Failed to read ctxIndexerUrl setting for MCP_INDEXER_URL: ${error instanceof Error ? error.message : String(error)}`); @@ -1470,7 +1775,7 @@ async function scaffoldCtxConfigFiles(workspaceDir, collectionName) { } function deactivate() { disposeIndexedWatcher(); - return stopProcesses(); + return Promise.all([stopProcesses(), stopHttpBridgeProcess()]); } module.exports = { activate, @@ -1503,8 +1808,207 @@ function getDefaultWindsurfMcpPath() { return path.join(os.homedir(), '.codeium', 'windsurf', 'mcp_config.json'); } -async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode) { - const configPath = path.join(root, '.mcp.json'); +function resolveHttpBridgeOptions() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + const serverModeRaw = settings.get('mcpServerMode') || 'bridge'; + const transportModeRaw = settings.get('mcpTransportMode') || 'sse-remote'; + const serverMode = typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge'; + const transportMode = typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote'; + if (serverMode !== 'bridge') { + vscode.window.showWarningMessage('Context Engine Uploader: MCP server mode is not "bridge"; HTTP bridge will connect to raw endpoints.'); + } + if (transportMode !== 'http') { + log('Context Engine Uploader: MCP transport mode is not "http"; HTTP bridge will still start but downstream configs may expect SSE.'); + } + const workspacePath = resolveBridgeWorkspacePath(); + if (!workspacePath) { + vscode.window.showErrorMessage('Context Engine Uploader: open a workspace or set contextEngineUploader.targetPath before starting HTTP MCP bridge.'); + return undefined; + } + let indexerUrl = (settings.get('mcpIndexerUrl') || 'http://localhost:8003/mcp').trim(); + let memoryUrl = (settings.get('mcpMemoryUrl') || 'http://localhost:8002/mcp').trim(); + indexerUrl = normalizeBridgeUrl(indexerUrl); + memoryUrl = normalizeBridgeUrl(memoryUrl); + let port = Number(settings.get('mcpBridgePort') || 30810); + if (!Number.isFinite(port) || port <= 0) { + port = 30810; + } + return { + workspacePath, + indexerUrl, + memoryUrl, + port + }; + } catch (error) { + log(`Failed to resolve HTTP bridge options: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } +} + +function resolveBridgeHttpUrl() { + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + let port = Number(settings.get('mcpBridgePort') || 30810); + if (!Number.isFinite(port) || port <= 0) { + port = 30810; + } + const hostname = '127.0.0.1'; + return `http://${hostname}:${port}/mcp`; + } catch (error) { + log(`Failed to resolve bridge HTTP URL: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } +} + +async function startHttpBridgeProcess() { + if (httpBridgeProcess) { + vscode.window.showInformationMessage(`Context Engine HTTP MCP bridge already running on port ${httpBridgePort || 'unknown'}.`); + return httpBridgePort; + } + const options = resolveHttpBridgeOptions(); + if (!options) { + return undefined; + } + const invocation = resolveBridgeCliInvocation(); + if (!invocation) { + vscode.window.showErrorMessage('Context Engine Uploader: unable to locate ctxce CLI for HTTP bridge.'); + return undefined; + } + const cliArgs = ['mcp-http-serve']; + if (options.workspacePath) { + cliArgs.push('--workspace', normalizeWorkspaceForBridge(options.workspacePath)); + } + if (options.indexerUrl) { + cliArgs.push('--indexer-url', options.indexerUrl); + } + if (options.memoryUrl) { + cliArgs.push('--memory-url', options.memoryUrl); + } + if (options.port) { + cliArgs.push('--port', String(options.port)); + } + const finalArgs = [...invocation.args, ...cliArgs]; + log(`Starting HTTP MCP bridge via ${invocation.command} ${finalArgs.join(' ')}`); + const child = spawn(invocation.command, finalArgs, { + cwd: options.workspacePath, + env: process.env + }); + httpBridgeProcess = child; + httpBridgePort = options.port; + httpBridgeWorkspace = options.workspacePath; + attachOutput(child, 'mcp-http'); + child.on('exit', (code, signal) => { + log(`HTTP MCP bridge exited with code ${code} signal ${signal || ''}`.trim()); + if (httpBridgeProcess === child) { + httpBridgeProcess = undefined; + httpBridgePort = undefined; + httpBridgeWorkspace = undefined; + } + }); + child.on('error', error => { + log(`HTTP MCP bridge process error: ${error instanceof Error ? error.message : String(error)}`); + if (httpBridgeProcess === child) { + httpBridgeProcess = undefined; + httpBridgePort = undefined; + httpBridgeWorkspace = undefined; + } + }); + vscode.window.showInformationMessage(`Context Engine HTTP MCP bridge listening on http://127.0.0.1:${options.port}/mcp`); + scheduleMcpConfigRefreshAfterBridge(); + return options.port; +} + +function stopHttpBridgeProcess() { + if (!httpBridgeProcess) { + return Promise.resolve(); + } + return terminateProcess( + httpBridgeProcess, + 'mcp-http', + () => { + httpBridgeProcess = undefined; + httpBridgePort = undefined; + httpBridgeWorkspace = undefined; + } + ); +} + +async function handleHttpBridgeSettingsChanged() { + const config = vscode.workspace.getConfiguration('contextEngineUploader'); + const shouldRun = !!config.get('autoStartMcpBridge', false); + const wasRunning = !!httpBridgeProcess; + if (httpBridgeProcess) { + await stopHttpBridgeProcess(); + } + if (shouldRun || wasRunning) { + const transportModeRaw = config.get('mcpTransportMode') || 'sse-remote'; + const serverModeRaw = config.get('mcpServerMode') || 'bridge'; + const transportMode = (typeof transportModeRaw === 'string' ? transportModeRaw.trim() : 'sse-remote') || 'sse-remote'; + const serverMode = (typeof serverModeRaw === 'string' ? serverModeRaw.trim() : 'bridge') || 'bridge'; + if (requiresHttpBridge(serverMode, transportMode)) { + await startHttpBridgeProcess(); + } else { + log('Context Engine Uploader: HTTP bridge settings changed, but current MCP wiring does not use the HTTP bridge; not restarting HTTP bridge.'); + } + } +} + +function resolveBridgeCliInvocation() { + const binPath = findLocalBridgeBin(); + if (binPath) { + return { + command: 'node', + args: [binPath], + kind: 'local' + }; + } + const isWindows = process.platform === 'win32'; + if (isWindows) { + return { + command: 'cmd', + args: ['/c', 'npx', '@context-engine-bridge/context-engine-mcp-bridge'], + kind: 'npx' + }; + } + return { + command: 'npx', + args: ['@context-engine-bridge/context-engine-mcp-bridge'], + kind: 'npx' + }; +} + +function findLocalBridgeBin() { + let localOnly = true; + let configured = ''; + try { + const settings = vscode.workspace.getConfiguration('contextEngineUploader'); + localOnly = settings.get('mcpBridgeLocalOnly', true); + configured = (settings.get('mcpBridgeBinPath') || '').trim(); + } catch (_) { + // ignore config lookup failures and fall back to env/npx behavior + } + + // When local-only is disabled, skip local resolution and always fall back to npx + if (localOnly === false) { + return undefined; + } + + if (configured && fs.existsSync(configured)) { + return path.resolve(configured); + } + + const envOverride = (process.env.CTXCE_BRIDGE_BIN || '').trim(); + if (envOverride && fs.existsSync(envOverride)) { + return path.resolve(envOverride); + } + + return undefined; +} + +async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode, serverMode = 'bridge') { + const bridgeWorkspace = resolveBridgeWorkspacePath(); + const configPath = path.join(bridgeWorkspace || root, '.mcp.json'); let config = { mcpServers: {} }; if (fs.existsSync(configPath)) { try { @@ -1526,7 +2030,25 @@ async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode) const servers = config.mcpServers; const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; - if (mode === 'http') { + if (serverMode === 'bridge') { + if (mode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + servers['context-engine'] = { + type: 'http', + url: bridgeUrl + }; + } else { + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace || root, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } + } else { + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace || root, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } + delete servers['qdrant-indexer']; + delete servers.memory; + } else if (mode === 'http') { // Direct HTTP MCP endpoints for Claude (.mcp.json) if (indexerUrl) { servers['qdrant-indexer'] = { @@ -1577,7 +2099,7 @@ async function writeClaudeMcpServers(root, indexerUrl, memoryUrl, transportMode) } } -async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transportMode) { +async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transportMode, serverMode = 'bridge', workspaceHint) { try { fs.mkdirSync(path.dirname(configPath), { recursive: true }); } catch (error) { @@ -1606,7 +2128,25 @@ async function writeWindsurfMcpServers(configPath, indexerUrl, memoryUrl, transp const servers = config.mcpServers; const mode = (typeof transportMode === 'string' ? transportMode.trim() : 'sse-remote') || 'sse-remote'; - if (mode === 'http') { + if (serverMode === 'bridge') { + const bridgeWorkspace = resolveBridgeWorkspacePath() || workspaceHint || ''; + if (mode === 'http') { + const bridgeUrl = resolveBridgeHttpUrl(); + if (bridgeUrl) { + servers['context-engine'] = { + serverUrl: bridgeUrl + }; + } else { + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } + } else { + const bridgeServer = buildBridgeServerConfig(bridgeWorkspace, indexerUrl, memoryUrl); + servers['context-engine'] = bridgeServer; + } + delete servers['qdrant-indexer']; + delete servers.memory; + } else if (mode === 'http') { // Direct HTTP MCP endpoints for Windsurf mcp_config.json if (indexerUrl) { servers['qdrant-indexer'] = { diff --git a/vscode-extension/context-engine-uploader/package.json b/vscode-extension/context-engine-uploader/package.json index 9d6c06ba..9cff4cc2 100644 --- a/vscode-extension/context-engine-uploader/package.json +++ b/vscode-extension/context-engine-uploader/package.json @@ -2,7 +2,7 @@ "name": "context-engine-uploader", "displayName": "Context Engine Uploader", "description": "Runs the Context-Engine remote upload client with a force sync on startup followed by watch mode. Requires Python with pip install requests urllib3 charset_normalizer.", - "version": "0.1.31", + "version": "0.1.32", "publisher": "context-engine", "engines": { "vscode": "^1.85.0" @@ -15,7 +15,8 @@ "onCommand:contextEngineUploader.start", "onCommand:contextEngineUploader.stop", "onCommand:contextEngineUploader.restart", - "onCommand:contextEngineUploader.promptEnhance" + "onCommand:contextEngineUploader.promptEnhance", + "onCommand:contextEngineUploader.authLogin" ], "main": "./extension.js", "icon": "assets/icon.png", @@ -41,6 +42,14 @@ "command": "contextEngineUploader.uploadGitHistory", "title": "Context Engine Uploader: Upload Git History (force sync bundle)" }, + { + "command": "contextEngineUploader.startMcpHttpBridge", + "title": "Context Engine Uploader: Start MCP HTTP Bridge" + }, + { + "command": "contextEngineUploader.stopMcpHttpBridge", + "title": "Context Engine Uploader: Stop MCP HTTP Bridge" + }, { "command": "contextEngineUploader.writeMcpConfig", "title": "Context Engine Uploader: Write MCP Config (.mcp.json)" @@ -56,6 +65,10 @@ { "command": "contextEngineUploader.promptEnhance", "title": "Context Engine Uploader: Prompt+ (Unicorn Mode)" + }, + { + "command": "contextEngineUploader.authLogin", + "title": "Context Engine Uploader: Sign In (ctxce auth)" } ], "configuration": { @@ -164,17 +177,51 @@ "type": "string", "enum": ["sse-remote", "http"], "default": "http", - "description": "Transport mode for Claude/Windsurf MCP configs: SSE via mcp-remote (sse-remote) or direct HTTP /mcp endpoints (http)." + "description": "Transport layer for MCP servers. 'sse-remote' runs stdio MCP processes behind an SSE tunnel; 'http' uses direct HTTP MCP endpoints. Combined with mcpServerMode this yields four modes: bridge-stdio, bridge-http, direct-sse, direct-http.", + "enumDescriptions": [ + "Use stdio MCP processes behind an SSE tunnel (bridge-stdio / direct-sse).", + "Use HTTP MCP endpoints directly (bridge-http / direct-http)." + ] + }, + "contextEngineUploader.autoStartMcpBridge": { + "type": "boolean", + "default": true, + "description": "When enabled and mcpServerMode='bridge' with mcpTransportMode='http', automatically start the local ctx-mcp-bridge HTTP server for the active workspace so IDE clients can connect over HTTP without manual commands. Has no effect in stdio/direct modes." + }, + "contextEngineUploader.mcpBridgePort": { + "type": "number", + "default": 30810, + "description": "Port used for the local ctx-mcp-bridge HTTP server when auto-starting from the VS Code extension. Change if multiple IDE windows need parallel bridges." + }, + "contextEngineUploader.mcpBridgeBinPath": { + "type": "string", + "default": "", + "description": "Optional path to the ctxce CLI binary/script to use for the MCP bridge in local development. When set and the file exists, the extension runs this path via the current Node executable instead of using CTXCE_BRIDGE_BIN or 'npx @context-engine-bridge/context-engine-mcp-bridge'." + }, + "contextEngineUploader.mcpBridgeLocalOnly": { + "type": "boolean", + "default": false, + "description": "Development toggle. When true (default) the extension prefers local bridge binaries resolved from mcpBridgeBinPath or CTXCE_BRIDGE_BIN before falling back to the published npm build via npx." + }, + "contextEngineUploader.mcpServerMode": { + "type": "string", + "enum": ["bridge", "direct"], + "default": "bridge", + "description": "MCP wiring style. 'bridge' writes a single 'context-engine' server that uses the ctxce MCP bridge (stdio when mcpTransportMode='sse-remote', HTTP when 'http'). 'direct' writes two servers ('qdrant-indexer' and 'memory'), using stdio when mcpTransportMode='sse-remote' and HTTP when 'http'.", + "enumDescriptions": [ + "Single 'context-engine' server powered by the ctxce bridge (stdio or HTTP depending on mcpTransportMode).", + "Two separate servers: 'qdrant-indexer' and 'memory' (stdio or HTTP depending on mcpTransportMode)." + ] }, "contextEngineUploader.mcpIndexerUrl": { "type": "string", - "default": "http://localhost:8001/sse", - "description": "Claude Code MCP server URL for the Qdrant indexer. Used when writing the project-local .mcp.json via 'Write MCP Config'." + "default": "http://localhost:8003/mcp", + "description": "Claude Code MCP server URL for the Qdrant indexer. Used when writing MCP configs." }, "contextEngineUploader.mcpMemoryUrl": { "type": "string", - "default": "http://localhost:8000/sse", - "description": "Claude Code MCP server URL for the memory/search MCP server. Used when writing the project-local .mcp.json via 'Write MCP Config'." + "default": "http://localhost:8002/mcp", + "description": "Claude Code MCP server URL for the memory/search MCP server. Used when writing MCP configs." }, "contextEngineUploader.ctxIndexerUrl": { "type": "string",